first pass at asynchronous file streaming -> hash store layer; fix non-windowed memory access in watch window when doing change visualization, causing major performance regression with large arrays; fix some extra jank with truncated-string-hover & help labels

src/df/gfx/df_gfx.c

@@ -10543,22 +10543,22 @@ internal B32
 df_help_label(String8 string)
 {
   B32 result = 0;
-  UI_Box *box = ui_build_box_from_stringf(0, "###%S_help_label", string);
-  UI_Key help_hoverer_key = ui_key_from_stringf(box->key, "###help_hoverer");
-  B32 box_is_hot = ((ui_key_match(ui_active_key(Side_Min), ui_key_zero()) || ui_key_match(ui_active_key(Side_Min), help_hoverer_key)) &&
-                    (ui_key_match(ui_active_key(Side_Max), ui_key_zero()) || ui_key_match(ui_active_key(Side_Min), help_hoverer_key)) &&
-                    contains_2f32(box->rect, ui_mouse()));
+  UI_Box *box = ui_build_box_from_stringf(UI_BoxFlag_Clickable, "###%S_help_label", string);
+  UI_Signal sig = ui_signal_from_box(box);
   UI_Parent(box)
   {
     UI_PrefWidth(ui_pct(1, 0)) ui_label(string);
-    if(box_is_hot) UI_PrefWidth(ui_em(2.25f, 1))
+    if(sig.hovering) UI_PrefWidth(ui_em(2.25f, 1))
     {
+      result = 1;
       ui_set_next_font(ui_icon_font());
       ui_set_next_text_alignment(UI_TextAlign_Center);
-      UI_Box *help_hoverer = ui_build_box_from_key(UI_BoxFlag_DrawText|UI_BoxFlag_DrawBorder|UI_BoxFlag_DrawHotEffects|UI_BoxFlag_Clickable, help_hoverer_key);
+      UI_Box *help_hoverer = ui_build_box_from_stringf(UI_BoxFlag_DrawText|UI_BoxFlag_DrawBorder|UI_BoxFlag_DrawHotEffects, "###help_hoverer_%S", string);
       ui_box_equip_display_string(help_hoverer, df_g_icon_kind_text_table[DF_IconKind_QuestionMark]);
-      UI_Signal sig = ui_signal_from_box(help_hoverer);
-      result = sig.hovering;
+      if(!contains_2f32(help_hoverer->rect, ui_mouse()))
+      {
+        result = 0;
+      }
     }
   }
   return result;

src/df/gfx/df_views.c

@@ -984,6 +984,7 @@ df_eval_watch_view_build(DF_Window *ws, DF_Panel *panel, DF_View *view, DF_EvalW
           case EVAL_EvalMode_Addr:
           {
             U64 size = tg_byte_size_from_graph_raddbg_key(parse_ctx.type_graph, parse_ctx.rdbg, row->eval.type_key);
+            size = Min(size, 64);
             Rng1U64 vaddr_rng = r1u64(row->eval.offset, row->eval.offset+size);
             CTRL_ProcessMemorySlice slice = ctrl_query_cached_data_from_process_vaddr_range(scratch.arena, process->ctrl_machine_id, process->ctrl_handle, vaddr_rng);
             for(U64 idx = 0; idx < (size+63)/64; idx += 1)
@@ -1243,19 +1244,7 @@ df_eval_watch_view_build(DF_Window *ws, DF_Panel *panel, DF_View *view, DF_EvalW
                   }
                   StringJoin join = {str8_lit(""), str8_lit(" "), str8_lit("")};
                   String8 error_string = str8_list_join(scratch.arena, &strings, &join);
-                  sig = df_error_label(error_string);
-                  if(sig.hovering)
-                  {
-                    UI_Tooltip
-                      UI_Font(df_font_from_slot(DF_FontSlot_Main))
-                      UI_FontSize(df_font_size_from_slot(ws, DF_FontSlot_Main))
-                    {
-                      for(EVAL_Error *error = row->errors.first; error != 0; error = error->next)
-                      {
-                        ui_label(error->text);
-                      }
-                    }
-                  }
+                  df_error_label(error_string);
                 }
                 
                 // rjf: hook -> call hook

src/file_stream/file_stream.c

@@ -0,0 +1,192 @@
+// Copyright (c) 2024 Epic Games Tools
+// Licensed under the MIT license (https://opensource.org/license/mit/)
+
+////////////////////////////////
+//~ rjf: Top-Level API
+
+internal void
+fs_init(void)
+{
+  Arena *arena = arena_alloc();
+  fs_shared = push_array(arena, FS_Shared, 1);
+  fs_shared->arena = arena;
+  fs_shared->slots_count = 1024;
+  fs_shared->stripes_count = 64;
+  fs_shared->slots = push_array(arena, FS_Slot, fs_shared->slots_count);
+  fs_shared->stripes = push_array(arena, FS_Stripe, fs_shared->stripes_count);
+  for(U64 idx = 0; idx < fs_shared->stripes_count; idx += 1)
+  {
+    fs_shared->stripes[idx].arena = arena_alloc();
+    fs_shared->stripes[idx].cv = os_condition_variable_alloc();
+    fs_shared->stripes[idx].rw_mutex = os_rw_mutex_alloc();
+  }
+  fs_shared->u2s_ring_size = KB(64);
+  fs_shared->u2s_ring_base = push_array_no_zero(arena, U8, fs_shared->u2s_ring_size);
+  fs_shared->u2s_ring_cv = os_condition_variable_alloc();
+  fs_shared->u2s_ring_mutex = os_mutex_alloc();
+  fs_shared->streamer_count = Min(4, os_logical_core_count()-1);
+  fs_shared->streamers = push_array(arena, OS_Handle, 1);
+  for(U64 idx = 0; idx < fs_shared->streamer_count; idx += 1)
+  {
+    fs_shared->streamers[idx] = os_launch_thread(fs_streamer_thread__entry_point, &idx, 0);
+  }
+}
+
+////////////////////////////////
+//~ rjf: Cache Interaction
+
+internal U128
+fs_hash_from_path(String8 path, U64 rewind_count, U64 endt_us)
+{
+  Temp scratch = scratch_begin(0, 0);
+  path = path_normalized_from_string(scratch.arena, path);
+  U128 path_key = hs_hash_from_data(path);
+  U128 result = hs_hash_from_key(path_key, rewind_count);
+  if(u128_match(result, u128_zero()))
+  {
+    U64 slot_idx = path_key.u64[0]%fs_shared->slots_count;
+    U64 stripe_idx = slot_idx%fs_shared->stripes_count;
+    FS_Slot *slot = &fs_shared->slots[slot_idx];
+    FS_Stripe *stripe = &fs_shared->stripes[stripe_idx];
+    OS_MutexScopeR(stripe->rw_mutex) for(;;)
+    {
+      FS_Node *node = 0;
+      for(FS_Node *n = slot->first; n != 0; n = n->next)
+      {
+        if(str8_match(path, n->path, 0))
+        {
+          node = n;
+          break;
+        }
+      }
+      if(node == 0) OS_MutexScopeRWPromote(stripe->rw_mutex)
+      {
+        node = push_array(stripe->arena, FS_Node, 1);
+        SLLQueuePush(slot->first, slot->last, node);
+        node->path = push_str8_copy(stripe->arena, path);
+      }
+      if(os_now_microseconds() >= ins_atomic_u64_eval(&node->last_time_requested_us)+1000000 &&
+         fs_u2s_enqueue_path(path, endt_us))
+      {
+        ins_atomic_u64_eval_assign(&node->last_time_requested_us, os_now_microseconds());
+      }
+      result = hs_hash_from_key(path_key, rewind_count);
+      if(u128_match(result, u128_zero()) && os_now_microseconds() <= endt_us)
+      {
+        os_condition_variable_wait_rw_r(stripe->cv, stripe->rw_mutex, endt_us);
+      }
+      else
+      {
+        break;
+      }
+    }
+  }
+  scratch_end(scratch);
+  return result;
+}
+
+////////////////////////////////
+//~ rjf: Streamer Threads
+
+internal B32
+fs_u2s_enqueue_path(String8 path, U64 endt_us)
+{
+  B32 result = 0;
+  path.size = Min(path.size, fs_shared->u2s_ring_size);
+  OS_MutexScope(fs_shared->u2s_ring_mutex) for(;;)
+  {
+    U64 unconsumed_size = fs_shared->u2s_ring_write_pos - fs_shared->u2s_ring_read_pos;
+    U64 available_size = fs_shared->u2s_ring_size - unconsumed_size;
+    if(available_size >= sizeof(U64) + path.size)
+    {
+      result = 1;
+      fs_shared->u2s_ring_write_pos += ring_write_struct(fs_shared->u2s_ring_base, fs_shared->u2s_ring_size, fs_shared->u2s_ring_write_pos, &path.size);
+      fs_shared->u2s_ring_write_pos += ring_write(fs_shared->u2s_ring_base, fs_shared->u2s_ring_size, fs_shared->u2s_ring_write_pos, path.str, path.size);
+      fs_shared->u2s_ring_write_pos += 7;
+      fs_shared->u2s_ring_write_pos -= fs_shared->u2s_ring_write_pos%8;
+      break;
+    }
+    os_condition_variable_wait(fs_shared->u2s_ring_cv, fs_shared->u2s_ring_mutex, endt_us);
+  }
+  if(result)
+  {
+    os_condition_variable_broadcast(fs_shared->u2s_ring_cv);
+  }
+  return result;
+}
+
+internal String8
+fs_u2s_dequeue_path(Arena *arena)
+{
+  String8 path = {0};
+  OS_MutexScope(fs_shared->u2s_ring_mutex) for(;;)
+  {
+    U64 unconsumed_size = fs_shared->u2s_ring_write_pos - fs_shared->u2s_ring_read_pos;
+    if(unconsumed_size >= sizeof(U64))
+    {
+      fs_shared->u2s_ring_read_pos += ring_write_struct(fs_shared->u2s_ring_base, fs_shared->u2s_ring_size, fs_shared->u2s_ring_read_pos, &path.size);
+      path.str = push_array(arena, U8, path.size);
+      fs_shared->u2s_ring_read_pos += ring_write(fs_shared->u2s_ring_base, fs_shared->u2s_ring_size, fs_shared->u2s_ring_read_pos, path.str, path.size);
+      fs_shared->u2s_ring_read_pos += 7;
+      fs_shared->u2s_ring_read_pos -= fs_shared->u2s_ring_read_pos%8;
+      break;
+    }
+    os_condition_variable_wait(fs_shared->u2s_ring_cv, fs_shared->u2s_ring_mutex, max_U64);
+  }
+  os_condition_variable_broadcast(fs_shared->u2s_ring_cv);
+  return path;
+}
+
+internal void
+fs_streamer_thread__entry_point(void *p)
+{
+  TCTX tctx_;
+  tctx_init_and_equip(&tctx_);
+  ThreadName("[fs] streamer #%I64u", (U64)p);
+  for(;;)
+  {
+    Temp scratch = scratch_begin(0, 0);
+    String8 path = fs_u2s_dequeue_path(scratch.arena);
+    FileProperties pre_props = os_properties_from_file_path(path);
+    OS_Handle file = os_file_open(OS_AccessFlag_Read|OS_AccessFlag_ShareRead|OS_AccessFlag_ShareWrite, path);
+    U64 data_arena_size = pre_props.size+ARENA_HEADER_SIZE;
+    data_arena_size += KB(4)-1;
+    data_arena_size -= data_arena_size%KB(4);
+    Arena *data_arena = arena_alloc__sized(data_arena_size, data_arena_size);
+    String8 data = os_string_from_file_range(data_arena, file, r1u64(0, pre_props.size));
+    os_file_close(file);
+    FileProperties post_props = os_properties_from_file_path(path);
+    if(pre_props.modified != post_props.modified)
+    {
+      arena_release(data_arena);
+      MemoryZeroStruct(&data);
+    }
+    else
+    {
+      U128 key = hs_hash_from_data(path);
+      hs_submit_data(key, &data_arena, data);
+      U64 slot_idx = key.u64[0]%fs_shared->slots_count;
+      U64 stripe_idx = slot_idx%fs_shared->stripes_count;
+      FS_Slot *slot = &fs_shared->slots[slot_idx];
+      FS_Stripe *stripe = &fs_shared->stripes[stripe_idx];
+      OS_MutexScopeW(stripe->rw_mutex)
+      {
+        FS_Node *node = 0;
+        for(FS_Node *n = slot->first; n != 0; n = n->next)
+        {
+          if(str8_match(n->path, path, 0))
+          {
+            node = n;
+            break;
+          }
+        }
+        if(node != 0)
+        {
+          node->timestamp = post_props.modified;
+        }
+      }
+      os_condition_variable_broadcast(stripe->cv);
+    }
+    scratch_end(scratch);
+  }
+}

src/file_stream/file_stream.h

@@ -0,0 +1,84 @@
+// Copyright (c) 2024 Epic Games Tools
+// Licensed under the MIT license (https://opensource.org/license/mit/)
+
+#ifndef FILE_STREAM_H
+#define FILE_STREAM_H
+
+////////////////////////////////
+//~ rjf: Per-Path Info Cache Types
+
+typedef struct FS_Node FS_Node;
+struct FS_Node
+{
+  FS_Node *next;
+  String8 path;
+  U64 timestamp;
+  U64 last_time_requested_us;
+};
+
+typedef struct FS_Slot FS_Slot;
+struct FS_Slot
+{
+  FS_Node *first;
+  FS_Node *last;
+};
+
+typedef struct FS_Stripe FS_Stripe;
+struct FS_Stripe
+{
+  Arena *arena;
+  OS_Handle cv;
+  OS_Handle rw_mutex;
+};
+
+////////////////////////////////
+//~ rjf: Shared State Bundle
+
+typedef struct FS_Shared FS_Shared;
+struct FS_Shared
+{
+  Arena *arena;
+  
+  // rjf: path info cache
+  U64 slots_count;
+  U64 stripes_count;
+  FS_Slot *slots;
+  FS_Stripe *stripes;
+  
+  // rjf: user -> streamer ring buffer
+  U64 u2s_ring_size;
+  U8 *u2s_ring_base;
+  U64 u2s_ring_write_pos;
+  U64 u2s_ring_read_pos;
+  OS_Handle u2s_ring_cv;
+  OS_Handle u2s_ring_mutex;
+  
+  // rjf: streamer threads
+  U64 streamer_count;
+  OS_Handle *streamers;
+};
+
+////////////////////////////////
+//~ rjf: Globals
+
+global FS_Shared *fs_shared = 0;
+
+////////////////////////////////
+//~ rjf: Top-Level API
+
+internal void fs_init(void);
+
+////////////////////////////////
+//~ rjf: Cache Interaction
+
+internal U128 fs_hash_from_path(String8 path, U64 rewind_count, U64 endt_us);
+
+////////////////////////////////
+//~ rjf: Streamer Threads
+
+internal B32 fs_u2s_enqueue_path(String8 path, U64 endt_us);
+internal String8 fs_u2s_dequeue_path(Arena *arena);
+
+internal void fs_streamer_thread__entry_point(void *p);
+
+#endif // FILE_STREAM_H

src/os/core/os_core.h

@@ -199,6 +199,7 @@ internal void os_condition_variable_broadcast(OS_Handle cv);
 #define OS_MutexScope(mutex) DeferLoop(os_mutex_take(mutex), os_mutex_drop(mutex))
 #define OS_MutexScopeR(mutex) DeferLoop(os_rw_mutex_take_r(mutex), os_rw_mutex_drop_r(mutex))
 #define OS_MutexScopeW(mutex) DeferLoop(os_rw_mutex_take_w(mutex), os_rw_mutex_drop_w(mutex))
+#define OS_MutexScopeRWPromote(mutex) DeferLoop((os_rw_mutex_drop_r(mutex), os_rw_mutex_take_w(mutex)), (os_rw_mutex_drop_w(mutex), os_rw_mutex_take_r(mutex)))
 
 ////////////////////////////////
 //~ rjf: @os_hooks Main Initialization API (Implemented Per-OS)

src/raddbg/raddbg.c

@@ -458,6 +458,7 @@ entry_point(int argc, char **argv)
       //- rjf: initialize stuff we depend on
       {
         hs_init();
+        fs_init();
         txt_init();
         dbgi_init();
         txti_init();

src/raddbg/raddbg.h

@@ -4,6 +4,8 @@
 ////////////////////////////////
 //~ rjf: Frontend/UI Pass Tasks
 //
+// [ ] committing needs to happen when navigating focus away for any reason
+//
 // [ ] source view -> floating margin/line-nums
 // [ ] theme colors -> more explicit about e.g. opaque backgrounds vs. floating
 //     & scrollbars etc.
@@ -75,7 +77,6 @@
 //     support it?
 //
 // [ ] escaping in config files - breakpoint labels etc.
-// [ ] focus changing between query bar & panel content via mouse
 //
 // [ ] visualize conversion failures
 //
@@ -142,10 +143,6 @@
 //      (partial) matches as you type, so you can stop typing once it gets the
 //      right function instead of having to type the entire function name.
 //
-//  [ ] Hovering over a source tab that is clipped should probably display the
-//      full thing that was in that tab (like the whole filename, etc.). Right
-//      now, hovering does nothing AFAICT.
-//
 //  [ ] ** I couldn't figure out how to really view threads in the debugger.
 //      The only place I found a thread list was in "The Scheduler", but it
 //      only lists threads by ID, which is hard to use. I can hover over them
@@ -178,18 +175,6 @@
 //  [ ] I had to go into the user file to change the font. That should probably
 //      be in the theme window?
 //
-//  [ ] The way the "commands" view worked was idiosyncratic. All the other
-//      views stay up, but that one goes away whenever I select a command for
-//      some reason.
-//   [ ] Also, I could not move the commands window anywhere AFAICT. It seems
-//       to just pop up over whatever window I currently have selected. This
-//       would make sense for a hotkey (which I assume is the way it was
-//       designed), but it seems like it should be permanent if you can select
-//       it from the View menu.
-//  [ ] If the command window is not wide enough, you cannot read the
-//      description of a command because it doesn't word-wrap, nor can you
-//      hover over it to get the description in a tooltip (AFAICT).
-//
 //  [ ] It'd be nice to have a "goto byte" option for source views, for jumping
 //      to error messages that are byte-based instead of line-based.
 //
@@ -230,15 +215,11 @@
 //     - place temp bp, attach "die on hit" flag or something like that?
 // [ ] auto-scroll output window
 //
-// [ ] C++ single & multi inheritance member visualization in watch window
+// [ ] C++ virtual inheritance member visualization in watch window
 
 ////////////////////////////////
 //~ rjf: Hot, Low Priority Tasks (UI Opinions, Less-Serious Jank, Preferences, Cleanup)
 //
-//  [ ] ** Directory picking is kind of busted, as it goes through the same
-//      path as file picking, and this doesn't give the user a clean path to
-//      actually pick a folder, just navigate with them
-//
 //  [ ] ** In the call stack, I would like to be able to click quickly and move
 //      around the stack. Right now, you can do that with the first and third
 //      column, but the second column drops down a context menu. Since right

src/raddbg/raddbg_main.cpp

@@ -6,6 +6,7 @@
 #include "os/os_inc.h"
 #include "mdesk/mdesk.h"
 #include "hash_store/hash_store.h"
+#include "file_stream/file_stream.h"
 #include "text_cache/text_cache.h"
 #include "path/path.h"
 #include "txti/txti.h"
@@ -49,6 +50,7 @@
 #include "os/os_inc.c"
 #include "mdesk/mdesk.c"
 #include "hash_store/hash_store.c"
+#include "file_stream/file_stream.c"
 #include "text_cache/text_cache.c"
 #include "path/path.c"
 #include "txti/txti.c"

src/ui/ui_core.c

@@ -1335,6 +1335,10 @@ ui_end_build(void)
             goto break_all_hover_string;
           }
         }
+        if(b != box && contains_2f32(b->rect, ui_state->mouse))
+        {
+          goto break_all_hover_string;
+        }
       }
     }
     break_all_hover_string:;