From 455dac958c948f838204891f024e4b08676b4cce Mon Sep 17 00:00:00 2001 From: Ryan Fleury Date: Fri, 26 Jan 2024 11:47:25 -0800 Subject: [PATCH] first pass at asynchronous file streaming -> hash store layer; fix non-windowed memory access in watch window when doing change visualization, causing major performance regression with large arrays; fix some extra jank with truncated-string-hover & help labels --- src/df/gfx/df_gfx.c | 18 ++-- src/df/gfx/df_views.c | 15 +-- src/file_stream/file_stream.c | 192 ++++++++++++++++++++++++++++++++++ src/file_stream/file_stream.h | 84 +++++++++++++++ src/os/core/os_core.h | 1 + src/raddbg/raddbg.c | 1 + src/raddbg/raddbg.h | 25 +---- src/raddbg/raddbg_main.cpp | 2 + src/ui/ui_core.c | 4 + 9 files changed, 298 insertions(+), 44 deletions(-) create mode 100644 src/file_stream/file_stream.c create mode 100644 src/file_stream/file_stream.h diff --git a/src/df/gfx/df_gfx.c b/src/df/gfx/df_gfx.c index ae318415..0878b83e 100644 --- a/src/df/gfx/df_gfx.c +++ b/src/df/gfx/df_gfx.c @@ -10543,22 +10543,22 @@ internal B32 df_help_label(String8 string) { B32 result = 0; - UI_Box *box = ui_build_box_from_stringf(0, "###%S_help_label", string); - UI_Key help_hoverer_key = ui_key_from_stringf(box->key, "###help_hoverer"); - B32 box_is_hot = ((ui_key_match(ui_active_key(Side_Min), ui_key_zero()) || ui_key_match(ui_active_key(Side_Min), help_hoverer_key)) && - (ui_key_match(ui_active_key(Side_Max), ui_key_zero()) || ui_key_match(ui_active_key(Side_Min), help_hoverer_key)) && - contains_2f32(box->rect, ui_mouse())); + UI_Box *box = ui_build_box_from_stringf(UI_BoxFlag_Clickable, "###%S_help_label", string); + UI_Signal sig = ui_signal_from_box(box); UI_Parent(box) { UI_PrefWidth(ui_pct(1, 0)) ui_label(string); - if(box_is_hot) UI_PrefWidth(ui_em(2.25f, 1)) + if(sig.hovering) UI_PrefWidth(ui_em(2.25f, 1)) { + result = 1; ui_set_next_font(ui_icon_font()); ui_set_next_text_alignment(UI_TextAlign_Center); - UI_Box *help_hoverer = ui_build_box_from_key(UI_BoxFlag_DrawText|UI_BoxFlag_DrawBorder|UI_BoxFlag_DrawHotEffects|UI_BoxFlag_Clickable, help_hoverer_key); + UI_Box *help_hoverer = ui_build_box_from_stringf(UI_BoxFlag_DrawText|UI_BoxFlag_DrawBorder|UI_BoxFlag_DrawHotEffects, "###help_hoverer_%S", string); ui_box_equip_display_string(help_hoverer, df_g_icon_kind_text_table[DF_IconKind_QuestionMark]); - UI_Signal sig = ui_signal_from_box(help_hoverer); - result = sig.hovering; + if(!contains_2f32(help_hoverer->rect, ui_mouse())) + { + result = 0; + } } } return result; diff --git a/src/df/gfx/df_views.c b/src/df/gfx/df_views.c index 5956a31f..dd48d56d 100644 --- a/src/df/gfx/df_views.c +++ b/src/df/gfx/df_views.c @@ -984,6 +984,7 @@ df_eval_watch_view_build(DF_Window *ws, DF_Panel *panel, DF_View *view, DF_EvalW case EVAL_EvalMode_Addr: { U64 size = tg_byte_size_from_graph_raddbg_key(parse_ctx.type_graph, parse_ctx.rdbg, row->eval.type_key); + size = Min(size, 64); Rng1U64 vaddr_rng = r1u64(row->eval.offset, row->eval.offset+size); CTRL_ProcessMemorySlice slice = ctrl_query_cached_data_from_process_vaddr_range(scratch.arena, process->ctrl_machine_id, process->ctrl_handle, vaddr_rng); for(U64 idx = 0; idx < (size+63)/64; idx += 1) @@ -1243,19 +1244,7 @@ df_eval_watch_view_build(DF_Window *ws, DF_Panel *panel, DF_View *view, DF_EvalW } StringJoin join = {str8_lit(""), str8_lit(" "), str8_lit("")}; String8 error_string = str8_list_join(scratch.arena, &strings, &join); - sig = df_error_label(error_string); - if(sig.hovering) - { - UI_Tooltip - UI_Font(df_font_from_slot(DF_FontSlot_Main)) - UI_FontSize(df_font_size_from_slot(ws, DF_FontSlot_Main)) - { - for(EVAL_Error *error = row->errors.first; error != 0; error = error->next) - { - ui_label(error->text); - } - } - } + df_error_label(error_string); } // rjf: hook -> call hook diff --git a/src/file_stream/file_stream.c b/src/file_stream/file_stream.c new file mode 100644 index 00000000..f0688c5c --- /dev/null +++ b/src/file_stream/file_stream.c @@ -0,0 +1,192 @@ +// Copyright (c) 2024 Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +//////////////////////////////// +//~ rjf: Top-Level API + +internal void +fs_init(void) +{ + Arena *arena = arena_alloc(); + fs_shared = push_array(arena, FS_Shared, 1); + fs_shared->arena = arena; + fs_shared->slots_count = 1024; + fs_shared->stripes_count = 64; + fs_shared->slots = push_array(arena, FS_Slot, fs_shared->slots_count); + fs_shared->stripes = push_array(arena, FS_Stripe, fs_shared->stripes_count); + for(U64 idx = 0; idx < fs_shared->stripes_count; idx += 1) + { + fs_shared->stripes[idx].arena = arena_alloc(); + fs_shared->stripes[idx].cv = os_condition_variable_alloc(); + fs_shared->stripes[idx].rw_mutex = os_rw_mutex_alloc(); + } + fs_shared->u2s_ring_size = KB(64); + fs_shared->u2s_ring_base = push_array_no_zero(arena, U8, fs_shared->u2s_ring_size); + fs_shared->u2s_ring_cv = os_condition_variable_alloc(); + fs_shared->u2s_ring_mutex = os_mutex_alloc(); + fs_shared->streamer_count = Min(4, os_logical_core_count()-1); + fs_shared->streamers = push_array(arena, OS_Handle, 1); + for(U64 idx = 0; idx < fs_shared->streamer_count; idx += 1) + { + fs_shared->streamers[idx] = os_launch_thread(fs_streamer_thread__entry_point, &idx, 0); + } +} + +//////////////////////////////// +//~ rjf: Cache Interaction + +internal U128 +fs_hash_from_path(String8 path, U64 rewind_count, U64 endt_us) +{ + Temp scratch = scratch_begin(0, 0); + path = path_normalized_from_string(scratch.arena, path); + U128 path_key = hs_hash_from_data(path); + U128 result = hs_hash_from_key(path_key, rewind_count); + if(u128_match(result, u128_zero())) + { + U64 slot_idx = path_key.u64[0]%fs_shared->slots_count; + U64 stripe_idx = slot_idx%fs_shared->stripes_count; + FS_Slot *slot = &fs_shared->slots[slot_idx]; + FS_Stripe *stripe = &fs_shared->stripes[stripe_idx]; + OS_MutexScopeR(stripe->rw_mutex) for(;;) + { + FS_Node *node = 0; + for(FS_Node *n = slot->first; n != 0; n = n->next) + { + if(str8_match(path, n->path, 0)) + { + node = n; + break; + } + } + if(node == 0) OS_MutexScopeRWPromote(stripe->rw_mutex) + { + node = push_array(stripe->arena, FS_Node, 1); + SLLQueuePush(slot->first, slot->last, node); + node->path = push_str8_copy(stripe->arena, path); + } + if(os_now_microseconds() >= ins_atomic_u64_eval(&node->last_time_requested_us)+1000000 && + fs_u2s_enqueue_path(path, endt_us)) + { + ins_atomic_u64_eval_assign(&node->last_time_requested_us, os_now_microseconds()); + } + result = hs_hash_from_key(path_key, rewind_count); + if(u128_match(result, u128_zero()) && os_now_microseconds() <= endt_us) + { + os_condition_variable_wait_rw_r(stripe->cv, stripe->rw_mutex, endt_us); + } + else + { + break; + } + } + } + scratch_end(scratch); + return result; +} + +//////////////////////////////// +//~ rjf: Streamer Threads + +internal B32 +fs_u2s_enqueue_path(String8 path, U64 endt_us) +{ + B32 result = 0; + path.size = Min(path.size, fs_shared->u2s_ring_size); + OS_MutexScope(fs_shared->u2s_ring_mutex) for(;;) + { + U64 unconsumed_size = fs_shared->u2s_ring_write_pos - fs_shared->u2s_ring_read_pos; + U64 available_size = fs_shared->u2s_ring_size - unconsumed_size; + if(available_size >= sizeof(U64) + path.size) + { + result = 1; + fs_shared->u2s_ring_write_pos += ring_write_struct(fs_shared->u2s_ring_base, fs_shared->u2s_ring_size, fs_shared->u2s_ring_write_pos, &path.size); + fs_shared->u2s_ring_write_pos += ring_write(fs_shared->u2s_ring_base, fs_shared->u2s_ring_size, fs_shared->u2s_ring_write_pos, path.str, path.size); + fs_shared->u2s_ring_write_pos += 7; + fs_shared->u2s_ring_write_pos -= fs_shared->u2s_ring_write_pos%8; + break; + } + os_condition_variable_wait(fs_shared->u2s_ring_cv, fs_shared->u2s_ring_mutex, endt_us); + } + if(result) + { + os_condition_variable_broadcast(fs_shared->u2s_ring_cv); + } + return result; +} + +internal String8 +fs_u2s_dequeue_path(Arena *arena) +{ + String8 path = {0}; + OS_MutexScope(fs_shared->u2s_ring_mutex) for(;;) + { + U64 unconsumed_size = fs_shared->u2s_ring_write_pos - fs_shared->u2s_ring_read_pos; + if(unconsumed_size >= sizeof(U64)) + { + fs_shared->u2s_ring_read_pos += ring_write_struct(fs_shared->u2s_ring_base, fs_shared->u2s_ring_size, fs_shared->u2s_ring_read_pos, &path.size); + path.str = push_array(arena, U8, path.size); + fs_shared->u2s_ring_read_pos += ring_write(fs_shared->u2s_ring_base, fs_shared->u2s_ring_size, fs_shared->u2s_ring_read_pos, path.str, path.size); + fs_shared->u2s_ring_read_pos += 7; + fs_shared->u2s_ring_read_pos -= fs_shared->u2s_ring_read_pos%8; + break; + } + os_condition_variable_wait(fs_shared->u2s_ring_cv, fs_shared->u2s_ring_mutex, max_U64); + } + os_condition_variable_broadcast(fs_shared->u2s_ring_cv); + return path; +} + +internal void +fs_streamer_thread__entry_point(void *p) +{ + TCTX tctx_; + tctx_init_and_equip(&tctx_); + ThreadName("[fs] streamer #%I64u", (U64)p); + for(;;) + { + Temp scratch = scratch_begin(0, 0); + String8 path = fs_u2s_dequeue_path(scratch.arena); + FileProperties pre_props = os_properties_from_file_path(path); + OS_Handle file = os_file_open(OS_AccessFlag_Read|OS_AccessFlag_ShareRead|OS_AccessFlag_ShareWrite, path); + U64 data_arena_size = pre_props.size+ARENA_HEADER_SIZE; + data_arena_size += KB(4)-1; + data_arena_size -= data_arena_size%KB(4); + Arena *data_arena = arena_alloc__sized(data_arena_size, data_arena_size); + String8 data = os_string_from_file_range(data_arena, file, r1u64(0, pre_props.size)); + os_file_close(file); + FileProperties post_props = os_properties_from_file_path(path); + if(pre_props.modified != post_props.modified) + { + arena_release(data_arena); + MemoryZeroStruct(&data); + } + else + { + U128 key = hs_hash_from_data(path); + hs_submit_data(key, &data_arena, data); + U64 slot_idx = key.u64[0]%fs_shared->slots_count; + U64 stripe_idx = slot_idx%fs_shared->stripes_count; + FS_Slot *slot = &fs_shared->slots[slot_idx]; + FS_Stripe *stripe = &fs_shared->stripes[stripe_idx]; + OS_MutexScopeW(stripe->rw_mutex) + { + FS_Node *node = 0; + for(FS_Node *n = slot->first; n != 0; n = n->next) + { + if(str8_match(n->path, path, 0)) + { + node = n; + break; + } + } + if(node != 0) + { + node->timestamp = post_props.modified; + } + } + os_condition_variable_broadcast(stripe->cv); + } + scratch_end(scratch); + } +} diff --git a/src/file_stream/file_stream.h b/src/file_stream/file_stream.h new file mode 100644 index 00000000..4d54a379 --- /dev/null +++ b/src/file_stream/file_stream.h @@ -0,0 +1,84 @@ +// Copyright (c) 2024 Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +#ifndef FILE_STREAM_H +#define FILE_STREAM_H + +//////////////////////////////// +//~ rjf: Per-Path Info Cache Types + +typedef struct FS_Node FS_Node; +struct FS_Node +{ + FS_Node *next; + String8 path; + U64 timestamp; + U64 last_time_requested_us; +}; + +typedef struct FS_Slot FS_Slot; +struct FS_Slot +{ + FS_Node *first; + FS_Node *last; +}; + +typedef struct FS_Stripe FS_Stripe; +struct FS_Stripe +{ + Arena *arena; + OS_Handle cv; + OS_Handle rw_mutex; +}; + +//////////////////////////////// +//~ rjf: Shared State Bundle + +typedef struct FS_Shared FS_Shared; +struct FS_Shared +{ + Arena *arena; + + // rjf: path info cache + U64 slots_count; + U64 stripes_count; + FS_Slot *slots; + FS_Stripe *stripes; + + // rjf: user -> streamer ring buffer + U64 u2s_ring_size; + U8 *u2s_ring_base; + U64 u2s_ring_write_pos; + U64 u2s_ring_read_pos; + OS_Handle u2s_ring_cv; + OS_Handle u2s_ring_mutex; + + // rjf: streamer threads + U64 streamer_count; + OS_Handle *streamers; +}; + +//////////////////////////////// +//~ rjf: Globals + +global FS_Shared *fs_shared = 0; + +//////////////////////////////// +//~ rjf: Top-Level API + +internal void fs_init(void); + +//////////////////////////////// +//~ rjf: Cache Interaction + +internal U128 fs_hash_from_path(String8 path, U64 rewind_count, U64 endt_us); + +//////////////////////////////// +//~ rjf: Streamer Threads + +internal B32 fs_u2s_enqueue_path(String8 path, U64 endt_us); +internal String8 fs_u2s_dequeue_path(Arena *arena); + +internal void fs_streamer_thread__entry_point(void *p); + +#endif // FILE_STREAM_H diff --git a/src/os/core/os_core.h b/src/os/core/os_core.h index 1f5c9d31..3ee15bd4 100644 --- a/src/os/core/os_core.h +++ b/src/os/core/os_core.h @@ -199,6 +199,7 @@ internal void os_condition_variable_broadcast(OS_Handle cv); #define OS_MutexScope(mutex) DeferLoop(os_mutex_take(mutex), os_mutex_drop(mutex)) #define OS_MutexScopeR(mutex) DeferLoop(os_rw_mutex_take_r(mutex), os_rw_mutex_drop_r(mutex)) #define OS_MutexScopeW(mutex) DeferLoop(os_rw_mutex_take_w(mutex), os_rw_mutex_drop_w(mutex)) +#define OS_MutexScopeRWPromote(mutex) DeferLoop((os_rw_mutex_drop_r(mutex), os_rw_mutex_take_w(mutex)), (os_rw_mutex_drop_w(mutex), os_rw_mutex_take_r(mutex))) //////////////////////////////// //~ rjf: @os_hooks Main Initialization API (Implemented Per-OS) diff --git a/src/raddbg/raddbg.c b/src/raddbg/raddbg.c index 1d27680b..9d4c6203 100644 --- a/src/raddbg/raddbg.c +++ b/src/raddbg/raddbg.c @@ -458,6 +458,7 @@ entry_point(int argc, char **argv) //- rjf: initialize stuff we depend on { hs_init(); + fs_init(); txt_init(); dbgi_init(); txti_init(); diff --git a/src/raddbg/raddbg.h b/src/raddbg/raddbg.h index 1ead77da..70136a6e 100644 --- a/src/raddbg/raddbg.h +++ b/src/raddbg/raddbg.h @@ -4,6 +4,8 @@ //////////////////////////////// //~ rjf: Frontend/UI Pass Tasks // +// [ ] committing needs to happen when navigating focus away for any reason +// // [ ] source view -> floating margin/line-nums // [ ] theme colors -> more explicit about e.g. opaque backgrounds vs. floating // & scrollbars etc. @@ -75,7 +77,6 @@ // support it? // // [ ] escaping in config files - breakpoint labels etc. -// [ ] focus changing between query bar & panel content via mouse // // [ ] visualize conversion failures // @@ -142,10 +143,6 @@ // (partial) matches as you type, so you can stop typing once it gets the // right function instead of having to type the entire function name. // -// [ ] Hovering over a source tab that is clipped should probably display the -// full thing that was in that tab (like the whole filename, etc.). Right -// now, hovering does nothing AFAICT. -// // [ ] ** I couldn't figure out how to really view threads in the debugger. // The only place I found a thread list was in "The Scheduler", but it // only lists threads by ID, which is hard to use. I can hover over them @@ -178,18 +175,6 @@ // [ ] I had to go into the user file to change the font. That should probably // be in the theme window? // -// [ ] The way the "commands" view worked was idiosyncratic. All the other -// views stay up, but that one goes away whenever I select a command for -// some reason. -// [ ] Also, I could not move the commands window anywhere AFAICT. It seems -// to just pop up over whatever window I currently have selected. This -// would make sense for a hotkey (which I assume is the way it was -// designed), but it seems like it should be permanent if you can select -// it from the View menu. -// [ ] If the command window is not wide enough, you cannot read the -// description of a command because it doesn't word-wrap, nor can you -// hover over it to get the description in a tooltip (AFAICT). -// // [ ] It'd be nice to have a "goto byte" option for source views, for jumping // to error messages that are byte-based instead of line-based. // @@ -230,15 +215,11 @@ // - place temp bp, attach "die on hit" flag or something like that? // [ ] auto-scroll output window // -// [ ] C++ single & multi inheritance member visualization in watch window +// [ ] C++ virtual inheritance member visualization in watch window //////////////////////////////// //~ rjf: Hot, Low Priority Tasks (UI Opinions, Less-Serious Jank, Preferences, Cleanup) // -// [ ] ** Directory picking is kind of busted, as it goes through the same -// path as file picking, and this doesn't give the user a clean path to -// actually pick a folder, just navigate with them -// // [ ] ** In the call stack, I would like to be able to click quickly and move // around the stack. Right now, you can do that with the first and third // column, but the second column drops down a context menu. Since right diff --git a/src/raddbg/raddbg_main.cpp b/src/raddbg/raddbg_main.cpp index d8474dcc..e441f418 100644 --- a/src/raddbg/raddbg_main.cpp +++ b/src/raddbg/raddbg_main.cpp @@ -6,6 +6,7 @@ #include "os/os_inc.h" #include "mdesk/mdesk.h" #include "hash_store/hash_store.h" +#include "file_stream/file_stream.h" #include "text_cache/text_cache.h" #include "path/path.h" #include "txti/txti.h" @@ -49,6 +50,7 @@ #include "os/os_inc.c" #include "mdesk/mdesk.c" #include "hash_store/hash_store.c" +#include "file_stream/file_stream.c" #include "text_cache/text_cache.c" #include "path/path.c" #include "txti/txti.c" diff --git a/src/ui/ui_core.c b/src/ui/ui_core.c index df90bd27..36dc4a6d 100644 --- a/src/ui/ui_core.c +++ b/src/ui/ui_core.c @@ -1335,6 +1335,10 @@ ui_end_build(void) goto break_all_hover_string; } } + if(b != box && contains_2f32(b->rect, ui_state->mouse)) + { + goto break_all_hover_string; + } } } break_all_hover_string:;