From 30f8e1675ecb8db5fa9aa54d8744db717bcf9efe Mon Sep 17 00:00:00 2001 From: Ryan Fleury Date: Thu, 23 May 2024 07:01:34 -0700 Subject: [PATCH] new multi-dbgi-capable fuzzy search layer first pass --- src/df/core/df_core.h | 2 +- src/df/gfx/df_gfx.c | 2 +- src/df/gfx/df_views.c | 55 ++-- src/df/gfx/df_views.h | 2 +- src/fuzzy_search/fuzzy_search.c | 523 +++++++++++++++++++++++++++++++- src/fuzzy_search/fuzzy_search.h | 114 +++++-- 6 files changed, 626 insertions(+), 72 deletions(-) diff --git a/src/df/core/df_core.h b/src/df/core/df_core.h index 49dbfa0d..a27f7416 100644 --- a/src/df/core/df_core.h +++ b/src/df/core/df_core.h @@ -1148,7 +1148,7 @@ struct DF_State // rjf: per-run caches U64 unwind_cache_reggen_idx; U64 unwind_cache_memgen_idx; - DF_RunUnwindCache unwind_caches[2]; + DF_RunUnwindCache unwind_caches[4]; U64 unwind_cache_gen; U64 tls_base_cache_reggen_idx; U64 tls_base_cache_memgen_idx; diff --git a/src/df/gfx/df_gfx.c b/src/df/gfx/df_gfx.c index 7d2a72ca..8b679140 100644 --- a/src/df/gfx/df_gfx.c +++ b/src/df/gfx/df_gfx.c @@ -8691,7 +8691,7 @@ df_eval_viz_windowed_row_list_from_viz_block_list(Arena *arena, DI_Scope *scope, for(U64 idx = visible_idx_range.min; idx < visible_idx_range.max; idx += 1) { // rjf: unpack info about this row - String8 name = push_str8f(arena, "Item %I64u", idx); // TODO(rjf): dbgi_fuzzy_item_string_from_rdi_target_element_idx(parse_ctx->rdi, block->dbgi_target, block->fzy_backing_items.v[idx].idx); + String8 name = fzy_item_string_from_rdi_target_element_idx(parse_ctx->rdi, block->fzy_target, block->fzy_backing_items.v[idx].idx); // rjf: get keys for this row DF_ExpandKey parent_key = block->parent_key; diff --git a/src/df/gfx/df_views.c b/src/df/gfx/df_views.c index 03867dd3..df97121f 100644 --- a/src/df/gfx/df_views.c +++ b/src/df/gfx/df_views.c @@ -543,7 +543,7 @@ df_watch_view_text_edit_state_from_pt(DF_WatchViewState *wv, DF_WatchViewPoint p //- rjf: windowed watch tree visualization (both single-line and multi-line) internal DF_EvalVizBlockList -df_eval_viz_block_list_from_watch_view_state(Arena *arena, DI_Scope *scope, DF_CtrlCtx *ctrl_ctx, EVAL_ParseCtx *parse_ctx, EVAL_String2ExprMap *macro_map, DF_View *view, DF_WatchViewState *ews) +df_eval_viz_block_list_from_watch_view_state(Arena *arena, DI_Scope *di_scope, FZY_Scope *fzy_scope, DF_CtrlCtx *ctrl_ctx, EVAL_ParseCtx *parse_ctx, EVAL_String2ExprMap *macro_map, DF_View *view, DF_WatchViewState *ews) { ProfBeginFunction(); Temp scratch = scratch_begin(&arena, 1); @@ -568,7 +568,7 @@ df_eval_viz_block_list_from_watch_view_state(Arena *arena, DI_Scope *scope, DF_C { DF_ExpandKey parent_key = df_parent_expand_key_from_eval_root(root); DF_ExpandKey key = df_expand_key_from_eval_root(root); - DF_EvalVizBlockList root_blocks = df_eval_viz_block_list_from_eval_view_expr_keys(arena, scope, ctrl_ctx, parse_ctx, macro_map, eval_view, root_expr_string, parent_key, key); + DF_EvalVizBlockList root_blocks = df_eval_viz_block_list_from_eval_view_expr_keys(arena, di_scope, ctrl_ctx, parse_ctx, macro_map, eval_view, root_expr_string, parent_key, key); df_eval_viz_block_list_concat__in_place(&blocks, &root_blocks); } } @@ -594,7 +594,7 @@ df_eval_viz_block_list_from_watch_view_state(Arena *arena, DI_Scope *scope, DF_C { DF_ExpandKey parent_key = df_expand_key_make(5381, 0); DF_ExpandKey key = df_expand_key_make(df_hash_from_expand_key(parent_key), num); - DF_EvalVizBlockList root_blocks = df_eval_viz_block_list_from_eval_view_expr_keys(arena, scope, ctrl_ctx, parse_ctx, macro_map, eval_view, root_expr_string, parent_key, key); + DF_EvalVizBlockList root_blocks = df_eval_viz_block_list_from_eval_view_expr_keys(arena, di_scope, ctrl_ctx, parse_ctx, macro_map, eval_view, root_expr_string, parent_key, key); df_eval_viz_block_list_concat__in_place(&blocks, &root_blocks); } } @@ -606,7 +606,7 @@ df_eval_viz_block_list_from_watch_view_state(Arena *arena, DI_Scope *scope, DF_C { DF_ExpandKey parent_key = df_expand_key_make(5381, 0); DF_ExpandKey key = df_expand_key_make(df_hash_from_expand_key(parent_key), num); - DF_EvalVizBlockList root_blocks = df_eval_viz_block_list_from_eval_view_expr_keys(arena, scope, ctrl_ctx, parse_ctx, macro_map, eval_view, root_expr_string, parent_key, key); + DF_EvalVizBlockList root_blocks = df_eval_viz_block_list_from_eval_view_expr_keys(arena, di_scope, ctrl_ctx, parse_ctx, macro_map, eval_view, root_expr_string, parent_key, key); df_eval_viz_block_list_concat__in_place(&blocks, &root_blocks); } } @@ -626,7 +626,7 @@ df_eval_viz_block_list_from_watch_view_state(Arena *arena, DI_Scope *scope, DF_C { DF_ExpandKey parent_key = df_expand_key_make(5381, 0); DF_ExpandKey key = df_expand_key_make(df_hash_from_expand_key(parent_key), num); - DF_EvalVizBlockList root_blocks = df_eval_viz_block_list_from_eval_view_expr_keys(arena, scope, ctrl_ctx, parse_ctx, macro_map, eval_view, root_expr_string, parent_key, key); + DF_EvalVizBlockList root_blocks = df_eval_viz_block_list_from_eval_view_expr_keys(arena, di_scope, ctrl_ctx, parse_ctx, macro_map, eval_view, root_expr_string, parent_key, key); df_eval_viz_block_list_concat__in_place(&blocks, &root_blocks); } } @@ -647,7 +647,6 @@ df_eval_viz_block_list_from_watch_view_state(Arena *arena, DI_Scope *scope, DF_C U64 thread_rip_unwind_vaddr = df_query_cached_rip_from_thread_unwind(thread, ctrl_ctx->unwind_count); DF_Entity *module = df_module_from_process_vaddr(process, thread_rip_unwind_vaddr); DF_Entity *dbgi = df_dbgi_from_module(module); - RDI_Parsed *rdi = df_rdi_from_dbgi(scope, dbgi); //- rjf: calculate top-level keys, expand root-level, grab root expansion node DF_ExpandKey parent_key = df_expand_key_make(5381, 0); @@ -658,8 +657,13 @@ df_eval_viz_block_list_from_watch_view_state(Arena *arena, DI_Scope *scope, DF_C //- rjf: query all filtered items from dbgi searching system U128 fuzzy_search_key = {(U64)view, df_hash_from_string(str8_struct(&view))}; B32 items_stale = 0; - // TODO(rjf) - FZY_ItemArray items = {0}; // dbgi_fuzzy_search_items_from_key_exe_query(scope, fuzzy_search_key, exe_path, filter, dbgi_target, os_now_microseconds()+100, &items_stale); + FZY_DbgiKey key = {df_full_path_from_entity(scratch.arena, dbgi), dbgi->timestamp}; + FZY_Params params = {fzy_target}; + { + params.dbgi_keys.count = 1; + params.dbgi_keys.v = &key; + } + FZY_ItemArray items = fzy_items_from_key_params_query(fzy_scope, fuzzy_search_key, ¶ms, filter, os_now_microseconds()+100, &items_stale); if(items_stale) { df_gfx_request_frame(); @@ -739,8 +743,7 @@ df_eval_viz_block_list_from_watch_view_state(Arena *arena, DI_Scope *scope, DF_C last_vb = df_eval_viz_block_split_and_continue(arena, &blocks, last_vb, sub_expand_item_idxs[sub_expand_idx]); // rjf: grab name for the expanded row - // TODO(rjf) - String8 name = {0}; // dbgi_fuzzy_item_string_from_rdi_target_element_idx(&dbgi->rdi, dbgi_target, sub_expand_keys[sub_expand_idx].child_num); + String8 name = fzy_item_string_from_rdi_target_element_idx(parse_ctx->rdi, fzy_target, sub_expand_keys[sub_expand_idx].child_num); // rjf: recurse for sub-expansion { @@ -752,8 +755,8 @@ df_eval_viz_block_list_from_watch_view_state(Arena *arena, DI_Scope *scope, DF_C df_cfg_table_push_unparsed_string(arena, &child_cfg, view_rule_string, DF_CfgSrc_User); } } - DF_Eval eval = df_eval_from_string(arena, scope, ctrl_ctx, parse_ctx, macro_map, name); - df_append_viz_blocks_for_parent__rec(arena, scope, eval_view, ctrl_ctx, parse_ctx, macro_map, parent_key, sub_expand_keys[sub_expand_idx], name, eval, 0, &child_cfg, 0, &blocks); + DF_Eval eval = df_eval_from_string(arena, di_scope, ctrl_ctx, parse_ctx, macro_map, name); + df_append_viz_blocks_for_parent__rec(arena, di_scope, eval_view, ctrl_ctx, parse_ctx, macro_map, parent_key, sub_expand_keys[sub_expand_idx], name, eval, 0, &child_cfg, 0, &blocks); } } df_eval_viz_block_end(&blocks, last_vb); @@ -817,7 +820,8 @@ internal void df_watch_view_build(DF_Window *ws, DF_Panel *panel, DF_View *view, DF_WatchViewState *ewv, B32 modifiable, U32 default_radix, Rng2F32 rect) { ProfBeginFunction(); - DI_Scope *scope = di_scope_open(); + DI_Scope *di_scope = di_scope_open(); + FZY_Scope *fzy_scope = fzy_scope_open(); Temp scratch = scratch_begin(0, 0); ////////////////////////////// @@ -837,7 +841,7 @@ df_watch_view_build(DF_Window *ws, DF_Panel *panel, DF_View *view, DF_WatchViewS ////////////////////////////// //- rjf: process * thread info -> parse_ctx // - EVAL_ParseCtx parse_ctx = df_eval_parse_ctx_from_process_vaddr(scope, process, thread_ip_vaddr); + EVAL_ParseCtx parse_ctx = df_eval_parse_ctx_from_process_vaddr(di_scope, process, thread_ip_vaddr); ////////////////////////////// //- rjf: determine autocompletion string @@ -912,7 +916,7 @@ df_watch_view_build(DF_Window *ws, DF_Panel *panel, DF_View *view, DF_WatchViewS // if(state_dirty) { - blocks = df_eval_viz_block_list_from_watch_view_state(scratch.arena, scope, &ctrl_ctx, &parse_ctx, ¯o_map, view, ewv); + blocks = df_eval_viz_block_list_from_watch_view_state(scratch.arena, di_scope, fzy_scope, &ctrl_ctx, &parse_ctx, ¯o_map, view, ewv); } ////////////////////////// @@ -1050,7 +1054,7 @@ df_watch_view_build(DF_Window *ws, DF_Panel *panel, DF_View *view, DF_WatchViewS ewv->text_edit_state_slots_count = u64_up_to_pow2(selection_dim.y+1); ewv->text_edit_state_slots_count = Max(ewv->text_edit_state_slots_count, 64); ewv->text_edit_state_slots = push_array(ewv->text_edit_arena, DF_WatchViewTextEditState*, ewv->text_edit_state_slots_count); - DF_EvalVizWindowedRowList rows = df_eval_viz_windowed_row_list_from_viz_block_list(scratch.arena, scope, &ctrl_ctx, &parse_ctx, ¯o_map, eval_view, default_radix, code_font, ui_top_font_size(), + DF_EvalVizWindowedRowList rows = df_eval_viz_windowed_row_list_from_viz_block_list(scratch.arena, di_scope, &ctrl_ctx, &parse_ctx, ¯o_map, eval_view, default_radix, code_font, ui_top_font_size(), r1s64(ui_scroll_list_row_from_item(&row_blocks, selection_tbl.min.y-1), ui_scroll_list_row_from_item(&row_blocks, selection_tbl.max.y-1)+1), &blocks); DF_EvalVizRow *row = rows.first; @@ -1082,7 +1086,7 @@ df_watch_view_build(DF_Window *ws, DF_Panel *panel, DF_View *view, DF_WatchViewS if(!ewv->text_editing && evt->slot == UI_EventActionSlot_Accept && selection_tbl.min.x <= 0) { taken = 1; - DF_EvalVizWindowedRowList rows = df_eval_viz_windowed_row_list_from_viz_block_list(scratch.arena, scope, &ctrl_ctx, &parse_ctx, ¯o_map, eval_view, default_radix, code_font, ui_top_font_size(), + DF_EvalVizWindowedRowList rows = df_eval_viz_windowed_row_list_from_viz_block_list(scratch.arena, di_scope, &ctrl_ctx, &parse_ctx, ¯o_map, eval_view, default_radix, code_font, ui_top_font_size(), r1s64(ui_scroll_list_row_from_item(&row_blocks, selection_tbl.min.y-1), ui_scroll_list_row_from_item(&row_blocks, selection_tbl.max.y-1)+1), &blocks); DF_EvalVizRow *row = rows.first; @@ -1181,13 +1185,13 @@ df_watch_view_build(DF_Window *ws, DF_Panel *panel, DF_View *view, DF_WatchViewS case DF_WatchViewColumnKind_Value: if(editing_complete && evt->slot != UI_EventActionSlot_Cancel) { - DF_EvalVizWindowedRowList rows = df_eval_viz_windowed_row_list_from_viz_block_list(scratch.arena, scope, &ctrl_ctx, &parse_ctx, ¯o_map, eval_view, default_radix, code_font, ui_top_font_size(), + DF_EvalVizWindowedRowList rows = df_eval_viz_windowed_row_list_from_viz_block_list(scratch.arena, di_scope, &ctrl_ctx, &parse_ctx, ¯o_map, eval_view, default_radix, code_font, ui_top_font_size(), r1s64(ui_scroll_list_row_from_item(&row_blocks, y-1), ui_scroll_list_row_from_item(&row_blocks, y-1)+1), &blocks); B32 success = 0; if(rows.first != 0) { - DF_Eval write_eval = df_eval_from_string(scratch.arena, scope, &ctrl_ctx, &parse_ctx, ¯o_map, new_string); + DF_Eval write_eval = df_eval_from_string(scratch.arena, di_scope, &ctrl_ctx, &parse_ctx, ¯o_map, new_string); success = df_commit_eval_value(parse_ctx.type_graph, parse_ctx.rdi, &ctrl_ctx, rows.first->eval, write_eval); } if(!success) @@ -1223,7 +1227,7 @@ df_watch_view_build(DF_Window *ws, DF_Panel *panel, DF_View *view, DF_WatchViewS { taken = 1; String8List strs = {0}; - DF_EvalVizWindowedRowList rows = df_eval_viz_windowed_row_list_from_viz_block_list(scratch.arena, scope, &ctrl_ctx, &parse_ctx, ¯o_map, eval_view, default_radix, code_font, ui_top_font_size(), + DF_EvalVizWindowedRowList rows = df_eval_viz_windowed_row_list_from_viz_block_list(scratch.arena, di_scope, &ctrl_ctx, &parse_ctx, ¯o_map, eval_view, default_radix, code_font, ui_top_font_size(), r1s64(ui_scroll_list_row_from_item(&row_blocks, selection_tbl.min.y-1), ui_scroll_list_row_from_item(&row_blocks, selection_tbl.max.y-1)+1), &blocks); DF_EvalVizRow *row = rows.first; @@ -1559,7 +1563,7 @@ df_watch_view_build(DF_Window *ws, DF_Panel *panel, DF_View *view, DF_WatchViewS //- rjf: viz blocks -> rows DF_EvalVizWindowedRowList rows = {0}; { - rows = df_eval_viz_windowed_row_list_from_viz_block_list(scratch.arena, scope, &ctrl_ctx, &parse_ctx, ¯o_map, eval_view, default_radix, code_font, ui_top_font_size(), r1s64(visible_row_rng.min-1, visible_row_rng.max), &blocks); + rows = df_eval_viz_windowed_row_list_from_viz_block_list(scratch.arena, di_scope, &ctrl_ctx, &parse_ctx, ¯o_map, eval_view, default_radix, code_font, ui_top_font_size(), r1s64(visible_row_rng.min-1, visible_row_rng.max), &blocks); } //- rjf: build table @@ -1624,7 +1628,7 @@ df_watch_view_build(DF_Window *ws, DF_Panel *panel, DF_View *view, DF_WatchViewS { Vec2F32 canvas_dim = v2f32(scroll_list_params.dim_px.x - ui_top_font_size()*1.5f, (row->skipped_size_in_rows+row->size_in_rows+row->chopped_size_in_rows)*scroll_list_params.row_height_px); - row->expand_ui_rule_spec->info.block_ui(ws, row->key, row->eval, row->edit_expr, scope, &ctrl_ctx, &parse_ctx, ¯o_map, row->expand_ui_rule_node, canvas_dim); + row->expand_ui_rule_spec->info.block_ui(ws, row->key, row->eval, row->edit_expr, di_scope, &ctrl_ctx, &parse_ctx, ¯o_map, row->expand_ui_rule_node, canvas_dim); } } } @@ -1899,7 +1903,7 @@ df_watch_view_build(DF_Window *ws, DF_Panel *panel, DF_View *view, DF_WatchViewS UI_Box *box = ui_build_box_from_stringf(UI_BoxFlag_Clip|UI_BoxFlag_Clickable, "###val_%I64x", row_hash); UI_Parent(box) { - row->value_ui_rule_spec->info.row_ui(row->key, row->eval, scope, &ctrl_ctx, &parse_ctx, ¯o_map, row->value_ui_rule_node); + row->value_ui_rule_spec->info.row_ui(row->key, row->eval, di_scope, &ctrl_ctx, &parse_ctx, ¯o_map, row->value_ui_rule_node); } sig = ui_signal_from_box(box); } @@ -2035,7 +2039,8 @@ df_watch_view_build(DF_Window *ws, DF_Panel *panel, DF_View *view, DF_WatchViewS } scratch_end(scratch); - di_scope_close(scope); + fzy_scope_close(fzy_scope); + di_scope_close(di_scope); ProfEnd(); } @@ -3251,7 +3256,7 @@ DF_VIEW_UI_FUNCTION_DEF(SymbolLister) //- rjf: query -> raddbg, filtered items U128 fuzzy_search_key = {(U64)view, df_hash_from_string(str8_struct(&view))}; B32 items_stale = 0; - // TODO(rjf) + // TODO(rjf): @fuzzy FZY_ItemArray items = {0}; // dbgi_fuzzy_search_items_from_key_exe_query(scope, fuzzy_search_key, exe_path, query, DBGI_FuzzySearchTarget_Procedures, os_now_microseconds()+100, &items_stale); if(items_stale) { diff --git a/src/df/gfx/df_views.h b/src/df/gfx/df_views.h index 961c85fc..80af1789 100644 --- a/src/df/gfx/df_views.h +++ b/src/df/gfx/df_views.h @@ -495,7 +495,7 @@ internal String8 df_string_from_eval_viz_row_column_kind(Arena *arena, DF_EvalVi internal DF_WatchViewTextEditState *df_watch_view_text_edit_state_from_pt(DF_WatchViewState *wv, DF_WatchViewPoint pt); //- rjf: windowed watch tree visualization -internal DF_EvalVizBlockList df_eval_viz_block_list_from_watch_view_state(Arena *arena, DI_Scope *scope, DF_CtrlCtx *ctrl_ctx, EVAL_ParseCtx *parse_ctx, EVAL_String2ExprMap *macro_map, DF_View *view, DF_WatchViewState *ews); +internal DF_EvalVizBlockList df_eval_viz_block_list_from_watch_view_state(Arena *arena, DI_Scope *di_scope, FZY_Scope *fzy_scope, DF_CtrlCtx *ctrl_ctx, EVAL_ParseCtx *parse_ctx, EVAL_String2ExprMap *macro_map, DF_View *view, DF_WatchViewState *ews); //- rjf: eval/watch views main hooks internal void df_watch_view_init(DF_WatchViewState *ewv, DF_View *view, DF_WatchViewFillKind fill_kind); diff --git a/src/fuzzy_search/fuzzy_search.c b/src/fuzzy_search/fuzzy_search.c index 0faa9c12..5d4ba473 100644 --- a/src/fuzzy_search/fuzzy_search.c +++ b/src/fuzzy_search/fuzzy_search.c @@ -5,15 +5,119 @@ //~ rjf: Helpers internal U64 -fzy_hash_from_string(String8 string, StringMatchFlags match_flags) +fzy_hash_from_string(U64 seed, String8 string) { - return 0; + U64 result = seed; + for(U64 i = 0; i < string.size; i += 1) + { + result = ((result << 5) + result) + string.str[i]; + } + return result; +} + +internal U64 +fzy_hash_from_params(FZY_Params *params) +{ + U64 hash = 5381; + hash = fzy_hash_from_string(hash, str8_struct(¶ms->target)); + for(U64 idx = 0; idx < params->dbgi_keys.count; idx += 1) + { + hash = fzy_hash_from_string(hash, str8_struct(¶ms->dbgi_keys.v[idx].timestamp)); + hash = fzy_hash_from_string(hash, params->dbgi_keys.v[idx].path); + } + return hash; } internal U64 fzy_item_num_from_array_element_idx__linear_search(FZY_ItemArray *array, U64 element_idx) { - return 0; + U64 fuzzy_item_num = 0; + for(U64 idx = 0; idx < array->count; idx += 1) + { + if(array->v[idx].idx == element_idx) + { + fuzzy_item_num = idx+1; + break; + } + } + return fuzzy_item_num; +} + +internal String8 +fzy_item_string_from_rdi_target_element_idx(RDI_Parsed *rdi, FZY_Target target, U64 element_idx) +{ + String8 result = {0}; + switch(target) + { + // NOTE(rjf): no default - warn if we miss a case + case FZY_Target_Procedures: + { + RDI_Procedure *proc = rdi_element_from_idx(rdi, procedures, element_idx); + U64 name_size = 0; + U8 *name_base = rdi_string_from_idx(rdi, proc->name_string_idx, &name_size); + result = str8(name_base, name_size); + }break; + case FZY_Target_GlobalVariables: + { + RDI_GlobalVariable *gvar = rdi_element_from_idx(rdi, global_variables, element_idx); + U64 name_size = 0; + U8 *name_base = rdi_string_from_idx(rdi, gvar->name_string_idx, &name_size); + result = str8(name_base, name_size); + }break; + case FZY_Target_ThreadVariables: + { + RDI_ThreadVariable *tvar = rdi_element_from_idx(rdi, thread_variables, element_idx); + U64 name_size = 0; + U8 *name_base = rdi_string_from_idx(rdi, tvar->name_string_idx, &name_size); + result = str8(name_base, name_size); + }break; + case FZY_Target_UDTs: + { + RDI_UDT *udt = rdi_element_from_idx(rdi, udts, element_idx); + RDI_TypeNode *type_node = rdi_element_from_idx(rdi, type_nodes, udt->self_type_idx); + U64 name_size = 0; + U8 *name_base = rdi_string_from_idx(rdi, type_node->user_defined.name_string_idx, &name_size); + result = str8(name_base, name_size); + }break; + case FZY_Target_COUNT:{}break; + } + return result; +} + +internal void +fzy_dbgi_key_list_push(Arena *arena, FZY_DbgiKeyList *list, FZY_DbgiKey key) +{ + FZY_DbgiKeyNode *n = push_array(arena, FZY_DbgiKeyNode, 1); + n->v = key; + SLLQueuePush(list->first, list->last, n); + list->count += 1; +} + +internal FZY_DbgiKeyArray +fzy_dbgi_key_array_from_list(Arena *arena, FZY_DbgiKeyList *list) +{ + FZY_DbgiKeyArray array = {0}; + array.v = push_array(arena, FZY_DbgiKey, list->count); + U64 idx = 0; + for(FZY_DbgiKeyNode *n = list->first; n != 0; n = n->next, idx += 1) + { + array.v[idx] = n->v; + } + return array; +} + +internal FZY_Params +fzy_params_copy(Arena *arena, FZY_Params *src) +{ + FZY_Params dst = {0}; + MemoryCopyStruct(&dst, src); + dst.dbgi_keys.v = push_array(arena, FZY_DbgiKey, dst.dbgi_keys.count); + MemoryCopy(dst.dbgi_keys.v, src->dbgi_keys.v, sizeof(FZY_DbgiKey)*src->dbgi_keys.count); + for(U64 idx = 0; idx < dst.dbgi_keys.count; idx += 1) + { + dst.dbgi_keys.v[idx].path = push_str8_copy(arena, dst.dbgi_keys.v[idx].path); + } + return dst; } //////////////////////////////// @@ -22,7 +126,29 @@ fzy_item_num_from_array_element_idx__linear_search(FZY_ItemArray *array, U64 ele internal void fzy_init(void) { - + Arena *arena = arena_alloc(); + fzy_shared = push_array(arena, FZY_Shared, 1); + fzy_shared->arena = arena; + fzy_shared->slots_count = 256; + fzy_shared->stripes_count = os_logical_core_count(); + fzy_shared->slots = push_array(arena, FZY_Slot, fzy_shared->slots_count); + fzy_shared->stripes = push_array(arena, FZY_Stripe, fzy_shared->stripes_count); + for(U64 idx = 0; idx < fzy_shared->stripes_count; idx += 1) + { + fzy_shared->stripes[idx].arena = arena_alloc(); + fzy_shared->stripes[idx].rw_mutex = os_rw_mutex_alloc(); + fzy_shared->stripes[idx].cv = os_condition_variable_alloc(); + } + fzy_shared->thread_count = Min(os_logical_core_count(), 2); + fzy_shared->threads = push_array(arena, FZY_Thread, fzy_shared->thread_count); + for(U64 idx = 0; idx < fzy_shared->thread_count; idx += 1) + { + fzy_shared->threads[idx].u2f_ring_mutex = os_mutex_alloc(); + fzy_shared->threads[idx].u2f_ring_cv = os_condition_variable_alloc(); + fzy_shared->threads[idx].u2f_ring_size = KB(64); + fzy_shared->threads[idx].u2f_ring_base = push_array_no_zero(arena, U8, fzy_shared->threads[idx].u2f_ring_size); + fzy_shared->threads[idx].thread = os_launch_thread(fzy_search_thread__entry_point, (void *)idx, 0); + } } //////////////////////////////// @@ -31,19 +157,59 @@ fzy_init(void) internal FZY_Scope * fzy_scope_open(void) { - + if(fzy_tctx == 0) + { + Arena *arena = arena_alloc(); + fzy_tctx = push_array(arena, FZY_TCTX, 1); + fzy_tctx->arena = arena; + } + FZY_Scope *scope = fzy_tctx->free_scope; + if(scope != 0) + { + SLLStackPop(fzy_tctx->free_scope); + } + else + { + scope = push_array_no_zero(fzy_tctx->arena, FZY_Scope, 1); + } + MemoryZeroStruct(scope); + return scope; } internal void fzy_scope_close(FZY_Scope *scope) { - + for(FZY_Touch *t = scope->first_touch, *next = 0; t != 0; t = next) + { + next = t->next; + SLLStackPush(fzy_tctx->free_touch, t); + if(t->node != 0) + { + ins_atomic_u64_dec_eval(&t->node->touch_count); + } + } + SLLStackPush(fzy_tctx->free_scope, scope); } internal void fzy_scope_touch_node__stripe_mutex_r_guarded(FZY_Scope *scope, FZY_Node *node) { - + if(node != 0) + { + ins_atomic_u64_inc_eval(&node->touch_count); + } + FZY_Touch *touch = fzy_tctx->free_touch; + if(touch != 0) + { + SLLStackPop(fzy_tctx->free_touch); + } + else + { + touch = push_array_no_zero(fzy_tctx->arena, FZY_Touch, 1); + } + MemoryZeroStruct(touch); + SLLQueuePush(scope->first_touch, scope->last_touch, touch); + touch->node = node; } //////////////////////////////// @@ -52,7 +218,88 @@ fzy_scope_touch_node__stripe_mutex_r_guarded(FZY_Scope *scope, FZY_Node *node) internal FZY_ItemArray fzy_items_from_key_params_query(FZY_Scope *scope, U128 key, FZY_Params *params, String8 query, U64 endt_us, B32 *stale_out) { + Temp scratch = scratch_begin(0, 0); + FZY_ItemArray items = {0}; + //- rjf: hash parameters + U64 params_hash = fzy_hash_from_params(params); + + //- rjf: unpack key + U64 slot_idx = key.u64[1]%fzy_shared->slots_count; + U64 stripe_idx = slot_idx%fzy_shared->stripes_count; + FZY_Slot *slot = &fzy_shared->slots[slot_idx]; + FZY_Stripe *stripe = &fzy_shared->stripes[stripe_idx]; + + //- rjf: query and/or request + OS_MutexScopeR(stripe->rw_mutex) for(;;) + { + // rjf: map key -> node + FZY_Node *node = 0; + for(FZY_Node *n = slot->first; n != 0; n = n->next) + { + if(u128_match(n->key, key)) + { + node = n; + break; + } + } + + // rjf: no node? -> allocate + if(node == 0) OS_MutexScopeRWPromote(stripe->rw_mutex) + { + node = push_array(stripe->arena, FZY_Node, 1); + SLLQueuePush(slot->first, slot->last, node); + node->key = key; + for(U64 idx = 0; idx < ArrayCount(node->buckets); idx += 1) + { + node->buckets[idx].arena = arena_alloc(); + } + } + + // rjf: try to grab last valid results for this key/query; determine if stale + B32 stale = 1; + if(params_hash == node->buckets[node->gen%ArrayCount(node->buckets)].params_hash && + node->gen != 0) + { + fzy_scope_touch_node__stripe_mutex_r_guarded(scope, node); + items = node->gen_items; + stale = !str8_match(query, node->buckets[node->gen%ArrayCount(node->buckets)].query, 0); + if(stale_out != 0) + { + *stale_out = stale; + } + } + + // rjf: if stale -> request again + if(stale) OS_MutexScopeRWPromote(stripe->rw_mutex) + { + if(node->gen <= node->submit_gen && node->submit_gen < node->gen + ArrayCount(node->buckets)-1) + { + node->submit_gen += 1; + arena_clear(node->buckets[node->submit_gen%ArrayCount(node->buckets)].arena); + node->buckets[node->submit_gen%ArrayCount(node->buckets)].query = push_str8_copy(node->buckets[node->submit_gen%ArrayCount(node->buckets)].arena, query); + node->buckets[node->submit_gen%ArrayCount(node->buckets)].params = fzy_params_copy(node->buckets[node->submit_gen%ArrayCount(node->buckets)].arena, params); + node->buckets[node->submit_gen%ArrayCount(node->buckets)].params_hash = params_hash; + } + if((node->submit_gen > node->gen+1 || os_now_microseconds() >= node->last_time_submitted_us+100000) && + fzy_u2s_enqueue_req(key, endt_us)) + { + node->last_time_submitted_us = os_now_microseconds(); + } + } + + // rjf: not stale, or timeout -> break + if(!stale || os_now_microseconds() >= endt_us) + { + break; + } + + // rjf: no results, but have time to wait -> wait + os_condition_variable_wait_rw_r(stripe->cv, stripe->rw_mutex, endt_us); + } + + scratch_end(scratch); + return items; } //////////////////////////////// @@ -61,23 +308,277 @@ fzy_items_from_key_params_query(FZY_Scope *scope, U128 key, FZY_Params *params, internal B32 fzy_u2s_enqueue_req(U128 key, U64 endt_us) { - + B32 sent = 0; + FZY_Thread *thread = &fzy_shared->threads[key.u64[1]%fzy_shared->thread_count]; + OS_MutexScope(thread->u2f_ring_mutex) for(;;) + { + U64 unconsumed_size = thread->u2f_ring_write_pos - thread->u2f_ring_read_pos; + U64 available_size = thread->u2f_ring_size - unconsumed_size; + if(available_size >= sizeof(U128)) + { + sent = 1; + thread->u2f_ring_write_pos += ring_write_struct(thread->u2f_ring_base, thread->u2f_ring_size, thread->u2f_ring_write_pos, &key); + break; + } + os_condition_variable_wait(thread->u2f_ring_cv, thread->u2f_ring_mutex, endt_us); + } + if(sent) + { + os_condition_variable_broadcast(thread->u2f_ring_cv); + } + return sent; } internal void fzy_u2s_dequeue_req(Arena *arena, FZY_Thread *thread, U128 *key_out) { - + OS_MutexScope(thread->u2f_ring_mutex) for(;;) + { + U64 unconsumed_size = thread->u2f_ring_write_pos - thread->u2f_ring_read_pos; + if(unconsumed_size >= sizeof(U128)) + { + thread->u2f_ring_read_pos += ring_read_struct(thread->u2f_ring_base, thread->u2f_ring_size, thread->u2f_ring_read_pos, key_out); + break; + } + os_condition_variable_wait(thread->u2f_ring_cv, thread->u2f_ring_mutex, max_U64); + } + os_condition_variable_broadcast(thread->u2f_ring_cv); } internal int fzy_qsort_compare_items(FZY_Item *a, FZY_Item *b) { - + int result = 0; + if(a->match_ranges.count > b->match_ranges.count) + { + result = -1; + } + else if(a->match_ranges.count < b->match_ranges.count) + { + result = +1; + } + else if(a->missed_size < b->missed_size) + { + result = -1; + } + else if(a->missed_size > b->missed_size) + { + result = +1; + } + return result; } internal void fzy_search_thread__entry_point(void *p) { - + ThreadNameF("[fzy] searcher #%I64u", (U64)p); + FZY_Thread *thread = &fzy_shared->threads[(U64)p]; + for(;;) + { + Temp scratch = scratch_begin(0, 0); + DI_Scope *di_scope = di_scope_open(); + + //////////////////////////// + //- rjf: dequeue next request + // + U128 key = {0}; + fzy_u2s_dequeue_req(scratch.arena, thread, &key); + U64 slot_idx = key.u64[1]%fzy_shared->slots_count; + U64 stripe_idx = slot_idx%fzy_shared->stripes_count; + FZY_Slot *slot = &fzy_shared->slots[slot_idx]; + FZY_Stripe *stripe = &fzy_shared->stripes[stripe_idx]; + + //////////////////////////// + //- rjf: grab next exe_path/query for this key + // + B32 task_is_good = 0; + Arena *task_arena = 0; + String8 query = {0}; + FZY_Params params = {FZY_Target_Procedures}; + U64 initial_submit_gen = 0; + OS_MutexScopeW(stripe->rw_mutex) + { + for(FZY_Node *n = slot->first; n != 0; n = n->next) + { + if(u128_match(n->key, key)) + { + FZY_Bucket *bucket = &n->buckets[n->submit_gen%ArrayCount(n->buckets)]; + task_is_good = 1; + initial_submit_gen = n->submit_gen; + task_arena = bucket->arena; + query = bucket->query; + params = bucket->params; + break; + } + } + } + + //////////////////////////// + //- rjf: params -> look up all rdis + // + U64 rdis_count = params.dbgi_keys.count; + RDI_Parsed **rdis = push_array(scratch.arena, RDI_Parsed *, rdis_count); + if(task_is_good) + { + for(U64 idx = 0; idx < rdis_count; idx += 1) + { + rdis[idx] = di_rdi_from_path_min_timestamp(di_scope, params.dbgi_keys.v[idx].path, params.dbgi_keys.v[idx].timestamp, max_U64); + } + } + + //////////////////////////// + //- rjf: search target -> info about search space + // + U64 table_ptr_off = 0; + U64 table_count_off = 0; + U64 element_name_idx_off = 0; + U64 element_size = 0; + if(task_is_good) + { + switch(params.target) + { + // NOTE(rjf): no default! + case FZY_Target_COUNT:{}break; + case FZY_Target_Procedures: + { + table_ptr_off = OffsetOf(RDI_Parsed, procedures); + table_count_off = OffsetOf(RDI_Parsed, procedures_count); + element_name_idx_off = OffsetOf(RDI_Procedure, name_string_idx); + element_size = sizeof(RDI_Procedure); + }break; + case FZY_Target_GlobalVariables: + { + table_ptr_off = OffsetOf(RDI_Parsed, global_variables); + table_count_off = OffsetOf(RDI_Parsed, global_variables_count); + element_name_idx_off = OffsetOf(RDI_GlobalVariable, name_string_idx); + element_size = sizeof(RDI_GlobalVariable); + }break; + case FZY_Target_ThreadVariables: + { + table_ptr_off = OffsetOf(RDI_Parsed, thread_variables); + table_count_off = OffsetOf(RDI_Parsed, thread_variables_count); + element_name_idx_off = OffsetOf(RDI_ThreadVariable, name_string_idx); + element_size = sizeof(RDI_ThreadVariable); + }break; + case FZY_Target_UDTs: + { + table_ptr_off = OffsetOf(RDI_Parsed, udts); + table_count_off = OffsetOf(RDI_Parsed, udts_count); + element_size = sizeof(RDI_UDT); + }break; + } + } + + //////////////////////////// + //- rjf: rdis * query * params -> item list + // + FZY_ItemChunkList items_list = {0}; + if(task_is_good) + { + U64 base_idx = 0; + for(U64 rdi_idx = 0; rdi_idx < rdis_count; rdi_idx += 1) + { + RDI_Parsed *rdi = rdis[rdi_idx]; + void *table_base = (U8*)rdi + table_ptr_off; + U64 element_count = *MemberFromOffset(U64 *, rdi, table_count_off); + for(U64 idx = 1; task_is_good && idx < element_count; idx += 1) + { + void *element = (U8 *)(*(void **)table_base) + element_size*idx; + U32 *name_idx_ptr = (U32 *)((U8 *)element + element_name_idx_off); + if(params.target == FZY_Target_UDTs) + { + RDI_UDT *udt = (RDI_UDT *)element; + RDI_TypeNode *type_node = rdi_element_from_idx(rdi, type_nodes, udt->self_type_idx); + name_idx_ptr = &type_node->user_defined.name_string_idx; + } + U32 name_idx = *name_idx_ptr; + U64 name_size = 0; + U8 *name_base = rdi_string_from_idx(rdi, name_idx, &name_size); + String8 name = str8(name_base, name_size); + if(name.size == 0) { continue; } + FuzzyMatchRangeList matches = fuzzy_match_find(task_arena, query, name); + if(matches.count == matches.needle_part_count) + { + FZY_ItemChunk *chunk = items_list.last; + if(chunk == 0 || chunk->count >= chunk->cap) + { + chunk = push_array(scratch.arena, FZY_ItemChunk, 1); + chunk->cap = 1024; + chunk->count = 0; + chunk->v = push_array_no_zero(scratch.arena, FZY_Item, chunk->cap); + SLLQueuePush(items_list.first, items_list.last, chunk); + items_list.chunk_count += 1; + } + chunk->v[chunk->count].idx = base_idx + idx; + chunk->v[chunk->count].match_ranges = matches; + chunk->v[chunk->count].missed_size = (name_size > matches.total_dim) ? (name_size-matches.total_dim) : 0; + chunk->count += 1; + items_list.total_count += 1; + } + if(idx%100 == 99) OS_MutexScopeR(stripe->rw_mutex) + { + for(FZY_Node *n = slot->first; n != 0; n = n->next) + { + if(u128_match(n->key, key) && n->submit_gen > initial_submit_gen) + { + task_is_good = 0; + break; + } + } + } + } + base_idx += element_count; + } + } + + //- rjf: item list -> item array + FZY_ItemArray items = {0}; + if(task_is_good) + { + items.count = items_list.total_count; + items.v = push_array_no_zero(task_arena, FZY_Item, items.count); + U64 idx = 0; + for(FZY_ItemChunk *chunk = items_list.first; chunk != 0; chunk = chunk->next) + { + MemoryCopy(items.v+idx, chunk->v, sizeof(FZY_Item)*chunk->count); + idx += chunk->count; + } + } + + //- rjf: sort item array + if(items.count != 0 && query.size != 0) + { + qsort(items.v, items.count, sizeof(FZY_Item), (int (*)(const void *, const void *))fzy_qsort_compare_items); + } + + //- rjf: commit to cache - busyloop on scope touches + if(task_is_good) + { + for(B32 done = 0; !done;) + { + B32 found = 0; + OS_MutexScopeW(stripe->rw_mutex) for(FZY_Node *n = slot->first; n != 0; n = n->next) + { + if(u128_match(n->key, key)) + { + if(n->touch_count == 0) + { + n->gen = initial_submit_gen; + n->gen_items = items; + done = 1; + } + found = 1; + break; + } + } + if(!found) + { + break; + } + } + } + + di_scope_close(di_scope); + scratch_end(scratch); + } } diff --git a/src/fuzzy_search/fuzzy_search.h b/src/fuzzy_search/fuzzy_search.h index 99fc5aa8..a2b83a1e 100644 --- a/src/fuzzy_search/fuzzy_search.h +++ b/src/fuzzy_search/fuzzy_search.h @@ -5,28 +5,12 @@ #define FUZZY_SEARCH_H //////////////////////////////// -//~ rjf: Fuzzy Search Types - -typedef enum FZY_Target -{ - FZY_Target_Procedures, - FZY_Target_GlobalVariables, - FZY_Target_ThreadVariables, - FZY_Target_UDTs, - FZY_Target_COUNT -} -FZY_Target; - -typedef struct FZY_Params FZY_Params; -struct FZY_Params -{ - FZY_Target target; -}; +//~ rjf: Result Types typedef struct FZY_Item FZY_Item; struct FZY_Item { - U64 idx; + U64 idx; // indexes into whole space of parameter tables. [rdis[0] element count) [rdis[1] element count) ... [rdis[n] element count) U64 missed_size; FuzzyMatchRangeList match_ranges; }; @@ -56,12 +40,65 @@ struct FZY_ItemArray U64 count; }; +//////////////////////////////// +//~ rjf: Search Parameter Types + +typedef enum FZY_Target +{ + FZY_Target_Procedures, + FZY_Target_GlobalVariables, + FZY_Target_ThreadVariables, + FZY_Target_UDTs, + FZY_Target_COUNT +} +FZY_Target; + +typedef struct FZY_DbgiKey FZY_DbgiKey; +struct FZY_DbgiKey +{ + String8 path; + U64 timestamp; +}; + +typedef struct FZY_DbgiKeyNode FZY_DbgiKeyNode; +struct FZY_DbgiKeyNode +{ + FZY_DbgiKeyNode *next; + FZY_DbgiKey v; +}; + +typedef struct FZY_DbgiKeyList FZY_DbgiKeyList; +struct FZY_DbgiKeyList +{ + FZY_DbgiKeyNode *first; + FZY_DbgiKeyNode *last; + U64 count; +}; + +typedef struct FZY_DbgiKeyArray FZY_DbgiKeyArray; +struct FZY_DbgiKeyArray +{ + FZY_DbgiKey *v; + U64 count; +}; + +typedef struct FZY_Params FZY_Params; +struct FZY_Params +{ + FZY_Target target; + FZY_DbgiKeyArray dbgi_keys; +}; + +//////////////////////////////// +//~ rjf: Cache Types + typedef struct FZY_Bucket FZY_Bucket; struct FZY_Bucket { Arena *arena; String8 query; - FZY_Target target; + FZY_Params params; + U64 params_hash; }; typedef struct FZY_Node FZY_Node; @@ -69,7 +106,7 @@ struct FZY_Node { FZY_Node *next; U128 key; - U64 scope_touch_count; + U64 touch_count; U64 last_time_submitted_us; FZY_Bucket buckets[3]; U64 gen; @@ -92,18 +129,6 @@ struct FZY_Stripe OS_Handle cv; }; -typedef struct FZY_Thread FZY_Thread; -struct FZY_Thread -{ - OS_Handle thread; - OS_Handle u2f_ring_mutex; - OS_Handle u2f_ring_cv; - U64 u2f_ring_size; - U8 *u2f_ring_base; - U64 u2f_ring_write_pos; - U64 u2f_ring_read_pos; -}; - //////////////////////////////// //~ rjf: Scoped Access Types @@ -133,6 +158,18 @@ struct FZY_TCTX //////////////////////////////// //~ rjf: Shared State Types +typedef struct FZY_Thread FZY_Thread; +struct FZY_Thread +{ + OS_Handle thread; + OS_Handle u2f_ring_mutex; + OS_Handle u2f_ring_cv; + U64 u2f_ring_size; + U8 *u2f_ring_base; + U64 u2f_ring_write_pos; + U64 u2f_ring_read_pos; +}; + typedef struct FZY_Shared FZY_Shared; struct FZY_Shared { @@ -149,11 +186,22 @@ struct FZY_Shared FZY_Thread *threads; }; +//////////////////////////////// +//~ rjf: Globals + +global FZY_Shared *fzy_shared = 0; +thread_static FZY_TCTX *fzy_tctx = 0; + //////////////////////////////// //~ rjf: Helpers -internal U64 fzy_hash_from_string(String8 string, StringMatchFlags match_flags); +internal U64 fzy_hash_from_string(U64 seed, String8 string); +internal U64 fzy_hash_from_params(FZY_Params *params); internal U64 fzy_item_num_from_array_element_idx__linear_search(FZY_ItemArray *array, U64 element_idx); +internal String8 fzy_item_string_from_rdi_target_element_idx(RDI_Parsed *rdi, FZY_Target target, U64 element_idx); +internal void fzy_dbgi_key_list_push(Arena *arena, FZY_DbgiKeyList *list, FZY_DbgiKey key); +internal FZY_DbgiKeyArray fzy_dbgi_key_array_from_list(Arena *arena, FZY_DbgiKeyList *list); +internal FZY_Params fzy_params_copy(Arena *arena, FZY_Params *src); //////////////////////////////// //~ rjf: Main Layer Initialization