From 38b113f1d0139998435dd7a7810e9b9009565135 Mon Sep 17 00:00:00 2001 From: Ryan Fleury Date: Thu, 25 Jan 2024 10:04:03 -0800 Subject: [PATCH] simplify & consolidate ctrl layer process memory caching mechanisms - move all non-necessarily-synchronous caching to background memory streaming thread cache --- src/ctrl/ctrl_core.c | 303 ++++++++++++++++-------------------------- src/ctrl/ctrl_core.h | 53 ++------ src/df/core/df_core.c | 2 +- src/df/gfx/df_views.c | 18 +-- 4 files changed, 123 insertions(+), 253 deletions(-) diff --git a/src/ctrl/ctrl_core.c b/src/ctrl/ctrl_core.c index a0363a3c..a974beb4 100644 --- a/src/ctrl/ctrl_core.c +++ b/src/ctrl/ctrl_core.c @@ -754,189 +754,12 @@ ctrl_process_read(CTRL_MachineID machine_id, CTRL_Handle process, Rng1U64 range, return actual_bytes_read; } -internal CTRL_ProcessMemorySlice -ctrl_query_cached_data_from_process_vaddr_range(Arena *arena, CTRL_MachineID machine_id, CTRL_Handle process, Rng1U64 range) -{ - CTRL_ProcessMemorySlice result = {0}; - if(range.max > range.min && - dim_1u64(range) <= MB(256) && - range.min <= 0x000FFFFFFFFFFFFFull && - range.max <= 0x000FFFFFFFFFFFFFull) - { - HS_Scope *scope = hs_scope_open(); - CTRL_ProcessMemoryCache *cache = &ctrl_state->process_memory_cache; - - //- rjf: unpack address range - Rng1U64 page_range = r1u64(AlignDownPow2(range.min, KB(4)), AlignPow2(range.max, KB(4))); - - //- rjf: setup output memory for read - void *read_out = push_array(arena, U8, dim_1u64(page_range)); - - //- rjf: unpack process/machine params - U64 hash = ctrl_hash_from_string(str8_struct(&process)); - U64 slot_idx = hash%cache->slots_count; - U64 stripe_idx = slot_idx%cache->stripes_count; - CTRL_ProcessMemoryCacheSlot *slot = &cache->slots[slot_idx]; - CTRL_ProcessMemoryCacheStripe *stripe = &cache->stripes[stripe_idx]; - - //- rjf: cache lookup & fill loop - for(U64 page_vaddr = page_range.min; page_vaddr < page_range.max; page_vaddr += KB(4)) - { - // rjf: unpack page base address - U64 lvl5_idx = (page_vaddr&0x00000000000FF000ull) >> 12; - U64 lvl4_idx = (page_vaddr&0x000000000FF00000ull) >> 20; - U64 lvl3_idx = (page_vaddr&0x0000000FF0000000ull) >> 28; - U64 lvl2_idx = (page_vaddr&0x00000FF000000000ull) >> 36; - U64 lvl1_idx = (page_vaddr&0x000FF00000000000ull) >> 44; - - // rjf: try to find node & read from it - B32 node_found = 0; - B32 page_found = 0; - B32 page_stale = 0; - OS_MutexScopeR(stripe->rw_mutex) - { - CTRL_ProcessMemoryCacheNode *node = 0; - for(CTRL_ProcessMemoryCacheNode *n = slot->first; n != 0; n = n->next) - { - if(ctrl_handle_match(process, n->process) && n->machine_id == machine_id) - { - node = n; - break; - } - } - if(node != 0) - { - node_found = 1; - CTRL_ProcessMemoryCacheNode1 *node1 = node->children[lvl1_idx]; - CTRL_ProcessMemoryCacheNode2 *node2 = node1 ? node1->children[lvl2_idx] : 0; - CTRL_ProcessMemoryCacheNode3 *node3 = node2 ? node2->children[lvl3_idx] : 0; - CTRL_ProcessMemoryCacheNode4 *node4 = node3 ? node3->children[lvl4_idx] : 0; - U128 page_hash = node4 ? node4->page_hashes[lvl5_idx] : u128_zero(); - B32 stale = (node4 && node4->page_memgen_idxs[lvl5_idx] < ctrl_memgen_idx()); - if(!u128_match(page_hash, u128_zero())) - { - page_stale = stale; - String8 page_data = hs_data_from_hash(scope, page_hash); - if(page_data.size >= KB(4)) - { - page_found = 1; - MemoryCopy((U8*)read_out + (page_vaddr-page_range.min), page_data.str, KB(4)); - } - else - { - page_stale = 1; - } - } - } - } - - // rjf: either node or page not found? -> need a hard-lock & fill - if(!node_found || !page_found || page_stale) OS_MutexScopeW(stripe->rw_mutex) - { - CTRL_ProcessMemoryCacheNode *node = 0; - for(CTRL_ProcessMemoryCacheNode *n = slot->first; n != 0; n = n->next) - { - if(ctrl_handle_match(process, n->process) && n->machine_id == machine_id) - { - node = n; - break; - } - } - if(node == 0) - { - Arena *node_arena = arena_alloc(); - node = push_array(node_arena, CTRL_ProcessMemoryCacheNode, 1); - node->arena = node_arena; - node->machine_id = machine_id; - node->process = process; - node->range_hash_slots_count = 1024; - node->range_hash_slots = push_array(node_arena, CTRL_ProcessMemoryRangeHashSlot, node->range_hash_slots_count); - DLLPushBack(slot->first, slot->last, node); - } - if(!page_found || page_stale) - { - CTRL_ProcessMemoryCacheNode1 *node1 = node->children[lvl1_idx]; - if(node1 == 0) - { - node1 = push_array(node->arena, CTRL_ProcessMemoryCacheNode1, 1); - node->children[lvl1_idx] = node1; - } - CTRL_ProcessMemoryCacheNode2 *node2 = node1->children[lvl2_idx]; - if(node2 == 0) - { - node2 = push_array(node->arena, CTRL_ProcessMemoryCacheNode2, 1); - node1->children[lvl2_idx] = node2; - } - CTRL_ProcessMemoryCacheNode3 *node3 = node2->children[lvl3_idx]; - if(node3 == 0) - { - node3 = push_array(node->arena, CTRL_ProcessMemoryCacheNode3, 1); - node2->children[lvl3_idx] = node3; - } - CTRL_ProcessMemoryCacheNode4 *node4 = node3->children[lvl4_idx]; - if(node4 == 0) - { - node4 = push_array(node->arena, CTRL_ProcessMemoryCacheNode4, 1); - node3->children[lvl4_idx] = node4; - } - Arena *page_arena = arena_alloc__sized(KB(8), KB(8)); - void *page_base = push_array_no_zero(page_arena, U8, KB(4)); - U64 actual_read_size = ctrl_process_read(machine_id, process, r1u64(page_vaddr, page_vaddr+KB(4)), page_base); - if(actual_read_size >= KB(4)) - { - node4->page_memgen_idxs[lvl5_idx] = ctrl_memgen_idx(); - if(page_stale) - { - MemoryCopy((U8*)read_out + (page_vaddr-page_range.min), page_base, KB(4)); - } - U64 page_key_data[] = - { - (U64)machine_id, - (U64)process.u64[0], - page_vaddr, - page_vaddr+KB(4), - }; - U128 page_key = hs_hash_from_data(str8((U8 *)page_key_data, sizeof(page_key_data))); - U128 page_hash = hs_submit_data(page_key, &page_arena, str8((U8 *)page_base, KB(4))); - node4->page_hashes[lvl5_idx] = page_hash; - } - else - { - arena_release(page_arena); - } - } - } - } - - //- rjf: fill result by skipping/chopping read memory - U64 byte_in_page_idx = (range.min&0x0000000000000FFFull) >> 0; - result.data.str = (U8*)read_out + byte_in_page_idx; - result.data.size = dim_1u64(range); - hs_scope_close(scope); - } - return result; -} - -internal CTRL_ProcessMemorySlice -ctrl_query_cached_zero_terminated_data_from_process_vaddr_limit(Arena *arena, CTRL_MachineID machine_id, CTRL_Handle process, U64 vaddr, U64 limit, U64 endt_us) -{ - CTRL_ProcessMemorySlice result = ctrl_query_cached_data_from_process_vaddr_range(arena, machine_id, process, r1u64(vaddr, vaddr+limit)); - for(U64 idx = 0; idx < result.data.size; idx += 1) - { - if(result.data.str[idx] == 0) - { - result.data.size = idx; - break; - } - } - return result; -} - internal B32 -ctrl_process_write_data(CTRL_MachineID machine_id, CTRL_Handle process, U64 vaddr, String8 data) +ctrl_process_write(CTRL_MachineID machine_id, CTRL_Handle process, Rng1U64 range, void *src) { ProfBeginFunction(); - B32 result = demon_write_memory(ctrl_demon_handle_from_ctrl(process), vaddr, data.str, data.size); + U64 size = dim_1u64(range); + B32 result = demon_write_memory(ctrl_demon_handle_from_ctrl(process), range.min, src, size); if(result) { ins_atomic_u64_inc_eval(&ctrl_state->memgen_idx); @@ -945,6 +768,23 @@ ctrl_process_write_data(CTRL_MachineID machine_id, CTRL_Handle process, U64 vadd return result; } +//- rjf: process memory cache interaction + +internal U128 +ctrl_hash_store_key_from_process_vaddr_range(CTRL_MachineID machine_id, CTRL_Handle process, Rng1U64 range, B32 zero_terminated) +{ + U64 key_hash_data[] = + { + (U64)machine_id, + (U64)process.u64[0], + range.min, + range.max, + (U64)zero_terminated, + }; + U128 key = hs_hash_from_data(str8((U8*)key_hash_data, sizeof(key_hash_data))); + return key; +} + internal U128 ctrl_stored_hash_from_process_vaddr_range(CTRL_MachineID machine_id, CTRL_Handle process, Rng1U64 range, B32 zero_terminated) { @@ -1056,6 +896,98 @@ ctrl_stored_hash_from_process_vaddr_range(CTRL_MachineID machine_id, CTRL_Handle return result; } +//- rjf: process memory cache reading helpers + +internal CTRL_ProcessMemorySlice +ctrl_query_cached_data_from_process_vaddr_range(Arena *arena, CTRL_MachineID machine_id, CTRL_Handle process, Rng1U64 range) +{ + CTRL_ProcessMemorySlice result = {0}; + if(range.max > range.min && + dim_1u64(range) <= MB(256) && + range.min <= 0x000FFFFFFFFFFFFFull && + range.max <= 0x000FFFFFFFFFFFFFull) + { + Temp scratch = scratch_begin(&arena, 1); + HS_Scope *scope = hs_scope_open(); + CTRL_ProcessMemoryCache *cache = &ctrl_state->process_memory_cache; + + //- rjf: unpack address range, prepare per-touched-page info + U64 page_size = KB(4); + Rng1U64 page_range = r1u64(AlignDownPow2(range.min, page_size), AlignPow2(range.max, page_size)); + U64 page_count = dim_1u64(page_range)/page_size; + U128 *page_hashes = push_array(scratch.arena, U128, page_count); + U128 *page_last_hashes = push_array(scratch.arena, U128, page_count); + + //- rjf: gather hashes for each page + for(U64 page_idx = 0; page_idx < page_count; page_idx += 1) + { + U64 page_base_vaddr = page_range.min + page_idx*page_size; + U128 page_hash = ctrl_stored_hash_from_process_vaddr_range(machine_id, process, r1u64(page_base_vaddr, page_base_vaddr+page_size), 0); + page_hashes[page_idx] = page_hash; + } + + //- rjf: gather last hashes for each page + for(U64 page_idx = 0; page_idx < page_count; page_idx += 1) + { + U64 page_base_vaddr = page_range.min + page_idx*page_size; + U128 page_key = ctrl_hash_store_key_from_process_vaddr_range(machine_id, process, r1u64(page_base_vaddr, page_base_vaddr+page_size), 0); + U128 page_last_hash = hs_hash_from_key(page_key, 1); + page_last_hashes[page_idx] = page_last_hash; + } + + //- rjf: setup output buffer + void *read_out = push_array(arena, U8, dim_1u64(range)); + + //- rjf: iterate pages, fill output + { + U64 write_off = 0; + for(U64 page_idx = 0; page_idx < page_count; page_idx += 1) + { + String8 data = hs_data_from_hash(scope, page_hashes[page_idx]); + Rng1U64 data_vaddr_range = r1u64(page_range.min + page_idx*page_size, page_range.min + page_idx*page_size+data.size); + String8 in_range_data = data; + if(page_idx == page_count-1 && data_vaddr_range.max > range.max) + { + in_range_data = str8_chop(in_range_data, data_vaddr_range.max-range.max); + } + if(page_idx == 0 && range.min > data_vaddr_range.min) + { + in_range_data = str8_skip(in_range_data, range.min-data_vaddr_range.min); + } + MemoryCopy((U8*)read_out+write_off, in_range_data.str, in_range_data.size); + write_off += in_range_data.size; + if(data.size < page_size) + { + write_off += page_size-data.size; + } + } + } + + //- rjf: fill result + result.data.str = (U8*)read_out; + result.data.size = dim_1u64(range); + + hs_scope_close(scope); + scratch_end(scratch); + } + return result; +} + +internal CTRL_ProcessMemorySlice +ctrl_query_cached_zero_terminated_data_from_process_vaddr_limit(Arena *arena, CTRL_MachineID machine_id, CTRL_Handle process, U64 vaddr, U64 limit, U64 endt_us) +{ + CTRL_ProcessMemorySlice result = ctrl_query_cached_data_from_process_vaddr_range(arena, machine_id, process, r1u64(vaddr, vaddr+limit)); + for(U64 idx = 0; idx < result.data.size; idx += 1) + { + if(result.data.str[idx] == 0) + { + result.data.size = idx; + break; + } + } + return result; +} + //- rjf: register reading/writing internal void * @@ -3175,6 +3107,7 @@ ctrl_mem_stream_thread__entry_point(void *p) Rng1U64 vaddr_range = {0}; B32 zero_terminated = 0; ctrl_u2ms_dequeue_req(&machine_id, &process, &vaddr_range, &zero_terminated); + U128 key = ctrl_hash_store_key_from_process_vaddr_range(machine_id, process, vaddr_range, zero_terminated); //- rjf: unpack process memory cache key U64 process_hash = ctrl_hash_from_string(str8_struct(&process)); @@ -3247,16 +3180,6 @@ ctrl_mem_stream_thread__entry_point(void *p) } } - //- rjf: determine key for this region - U64 key_hash_data[] = - { - (U64)machine_id, - (U64)process.u64[0], - vaddr_range.min, - vaddr_range.min + zero_terminated_size, - }; - U128 key = hs_hash_from_data(str8((U8 *)key_hash_data, sizeof(key_hash_data))); - //- rjf: read successful -> submit to hash store U128 hash = {0}; if(got_task && range_base != 0) diff --git a/src/ctrl/ctrl_core.h b/src/ctrl/ctrl_core.h index 2187459c..2ec06297 100644 --- a/src/ctrl/ctrl_core.h +++ b/src/ctrl/ctrl_core.h @@ -311,48 +311,6 @@ struct CTRL_EventList //////////////////////////////// //~ rjf: Process Memory Cache Types -// NOTE(rjf): -// -// Process memory is cached with a 5-level page table. Each level has 256 -// slots, and is indexed into with 8 bits. Each index is extracted from a -// virtual address in the following manner: -// -// |1------||2------||3------||4------||5------||byte-n-page| -// xxxxxxxx xxxx0000 00000000 00000000 00000000 00000000 0000xxxx xxxxxxxx -// -// The top 12 bits are not used (a 52-bit address space is supported at most). -// The next 8 most-significant-bits are used to index into the level 1 table. -// The next 8 are used to index into the level 2. Then the level 3. Then the -// level 4. At the level 4 table, instead of pointing to other tables, each -// slot points at the base address of a cached 4KB page. The next 8 bits are -// used to index into that table. The final 12 bits in the address are used -// to refer to unique bytes within each page. - -typedef struct CTRL_ProcessMemoryCacheNode4 CTRL_ProcessMemoryCacheNode4; -struct CTRL_ProcessMemoryCacheNode4 -{ - U64 page_memgen_idxs[256]; - U128 page_hashes[256]; -}; - -typedef struct CTRL_ProcessMemoryCacheNode3 CTRL_ProcessMemoryCacheNode3; -struct CTRL_ProcessMemoryCacheNode3 -{ - CTRL_ProcessMemoryCacheNode4 *children[256]; -}; - -typedef struct CTRL_ProcessMemoryCacheNode2 CTRL_ProcessMemoryCacheNode2; -struct CTRL_ProcessMemoryCacheNode2 -{ - CTRL_ProcessMemoryCacheNode3 *children[256]; -}; - -typedef struct CTRL_ProcessMemoryCacheNode1 CTRL_ProcessMemoryCacheNode1; -struct CTRL_ProcessMemoryCacheNode1 -{ - CTRL_ProcessMemoryCacheNode2 *children[256]; -}; - typedef struct CTRL_ProcessMemoryRangeHashNode CTRL_ProcessMemoryRangeHashNode; struct CTRL_ProcessMemoryRangeHashNode { @@ -379,7 +337,6 @@ struct CTRL_ProcessMemoryCacheNode Arena *arena; CTRL_MachineID machine_id; CTRL_Handle process; - CTRL_ProcessMemoryCacheNode1 *children[256]; U64 range_hash_slots_count; CTRL_ProcessMemoryRangeHashSlot *range_hash_slots; }; @@ -410,6 +367,7 @@ typedef struct CTRL_ProcessMemorySlice CTRL_ProcessMemorySlice; struct CTRL_ProcessMemorySlice { String8 data; + U64 *byte_good_flags; B32 fresh; }; @@ -567,10 +525,15 @@ internal Architecture ctrl_arch_from_handle(CTRL_MachineID machine, CTRL_Handle //- rjf: process memory reading/writing internal U64 ctrl_process_read(CTRL_MachineID machine_id, CTRL_Handle process, Rng1U64 range, void *dst); +internal B32 ctrl_process_write(CTRL_MachineID machine_id, CTRL_Handle process, Rng1U64 range, void *src); + +//- rjf: process memory cache interaction +internal U128 ctrl_hash_store_key_from_process_vaddr_range(CTRL_MachineID machine_id, CTRL_Handle process, Rng1U64 range, B32 zero_terminated); +internal U128 ctrl_stored_hash_from_process_vaddr_range(CTRL_MachineID machine_id, CTRL_Handle process, Rng1U64 range, B32 zero_terminated); + +//- rjf: process memory cache reading helpers internal CTRL_ProcessMemorySlice ctrl_query_cached_data_from_process_vaddr_range(Arena *arena, CTRL_MachineID machine_id, CTRL_Handle process, Rng1U64 range); internal CTRL_ProcessMemorySlice ctrl_query_cached_zero_terminated_data_from_process_vaddr_limit(Arena *arena, CTRL_MachineID machine_id, CTRL_Handle process, U64 vaddr, U64 limit, U64 endt_us); -internal B32 ctrl_process_write_data(CTRL_MachineID machine_id, CTRL_Handle process, U64 vaddr, String8 data); -internal U128 ctrl_stored_hash_from_process_vaddr_range(CTRL_MachineID machine_id, CTRL_Handle process, Rng1U64 range, B32 zero_terminated); //- rjf: register reading/writing internal void *ctrl_reg_block_from_thread(CTRL_MachineID machine_id, CTRL_Handle thread); diff --git a/src/df/core/df_core.c b/src/df/core/df_core.c index 62f0ca98..3d406cb5 100644 --- a/src/df/core/df_core.c +++ b/src/df/core/df_core.c @@ -4878,7 +4878,7 @@ df_commit_eval_value(TG_Graph *graph, RADDBG_Parsed *rdbg, DF_CtrlCtx *ctrl_ctx, default:{}break; case EVAL_EvalMode_Addr: { - ctrl_process_write_data(process->ctrl_machine_id, process->ctrl_handle, dst_eval.offset, commit_data); + ctrl_process_write(process->ctrl_machine_id, process->ctrl_handle, r1u64(dst_eval.offset, dst_eval.offset+commit_data.size), commit_data.str); }break; case EVAL_EvalMode_Reg: { diff --git a/src/df/gfx/df_views.c b/src/df/gfx/df_views.c index 2e0b8ca1..dc66c9c6 100644 --- a/src/df/gfx/df_views.c +++ b/src/df/gfx/df_views.c @@ -1032,23 +1032,7 @@ df_eval_watch_view_build(DF_Window *ws, DF_Panel *panel, DF_View *view, DF_EvalW //- rjf: build normal row if(!(row->flags & DF_EvalVizRowFlag_Canvas)) { - B32 row_is_fresh = 0; - if(row->eval.mode == EVAL_EvalMode_Addr) - { - Temp temp = temp_begin(scratch.arena); - U64 vaddr = row->eval.offset; - U64 size = tg_byte_size_from_graph_raddbg_key(parse_ctx.type_graph, parse_ctx.rdbg, row->eval.type_key); - CTRL_ProcessMemorySlice slice = ctrl_query_cached_data_from_process_vaddr_range(scratch.arena, process->ctrl_machine_id, process->ctrl_handle, r1u64(vaddr, vaddr+size)); - row_is_fresh = slice.fresh; - temp_end(temp); - } - ui_set_next_flags(disabled_flags|(UI_BoxFlag_DrawBackground*row_is_fresh)); - if(row_is_fresh) - { - Vec4F32 fresh_bg_color = df_rgba_from_theme_color(DF_ThemeColor_Highlight0); - fresh_bg_color.w *= 0.25f; - ui_set_next_background_color(fresh_bg_color); - } + ui_set_next_flags(disabled_flags); UI_NamedTableVectorF("row_%I64x", row_hash) { //- rjf: expression