From 0e352f92df21d47fb32271a7b9ae86e67612eb86 Mon Sep 17 00:00:00 2001 From: Ryan Fleury Date: Thu, 22 Feb 2024 14:45:09 -0800 Subject: [PATCH] separate per-name-map baking stages from each other, and from top-level name map section build; do them all independently --- src/lib_raddbgi_make/raddbgi_make.c | 204 ++++++++++++------------ src/lib_raddbgi_make/raddbgi_make.h | 3 +- src/raddbgi_from_pdb/raddbgi_from_pdb.c | 42 ++++- src/raddbgi_from_pdb/raddbgi_from_pdb.h | 9 +- src/task_system/task_system.c | 9 ++ src/task_system/task_system.h | 16 ++ 6 files changed, 171 insertions(+), 112 deletions(-) diff --git a/src/lib_raddbgi_make/raddbgi_make.c b/src/lib_raddbgi_make/raddbgi_make.c index af479b17..2b1b7914 100644 --- a/src/lib_raddbgi_make/raddbgi_make.c +++ b/src/lib_raddbgi_make/raddbgi_make.c @@ -2959,7 +2959,7 @@ rdim_bake_scope_vmap_section_list_from_params(RDIM_Arena *arena, RDIM_BakeParams //- rjf: name maps RDI_PROC RDIM_BakeSectionList -rdim_bake_name_map_section_list_from_params_maps(RDIM_Arena *arena, RDIM_BakeStringMap *strings, RDIM_BakeIdxRunMap *idx_runs, RDIM_BakeParams *params, RDIM_BakeNameMap *name_maps[RDI_NameMapKind_COUNT]) +rdim_bake_top_level_name_map_section_list_from_params_maps(RDIM_Arena *arena, RDIM_BakeStringMap *strings, RDIM_BakeIdxRunMap *idx_runs, RDIM_BakeParams *params, RDIM_BakeNameMap *name_maps[RDI_NameMapKind_COUNT]) { RDIM_BakeSectionList sections = {0}; @@ -2986,107 +2986,9 @@ rdim_bake_name_map_section_list_from_params_maps(RDIM_Arena *arena, RDIM_BakeStr RDI_NameMap *dst_map = &dst_maps[dst_map_idx]; RDIM_BakeNameMap *src_map = name_maps[k]; if(src_map == 0 || src_map->name_count == 0) { continue; } - - // rjf: bake name map - RDI_U32 baked_buckets_count = src_map->name_count; - RDI_U32 baked_nodes_count = src_map->name_count; - RDI_NameMapBucket *baked_buckets = rdim_push_array(arena, RDI_NameMapBucket, baked_buckets_count); - RDI_NameMapNode *baked_nodes = rdim_push_array_no_zero(arena, RDI_NameMapNode, baked_nodes_count); - { - RDIM_Temp scratch = rdim_scratch_begin(&arena, 1); - - // rjf: setup the final bucket layouts - typedef struct RDIM_NameMapSemiNode RDIM_NameMapSemiNode; - struct RDIM_NameMapSemiNode - { - RDIM_NameMapSemiNode *next; - RDIM_BakeNameMapNode *node; - }; - typedef struct RDIM_NameMapSemiBucket RDIM_NameMapSemiBucket; - struct RDIM_NameMapSemiBucket - { - RDIM_NameMapSemiNode *first; - RDIM_NameMapSemiNode *last; - RDI_U64 count; - }; - RDIM_NameMapSemiBucket *sbuckets = rdim_push_array(scratch.arena, RDIM_NameMapSemiBucket, baked_buckets_count); - for(RDIM_BakeNameMapNode *node = src_map->first; - node != 0; - node = node->order_next) - { - RDI_U64 hash = rdi_hash(node->string.str, node->string.size); - RDI_U64 bi = hash%baked_buckets_count; - RDIM_NameMapSemiNode *snode = rdim_push_array(scratch.arena, RDIM_NameMapSemiNode, 1); - SLLQueuePush(sbuckets[bi].first, sbuckets[bi].last, snode); - snode->node = node; - sbuckets[bi].count += 1; - } - - // rjf: convert to serialized buckets & nodes - { - RDI_NameMapBucket *bucket_ptr = baked_buckets; - RDI_NameMapNode *node_ptr = baked_nodes; - for(RDI_U32 i = 0; i < baked_buckets_count; i += 1, bucket_ptr += 1) - { - bucket_ptr->first_node = (RDI_U32)(node_ptr - baked_nodes); - bucket_ptr->node_count = sbuckets[i].count; - for(RDIM_NameMapSemiNode *snode = sbuckets[i].first; - snode != 0; - snode = snode->next) - { - RDIM_BakeNameMapNode *node = snode->node; - - // rjf: cons name and index(es) - RDI_U32 string_idx = rdim_bake_idx_from_string(strings, node->string); - RDI_U32 match_count = node->val_count; - RDI_U32 idx = 0; - if(match_count == 1) - { - idx = node->val_first->val[0]; - } - else - { - RDI_U64 temp_pos = rdim_arena_pos(scratch.arena); - RDI_U32 *idx_run = rdim_push_array_no_zero(scratch.arena, RDI_U32, match_count); - RDI_U32 *idx_ptr = idx_run; - for(RDIM_BakeNameMapValNode *idxnode = node->val_first; - idxnode != 0; - idxnode = idxnode->next) - { - for(RDI_U32 i = 0; i < sizeof(idxnode->val)/sizeof(idxnode->val[0]); i += 1) - { - if(idxnode->val[i] == 0) - { - goto dblbreak; - } - *idx_ptr = idxnode->val[i]; - idx_ptr += 1; - } - } - dblbreak:; - idx = rdim_bake_idx_from_idx_run(idx_runs, idx_run, match_count); - rdim_arena_pop_to(scratch.arena, temp_pos); - } - - // rjf: write to node - node_ptr->string_idx = string_idx; - node_ptr->match_count = match_count; - node_ptr->match_idx_or_idx_run_first = idx; - node_ptr += 1; - } - } - } - rdim_scratch_end(scratch); - } - - // rjf: fill output header, and push sections for buckets/nodes dst_map->kind = k; dst_map->bucket_data_idx = (RDI_U32)rdim_bake_section_idx_from_params_tag_idx(params, RDI_DataSectionTag_NameMapBuckets, (RDI_U64)k); // TODO(rjf): @u64_to_u32 dst_map->node_data_idx = (RDI_U32)rdim_bake_section_idx_from_params_tag_idx(params, RDI_DataSectionTag_NameMapNodes, (RDI_U64)k); // TODO(rjf): @u64_to_u32 - rdim_bake_section_list_push_new(arena, §ions, baked_buckets, sizeof(RDI_NameMapBucket)* baked_buckets_count, RDI_DataSectionTag_NameMapBuckets, (RDI_U64)k); - rdim_bake_section_list_push_new(arena, §ions, baked_nodes, sizeof(RDI_NameMapNode) * baked_nodes_count, RDI_DataSectionTag_NameMapNodes, (RDI_U64)k); - - // rjf: inc dst_map_idx += 1; } } @@ -3096,6 +2998,110 @@ rdim_bake_name_map_section_list_from_params_maps(RDIM_Arena *arena, RDIM_BakeStr return sections; } +RDI_PROC RDIM_BakeSectionList +rdim_bake_name_map_section_list_from_params_kind_map(RDIM_Arena *arena, RDIM_BakeStringMap *strings, RDIM_BakeIdxRunMap *idx_runs, RDIM_BakeParams *params, RDI_NameMapKind k, RDIM_BakeNameMap *map) +{ + RDIM_BakeSectionList sections = {0}; + if(map != 0 && map->name_count != 0) + { + RDI_U32 baked_buckets_count = map->name_count; + RDI_U32 baked_nodes_count = map->name_count; + RDI_NameMapBucket *baked_buckets = rdim_push_array(arena, RDI_NameMapBucket, baked_buckets_count); + RDI_NameMapNode *baked_nodes = rdim_push_array_no_zero(arena, RDI_NameMapNode, baked_nodes_count); + { + RDIM_Temp scratch = rdim_scratch_begin(&arena, 1); + + // rjf: setup the final bucket layouts + typedef struct RDIM_NameMapSemiNode RDIM_NameMapSemiNode; + struct RDIM_NameMapSemiNode + { + RDIM_NameMapSemiNode *next; + RDIM_BakeNameMapNode *node; + }; + typedef struct RDIM_NameMapSemiBucket RDIM_NameMapSemiBucket; + struct RDIM_NameMapSemiBucket + { + RDIM_NameMapSemiNode *first; + RDIM_NameMapSemiNode *last; + RDI_U64 count; + }; + RDIM_NameMapSemiBucket *sbuckets = rdim_push_array(scratch.arena, RDIM_NameMapSemiBucket, baked_buckets_count); + for(RDIM_BakeNameMapNode *node = map->first; + node != 0; + node = node->order_next) + { + RDI_U64 hash = rdi_hash(node->string.str, node->string.size); + RDI_U64 bi = hash%baked_buckets_count; + RDIM_NameMapSemiNode *snode = rdim_push_array(scratch.arena, RDIM_NameMapSemiNode, 1); + SLLQueuePush(sbuckets[bi].first, sbuckets[bi].last, snode); + snode->node = node; + sbuckets[bi].count += 1; + } + + // rjf: convert to serialized buckets & nodes + { + RDI_NameMapBucket *bucket_ptr = baked_buckets; + RDI_NameMapNode *node_ptr = baked_nodes; + for(RDI_U32 i = 0; i < baked_buckets_count; i += 1, bucket_ptr += 1) + { + bucket_ptr->first_node = (RDI_U32)(node_ptr - baked_nodes); + bucket_ptr->node_count = sbuckets[i].count; + for(RDIM_NameMapSemiNode *snode = sbuckets[i].first; + snode != 0; + snode = snode->next) + { + RDIM_BakeNameMapNode *node = snode->node; + + // rjf: cons name and index(es) + RDI_U32 string_idx = rdim_bake_idx_from_string(strings, node->string); + RDI_U32 match_count = node->val_count; + RDI_U32 idx = 0; + if(match_count == 1) + { + idx = node->val_first->val[0]; + } + else + { + RDI_U64 temp_pos = rdim_arena_pos(scratch.arena); + RDI_U32 *idx_run = rdim_push_array_no_zero(scratch.arena, RDI_U32, match_count); + RDI_U32 *idx_ptr = idx_run; + for(RDIM_BakeNameMapValNode *idxnode = node->val_first; + idxnode != 0; + idxnode = idxnode->next) + { + for(RDI_U32 i = 0; i < sizeof(idxnode->val)/sizeof(idxnode->val[0]); i += 1) + { + if(idxnode->val[i] == 0) + { + goto dblbreak; + } + *idx_ptr = idxnode->val[i]; + idx_ptr += 1; + } + } + dblbreak:; + idx = rdim_bake_idx_from_idx_run(idx_runs, idx_run, match_count); + rdim_arena_pop_to(scratch.arena, temp_pos); + } + + // rjf: write to node + node_ptr->string_idx = string_idx; + node_ptr->match_count = match_count; + node_ptr->match_idx_or_idx_run_first = idx; + node_ptr += 1; + } + } + } + rdim_scratch_end(scratch); + } + + // rjf: sections for buckets/nodes + rdim_bake_section_list_push_new(arena, §ions, baked_buckets, sizeof(RDI_NameMapBucket)* baked_buckets_count, RDI_DataSectionTag_NameMapBuckets, (RDI_U64)k); + rdim_bake_section_list_push_new(arena, §ions, baked_nodes, sizeof(RDI_NameMapNode) * baked_nodes_count, RDI_DataSectionTag_NameMapNodes, (RDI_U64)k); + } + return sections; +} + //- rjf: file paths RDI_PROC RDIM_BakeSectionList diff --git a/src/lib_raddbgi_make/raddbgi_make.h b/src/lib_raddbgi_make/raddbgi_make.h index 474ed2bb..f412e9f3 100644 --- a/src/lib_raddbgi_make/raddbgi_make.h +++ b/src/lib_raddbgi_make/raddbgi_make.h @@ -1178,7 +1178,8 @@ RDI_PROC RDIM_BakeSectionList rdim_bake_scope_section_list_from_params(RDIM_Aren RDI_PROC RDIM_BakeSectionList rdim_bake_scope_vmap_section_list_from_params(RDIM_Arena *arena, RDIM_BakeParams *params); //- rjf: name maps -RDI_PROC RDIM_BakeSectionList rdim_bake_name_map_section_list_from_params_maps(RDIM_Arena *arena, RDIM_BakeStringMap *strings, RDIM_BakeIdxRunMap *idx_runs, RDIM_BakeParams *params, RDIM_BakeNameMap *name_maps[RDI_NameMapKind_COUNT]); +RDI_PROC RDIM_BakeSectionList rdim_bake_top_level_name_map_section_list_from_params_maps(RDIM_Arena *arena, RDIM_BakeStringMap *strings, RDIM_BakeIdxRunMap *idx_runs, RDIM_BakeParams *params, RDIM_BakeNameMap *name_maps[RDI_NameMapKind_COUNT]); +RDI_PROC RDIM_BakeSectionList rdim_bake_name_map_section_list_from_params_kind_map(RDIM_Arena *arena, RDIM_BakeStringMap *strings, RDIM_BakeIdxRunMap *idx_runs, RDIM_BakeParams *params, RDI_NameMapKind k, RDIM_BakeNameMap *map); //- rjf: file paths RDI_PROC RDIM_BakeSectionList rdim_bake_file_path_section_list_from_path_tree(RDIM_Arena *arena, RDIM_BakeStringMap *strings, RDIM_BakePathTree *path_tree); diff --git a/src/raddbgi_from_pdb/raddbgi_from_pdb.c b/src/raddbgi_from_pdb/raddbgi_from_pdb.c index bc80f0a0..56e4a8c5 100644 --- a/src/raddbgi_from_pdb/raddbgi_from_pdb.c +++ b/src/raddbgi_from_pdb/raddbgi_from_pdb.c @@ -3518,11 +3518,11 @@ p2r_bake_type_nodes_task__entry_point(Arena *arena, void *p) } internal void * -p2r_bake_name_maps_task__entry_point(Arena *arena, void *p) +p2r_bake_name_map_task__entry_point(Arena *arena, void *p) { - P2R_BakeNameMapsIn *in = (P2R_BakeNameMapsIn *)p; + P2R_BakeNameMapIn *in = (P2R_BakeNameMapIn *)p; RDIM_BakeSectionList *s = push_array(arena, RDIM_BakeSectionList, 1); - ProfScope("bake name maps") *s = rdim_bake_name_map_section_list_from_params_maps(arena, in->strings, in->idx_runs, in->params, in->name_maps); + ProfScope("bake name map %i", in->kind) *s = rdim_bake_name_map_section_list_from_params_kind_map(arena, in->strings, in->idx_runs, in->params, in->kind, in->map); return s; } @@ -3541,6 +3541,7 @@ p2r_bake_idx_runs_task__entry_point(Arena *arena, void *p) internal P2R_Bake2Serialize * p2r_bake(Arena *arena, P2R_Convert2Bake *in) { + Temp scratch = scratch_begin(&arena, 1); RDIM_BakeParams *params = &in->bake_params; RDIM_BakeSectionList sections = {0}; @@ -3634,12 +3635,33 @@ p2r_bake(Arena *arena, P2R_Convert2Bake *in) //- rjf: kick off pass 3 tasks P2R_BakeTypeNodesIn bake_type_nodes_in = {strings, idx_runs, params}; TS_Ticket bake_type_nodes_ticket = ts_kickoff(p2r_bake_type_nodes_task__entry_point, 0, &bake_type_nodes_in); - P2R_BakeNameMapsIn bake_name_maps_in = {strings, idx_runs, params}; - MemoryCopyArray(bake_name_maps_in.name_maps, name_maps); - TS_Ticket bake_name_maps_ticket = ts_kickoff(p2r_bake_name_maps_task__entry_point, 0, &bake_name_maps_in); + TS_TicketList bake_name_maps_tickets = {0}; + for(RDI_NameMapKind k = (RDI_NameMapKind)(RDI_NameMapKind_NULL+1); + k < RDI_NameMapKind_COUNT; + k = (RDI_NameMapKind)(k+1)) + { + if(name_maps[k] == 0 || name_maps[k]->name_count == 0) + { + continue; + } + P2R_BakeNameMapIn *in = push_array(scratch.arena, P2R_BakeNameMapIn, 1); + in->strings = strings; + in->idx_runs = idx_runs; + in->params = params; + in->kind = k; + in->map = name_maps[k]; + ts_ticket_list_push(scratch.arena, &bake_name_maps_tickets, ts_kickoff(p2r_bake_name_map_task__entry_point, 0, in)); + } P2R_BakeIdxRunsIn bake_idx_runs_in = {idx_runs}; TS_Ticket bake_idx_runs_ticket = ts_kickoff(p2r_bake_idx_runs_task__entry_point, 0, &bake_idx_runs_in); + //- rjf: bake top-level name maps section + ProfScope("top level name maps section") + { + RDIM_BakeSectionList s = rdim_bake_top_level_name_map_section_list_from_params_maps(arena, strings, idx_runs, params, name_maps); + rdim_bake_section_list_concat_in_place(§ions, &s); + } + //- rjf: join units ProfScope("units") { @@ -3734,8 +3756,11 @@ p2r_bake(Arena *arena, P2R_Convert2Bake *in) //- rjf: join name maps ProfScope("name map") { - RDIM_BakeSectionList *s = ts_join_struct(bake_name_maps_ticket, max_U64, RDIM_BakeSectionList); - rdim_bake_section_list_concat_in_place(§ions, s); + for(TS_TicketNode *n = bake_name_maps_tickets.first; n != 0; n = n->next) + { + RDIM_BakeSectionList *s = ts_join_struct(n->v, max_U64, RDIM_BakeSectionList); + rdim_bake_section_list_concat_in_place(§ions, s); + } } //- rjf: join index runs @@ -3748,6 +3773,7 @@ p2r_bake(Arena *arena, P2R_Convert2Bake *in) //- rjf: fill & return P2R_Bake2Serialize *out = push_array(arena, P2R_Bake2Serialize, 1); out->sections = sections; + scratch_end(scratch); return out; } diff --git a/src/raddbgi_from_pdb/raddbgi_from_pdb.h b/src/raddbgi_from_pdb/raddbgi_from_pdb.h index ce6bc4bf..9041f5ce 100644 --- a/src/raddbgi_from_pdb/raddbgi_from_pdb.h +++ b/src/raddbgi_from_pdb/raddbgi_from_pdb.h @@ -326,13 +326,14 @@ struct P2R_BakeTypeNodesIn RDIM_BakeParams *params; }; -typedef struct P2R_BakeNameMapsIn P2R_BakeNameMapsIn; -struct P2R_BakeNameMapsIn +typedef struct P2R_BakeNameMapIn P2R_BakeNameMapIn; +struct P2R_BakeNameMapIn { RDIM_BakeStringMap *strings; RDIM_BakeIdxRunMap *idx_runs; RDIM_BakeParams *params; - RDIM_BakeNameMap *name_maps[RDI_NameMapKind_COUNT]; + RDI_NameMapKind kind; + RDIM_BakeNameMap *map; }; typedef struct P2R_BakeIdxRunsIn P2R_BakeIdxRunsIn; @@ -428,7 +429,7 @@ internal void *p2r_bake_strings_task__entry_point(Arena *arena, void *p); //- rjf: pass 3: idx-run-map-dependent debug info stream builds internal void *p2r_bake_type_nodes_task__entry_point(Arena *arena, void *p); -internal void *p2r_bake_name_maps_task__entry_point(Arena *arena, void *p); +internal void *p2r_bake_name_map_task__entry_point(Arena *arena, void *p); internal void *p2r_bake_idx_runs_task__entry_point(Arena *arena, void *p); //////////////////////////////// diff --git a/src/task_system/task_system.c b/src/task_system/task_system.c index aa3902f4..fb79b9c0 100644 --- a/src/task_system/task_system.c +++ b/src/task_system/task_system.c @@ -8,6 +8,15 @@ ts_ticket_zero(void) return ticket; } +internal void +ts_ticket_list_push(Arena *arena, TS_TicketList *list, TS_Ticket ticket) +{ + TS_TicketNode *n = push_array(arena, TS_TicketNode, 1); + n->v = ticket; + SLLQueuePush(list->first, list->last, n); + list->count += 1; +} + //////////////////////////////// //~ rjf: Top-Level Layer Initialization diff --git a/src/task_system/task_system.h b/src/task_system/task_system.h index 690de476..ccf06755 100644 --- a/src/task_system/task_system.h +++ b/src/task_system/task_system.h @@ -16,6 +16,21 @@ struct TS_Ticket U64 u64[2]; }; +typedef struct TS_TicketNode TS_TicketNode; +struct TS_TicketNode +{ + TS_TicketNode *next; + TS_Ticket v; +}; + +typedef struct TS_TicketList TS_TicketList; +struct TS_TicketList +{ + TS_TicketNode *first; + TS_TicketNode *last; + U64 count; +}; + //////////////////////////////// //~ rjf: Task Request Types @@ -96,6 +111,7 @@ global TS_Shared *ts_shared = 0; //~ rjf: Basic Type Functions internal TS_Ticket ts_ticket_zero(void); +internal void ts_ticket_list_push(Arena *arena, TS_TicketList *list, TS_Ticket ticket); //////////////////////////////// //~ rjf: Top-Level Layer Initialization