From 8b4f97223e2a447a54229bcfe19a8ee24615d3a8 Mon Sep 17 00:00:00 2001 From: Nikita Smith Date: Mon, 3 Mar 2025 17:09:01 -0800 Subject: [PATCH] moved async bakers from PDB converter to a stand-alone file to reuse them in DWARF converter --- src/lib_rdi_make/rdi_make.c | 22 + src/lib_rdi_make/rdi_make.h | 15 + src/raddbg/raddbg_main.c | 2 + src/rdi_from_pdb/rdi_from_pdb.c | 877 +------------------------- src/rdi_from_pdb/rdi_from_pdb.h | 308 --------- src/rdi_from_pdb/rdi_from_pdb_main.c | 2 + src/rdi_make/rdi_make_help.c | 901 +++++++++++++++++++++++++++ src/rdi_make/rdi_make_help.h | 330 ++++++++++ 8 files changed, 1280 insertions(+), 1177 deletions(-) create mode 100644 src/rdi_make/rdi_make_help.c create mode 100644 src/rdi_make/rdi_make_help.h diff --git a/src/lib_rdi_make/rdi_make.c b/src/lib_rdi_make/rdi_make.c index 3f37b9f5..272e2dd9 100644 --- a/src/lib_rdi_make/rdi_make.c +++ b/src/lib_rdi_make/rdi_make.c @@ -652,6 +652,28 @@ rdim_unit_chunk_list_concat_in_place(RDIM_UnitChunkList *dst, RDIM_UnitChunkList //////////////////////////////// //~ rjf: [Building] Type Info Building +RDI_PROC RDIM_Type ** +rdim_array_from_type_list(RDIM_Arena *arena, RDIM_TypeList list) +{ + RDIM_Type **arr = push_array(arena, RDIM_Type *, list.count); + U64 i = 0; + for(RDIM_TypeNode *n = list.first; n != 0; n = n->next, ++i) + { + arr[i] = n->v; + } + return arr; +} + +RDI_PROC RDIM_TypeNode * +rdim_type_list_push(RDIM_Arena *arena, RDIM_TypeList *list, RDIM_Type *v) +{ + RDIM_TypeNode *n = push_array(arena, RDIM_TypeNode, 1); + n->v = v; + SLLQueuePush(list->first, list->last, n); + list->count += 1; + return n; +} + RDI_PROC RDIM_Type * rdim_type_chunk_list_push(RDIM_Arena *arena, RDIM_TypeChunkList *list, RDI_U64 cap) { diff --git a/src/lib_rdi_make/rdi_make.h b/src/lib_rdi_make/rdi_make.h index 47ff5ade..176dbbac 100644 --- a/src/lib_rdi_make/rdi_make.h +++ b/src/lib_rdi_make/rdi_make.h @@ -610,6 +610,21 @@ struct RDIM_Type struct RDIM_UDT *udt; }; +typedef struct RDIM_TypeNode RDIM_TypeNode; +struct RDIM_TypeNode +{ + struct RDIM_TypeNode *next; + RDIM_Type *v; +}; + +typedef struct RDIM_TypeList RDIM_TypeList; +struct RDIM_TypeList +{ + U64 count; + RDIM_TypeNode *first; + RDIM_TypeNode *last; +}; + typedef struct RDIM_TypeChunkNode RDIM_TypeChunkNode; struct RDIM_TypeChunkNode { diff --git a/src/raddbg/raddbg_main.c b/src/raddbg/raddbg_main.c index e47ebe6e..54ca6ba9 100644 --- a/src/raddbg/raddbg_main.c +++ b/src/raddbg/raddbg_main.c @@ -333,6 +333,7 @@ #include "async/async.h" #include "rdi_format/rdi_format_local.h" #include "rdi_make/rdi_make_local.h" +#include "rdi_make/rdi_make_help.h" #include "mdesk/mdesk.h" #include "hash_store/hash_store.h" #include "file_stream/file_stream.h" @@ -375,6 +376,7 @@ #include "async/async.c" #include "rdi_format/rdi_format_local.c" #include "rdi_make/rdi_make_local.c" +#include "rdi_make/rdi_make_help.c" #include "mdesk/mdesk.c" #include "hash_store/hash_store.c" #include "file_stream/file_stream.c" diff --git a/src/rdi_from_pdb/rdi_from_pdb.c b/src/rdi_from_pdb/rdi_from_pdb.c index a56b5e6f..a2f067b6 100644 --- a/src/rdi_from_pdb/rdi_from_pdb.c +++ b/src/rdi_from_pdb/rdi_from_pdb.c @@ -3977,387 +3977,6 @@ p2r_convert(Arena *arena, P2R_User2Convert *in) return out; } -//////////////////////////////// -//~ rjf: Baking Stage Tasks - -//- rjf: bake string map building - -#define p2r_make_string_map_if_needed() do {if(in->maps[thread_idx] == 0) ProfScope("make map") {in->maps[thread_idx] = rdim_bake_string_map_loose_make(arena, in->top);}} while(0) - -ASYNC_WORK_DEF(p2r_bake_src_files_strings_work) -{ - ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; - P2R_BakeSrcFilesStringsIn *in = (P2R_BakeSrcFilesStringsIn *)input; - p2r_make_string_map_if_needed(); - ProfScope("bake src file strings") rdim_bake_string_map_loose_push_src_files(arena, in->top, in->maps[thread_idx], in->list); - ProfEnd(); - return 0; -} - -ASYNC_WORK_DEF(p2r_bake_units_strings_work) -{ - ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; - P2R_BakeUnitsStringsIn *in = (P2R_BakeUnitsStringsIn *)input; - p2r_make_string_map_if_needed(); - ProfScope("bake unit strings") rdim_bake_string_map_loose_push_units(arena, in->top, in->maps[thread_idx], in->list); - ProfEnd(); - return 0; -} - -ASYNC_WORK_DEF(p2r_bake_types_strings_work) -{ - ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; - P2R_BakeTypesStringsIn *in = (P2R_BakeTypesStringsIn *)input; - p2r_make_string_map_if_needed(); - ProfScope("bake type strings") - { - for(P2R_BakeTypesStringsInNode *n = in->first; n != 0; n = n->next) - { - rdim_bake_string_map_loose_push_type_slice(arena, in->top, in->maps[thread_idx], n->v, n->count); - } - } - ProfEnd(); - return 0; -} - -ASYNC_WORK_DEF(p2r_bake_udts_strings_work) -{ - ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; - P2R_BakeUDTsStringsIn *in = (P2R_BakeUDTsStringsIn *)input; - p2r_make_string_map_if_needed(); - ProfScope("bake udt strings") - { - for(P2R_BakeUDTsStringsInNode *n = in->first; n != 0; n = n->next) - { - rdim_bake_string_map_loose_push_udt_slice(arena, in->top, in->maps[thread_idx], n->v, n->count); - } - } - ProfEnd(); - return 0; -} - -ASYNC_WORK_DEF(p2r_bake_symbols_strings_work) -{ - ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; - P2R_BakeSymbolsStringsIn *in = (P2R_BakeSymbolsStringsIn *)input; - p2r_make_string_map_if_needed(); - ProfScope("bake symbol strings") - { - for(P2R_BakeSymbolsStringsInNode *n = in->first; n != 0; n = n->next) - { - rdim_bake_string_map_loose_push_symbol_slice(arena, in->top, in->maps[thread_idx], n->v, n->count); - } - } - ProfEnd(); - return 0; -} - -ASYNC_WORK_DEF(p2r_bake_inline_site_strings_work) -{ - ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; - P2R_BakeInlineSiteStringsIn *in = input; - p2r_make_string_map_if_needed(); - ProfScope("bake inline site strings") - { - for(P2R_BakeInlineSiteStringsInNode *n = in->first; n != 0; n = n->next) - { - rdim_bake_string_map_loose_push_inline_site_slice(arena, in->top, in->maps[thread_idx], n->v, n->count); - } - } - ProfEnd(); - return 0; -} - -ASYNC_WORK_DEF(p2r_bake_scopes_strings_work) -{ - ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; - P2R_BakeScopesStringsIn *in = (P2R_BakeScopesStringsIn *)input; - p2r_make_string_map_if_needed(); - ProfScope("bake scope strings") - { - for(P2R_BakeScopesStringsInNode *n = in->first; n != 0; n = n->next) - { - rdim_bake_string_map_loose_push_scope_slice(arena, in->top, in->maps[thread_idx], n->v, n->count); - } - } - ProfEnd(); - return 0; -} - -ASYNC_WORK_DEF(p2r_bake_line_tables_work) -{ - ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; - P2R_BakeLineTablesIn *in = (P2R_BakeLineTablesIn *)input; - RDIM_LineTableBakeResult *out = push_array(arena, RDIM_LineTableBakeResult, 1); - ProfScope("bake line tables") *out = rdim_bake_line_tables(arena, in->line_tables); - ProfEnd(); - return out; -} - -#undef p2r_make_string_map_if_needed - -//- rjf: bake string map joining - -ASYNC_WORK_DEF(p2r_bake_string_map_join_work) -{ - ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; - P2R_JoinBakeStringMapSlotsIn *in = (P2R_JoinBakeStringMapSlotsIn *)input; - ProfScope("join bake string maps") - { - for(U64 src_map_idx = 0; src_map_idx < in->src_maps_count; src_map_idx += 1) - { - for(U64 slot_idx = in->slot_idx_range.min; slot_idx < in->slot_idx_range.max; slot_idx += 1) - { - B32 src_slots_good = (in->src_maps[src_map_idx] != 0 && in->src_maps[src_map_idx]->slots != 0); - B32 dst_slot_is_zero = (in->dst_map->slots[slot_idx] == 0); - if(src_slots_good && dst_slot_is_zero) - { - in->dst_map->slots[slot_idx] = in->src_maps[src_map_idx]->slots[slot_idx]; - } - else if(src_slots_good && in->src_maps[src_map_idx]->slots[slot_idx] != 0) - { - rdim_bake_string_chunk_list_concat_in_place(in->dst_map->slots[slot_idx], in->src_maps[src_map_idx]->slots[slot_idx]); - } - } - } - } - ProfEnd(); - return 0; -} - -//- rjf: bake string map sorting - -ASYNC_WORK_DEF(p2r_bake_string_map_sort_work) -{ - ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; - P2R_SortBakeStringMapSlotsIn *in = (P2R_SortBakeStringMapSlotsIn *)input; - ProfScope("sort bake string chunk list map range") - { - for(U64 slot_idx = in->slot_idx; - slot_idx < in->slot_idx+in->slot_count; - slot_idx += 1) - { - if(in->src_map->slots[slot_idx] != 0) - { - if(in->src_map->slots[slot_idx]->total_count > 1) - { - in->dst_map->slots[slot_idx] = push_array(arena, RDIM_BakeStringChunkList, 1); - *in->dst_map->slots[slot_idx] = rdim_bake_string_chunk_list_sorted_from_unsorted(arena, in->src_map->slots[slot_idx]); - } - else - { - in->dst_map->slots[slot_idx] = in->src_map->slots[slot_idx]; - } - } - } - } - ProfEnd(); - return 0; -} - -//- rjf: pass 1: interner/deduper map builds - -ASYNC_WORK_DEF(p2r_build_bake_name_map_work) -{ - ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; - P2R_BuildBakeNameMapIn *in = (P2R_BuildBakeNameMapIn *)input; - RDIM_BakeNameMap *name_map = 0; - ProfScope("build name map %i", in->k) name_map = rdim_bake_name_map_from_kind_params(arena, in->k, in->params); - ProfEnd(); - return name_map; -} - -//- rjf: pass 2: string-map-dependent debug info stream builds - -ASYNC_WORK_DEF(p2r_bake_units_work) -{ - ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; - P2R_BakeUnitsIn *in = (P2R_BakeUnitsIn *)input; - RDIM_UnitBakeResult *out = push_array(arena, RDIM_UnitBakeResult, 1); - ProfScope("bake units") *out = rdim_bake_units(arena, in->strings, in->path_tree, in->units); - ProfEnd(); - return out; -} - -ASYNC_WORK_DEF(p2r_bake_unit_vmap_work) -{ - ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; - P2R_BakeUnitVMapIn *in = (P2R_BakeUnitVMapIn *)input; - RDIM_UnitVMapBakeResult *out = push_array(arena, RDIM_UnitVMapBakeResult, 1); - ProfScope("bake unit vmap") *out = rdim_bake_unit_vmap(arena, in->units); - ProfEnd(); - return out; -} - -ASYNC_WORK_DEF(p2r_bake_src_files_work) -{ - ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; - P2R_BakeSrcFilesIn *in = (P2R_BakeSrcFilesIn *)input; - RDIM_SrcFileBakeResult *out = push_array(arena, RDIM_SrcFileBakeResult, 1); - ProfScope("bake src files") *out = rdim_bake_src_files(arena, in->strings, in->path_tree, in->src_files); - ProfEnd(); - return out; -} - -ASYNC_WORK_DEF(p2r_bake_udts_work) -{ - ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; - P2R_BakeUDTsIn *in = (P2R_BakeUDTsIn *)input; - RDIM_UDTBakeResult *out = push_array(arena, RDIM_UDTBakeResult, 1); - ProfScope("bake udts") *out = rdim_bake_udts(arena, in->strings, in->udts); - ProfEnd(); - return out; -} - -ASYNC_WORK_DEF(p2r_bake_global_variables_work) -{ - ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; - P2R_BakeGlobalVariablesIn *in = (P2R_BakeGlobalVariablesIn *)input; - RDIM_GlobalVariableBakeResult *out = push_array(arena, RDIM_GlobalVariableBakeResult, 1); - ProfScope("bake global variables") *out = rdim_bake_global_variables(arena, in->strings, in->global_variables); - ProfEnd(); - return out; -} - -ASYNC_WORK_DEF(p2r_bake_global_vmap_work) -{ - ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; - P2R_BakeGlobalVMapIn *in = (P2R_BakeGlobalVMapIn *)input; - RDIM_GlobalVMapBakeResult *out = push_array(arena, RDIM_GlobalVMapBakeResult, 1); - ProfScope("bake global vmap") *out = rdim_bake_global_vmap(arena, in->global_variables); - ProfEnd(); - return out; -} - -ASYNC_WORK_DEF(p2r_bake_thread_variables_work) -{ - ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; - P2R_BakeThreadVariablesIn *in = (P2R_BakeThreadVariablesIn *)input; - RDIM_ThreadVariableBakeResult *out = push_array(arena, RDIM_ThreadVariableBakeResult, 1); - ProfScope("bake thread variables") *out = rdim_bake_thread_variables(arena, in->strings, in->thread_variables); - ProfEnd(); - return out; -} - -ASYNC_WORK_DEF(p2r_bake_procedures_work) -{ - ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; - P2R_BakeProceduresIn *in = (P2R_BakeProceduresIn *)input; - RDIM_ProcedureBakeResult *out = push_array(arena, RDIM_ProcedureBakeResult, 1); - ProfScope("bake procedures") *out = rdim_bake_procedures(arena, in->strings, in->procedures); - ProfEnd(); - return out; -} - -ASYNC_WORK_DEF(p2r_bake_scopes_work) -{ - ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; - P2R_BakeScopesIn *in = (P2R_BakeScopesIn *)input; - RDIM_ScopeBakeResult *out = push_array(arena, RDIM_ScopeBakeResult, 1); - ProfScope("bake scopes") *out = rdim_bake_scopes(arena, in->strings, in->scopes); - ProfEnd(); - return out; -} - -ASYNC_WORK_DEF(p2r_bake_scope_vmap_work) -{ - ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; - P2R_BakeScopeVMapIn *in = (P2R_BakeScopeVMapIn *)input; - RDIM_ScopeVMapBakeResult *out = push_array(arena, RDIM_ScopeVMapBakeResult, 1); - ProfScope("bake scope vmap") *out = rdim_bake_scope_vmap(arena, in->scopes); - ProfEnd(); - return out; -} - -ASYNC_WORK_DEF(p2r_bake_inline_sites_work) -{ - ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; - P2R_BakeInlineSitesIn *in = (P2R_BakeInlineSitesIn *)input; - RDIM_InlineSiteBakeResult *out = push_array(arena, RDIM_InlineSiteBakeResult, 1); - ProfScope("bake inline sites") *out = rdim_bake_inline_sites(arena, in->strings, in->inline_sites); - ProfEnd(); - return out; -} - -ASYNC_WORK_DEF(p2r_bake_file_paths_work) -{ - ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; - P2R_BakeFilePathsIn *in = (P2R_BakeFilePathsIn *)input; - RDIM_FilePathBakeResult *out = push_array(arena, RDIM_FilePathBakeResult, 1); - ProfScope("bake file paths") *out = rdim_bake_file_paths(arena, in->strings, in->path_tree); - ProfEnd(); - return out; -} - -ASYNC_WORK_DEF(p2r_bake_strings_work) -{ - ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; - P2R_BakeStringsIn *in = (P2R_BakeStringsIn *)input; - RDIM_StringBakeResult *out = push_array(arena, RDIM_StringBakeResult, 1); - ProfScope("bake strings") *out = rdim_bake_strings(arena, in->strings); - ProfEnd(); - return out; -} - -//- rjf: pass 3: idx-run-map-dependent debug info stream builds - -ASYNC_WORK_DEF(p2r_bake_type_nodes_work) -{ - ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; - P2R_BakeTypeNodesIn *in = (P2R_BakeTypeNodesIn *)input; - RDIM_TypeNodeBakeResult *out = push_array(arena, RDIM_TypeNodeBakeResult, 1); - ProfScope("bake type nodes") *out = rdim_bake_types(arena, in->strings, in->idx_runs, in->types); - ProfEnd(); - return out; -} - -ASYNC_WORK_DEF(p2r_bake_name_map_work) -{ - ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; - P2R_BakeNameMapIn *in = (P2R_BakeNameMapIn *)input; - RDIM_NameMapBakeResult *out = push_array(arena, RDIM_NameMapBakeResult, 1); - ProfScope("bake name map %i", in->kind) *out = rdim_bake_name_map(arena, in->strings, in->idx_runs, in->map); - ProfEnd(); - return out; -} - -ASYNC_WORK_DEF(p2r_bake_idx_runs_work) -{ - ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; - P2R_BakeIdxRunsIn *in = (P2R_BakeIdxRunsIn *)input; - RDIM_IndexRunBakeResult *out = push_array(arena, RDIM_IndexRunBakeResult, 1); - ProfScope("bake idx runs") *out = rdim_bake_index_runs(arena, in->idx_runs); - ProfEnd(); - return out; -} - //////////////////////////////// //~ rjf: Top-Level Initialization @@ -4381,468 +4000,14 @@ p2r_init(void) internal P2R_Bake2Serialize * p2r_bake(Arena *arena, P2R_Convert2Bake *in) { - Temp scratch = scratch_begin(&arena, 1); - RDIM_BakeParams *in_params = &in->bake_params; - P2R_Bake2Serialize *out = push_array(arena, P2R_Bake2Serialize, 1); - RDIM_BakeResults *out_results = &out->bake_results; - - ////////////////////////////// - //- rjf: kick off line tables baking - // - ASYNC_Task *bake_line_tables_task = 0; - { - P2R_BakeLineTablesIn *in = push_array(scratch.arena, P2R_BakeLineTablesIn, 1); - in->line_tables = &in_params->line_tables; - bake_line_tables_task = async_task_launch(scratch.arena, p2r_bake_line_tables_work, .input = in); - } - - ////////////////////////////// - //- rjf: build interned path tree - // - RDIM_BakePathTree *path_tree = 0; - ProfScope("build interned path tree") - { - path_tree = rdim_bake_path_tree_from_params(arena, in_params); - } - - ////////////////////////////// - //- rjf: kick off string map building tasks - // - RDIM_BakeStringMapTopology bake_string_map_topology = {(64 + - in_params->procedures.total_count*1 + - in_params->global_variables.total_count*1 + - in_params->thread_variables.total_count*1 + - in_params->types.total_count/2)}; - RDIM_BakeStringMapLoose **bake_string_maps__in_progress = push_array(scratch.arena, RDIM_BakeStringMapLoose *, async_thread_count()); - ASYNC_TaskList bake_string_map_build_tasks = {0}; - { - // rjf: src files - ProfScope("kick off src files string map build task") - { - P2R_BakeSrcFilesStringsIn *in = push_array(scratch.arena, P2R_BakeSrcFilesStringsIn, 1); - in->top = &bake_string_map_topology; - in->maps = bake_string_maps__in_progress; - in->list = &in_params->src_files; - async_task_list_push(scratch.arena, &bake_string_map_build_tasks, async_task_launch(scratch.arena, p2r_bake_src_files_strings_work, .input = in)); - } - - // rjf: units - ProfScope("kick off units string map build task") - { - P2R_BakeUnitsStringsIn *in = push_array(scratch.arena, P2R_BakeUnitsStringsIn, 1); - in->top = &bake_string_map_topology; - in->maps = bake_string_maps__in_progress; - in->list = &in_params->units; - async_task_list_push(scratch.arena, &bake_string_map_build_tasks, async_task_launch(scratch.arena, p2r_bake_units_strings_work, .input = in)); - } - - // rjf: types - ProfScope("kick off types string map build tasks") - { - U64 items_per_task = 4096; - U64 num_tasks = (in_params->types.total_count+items_per_task-1)/items_per_task; - RDIM_TypeChunkNode *chunk = in_params->types.first; - U64 chunk_off = 0; - for(U64 task_idx = 0; task_idx < num_tasks; task_idx += 1) - { - P2R_BakeTypesStringsIn *in = push_array(scratch.arena, P2R_BakeTypesStringsIn, 1); - in->top = &bake_string_map_topology; - in->maps = bake_string_maps__in_progress; - U64 items_left = items_per_task; - for(;chunk != 0 && items_left > 0;) - { - U64 items_in_this_chunk = Min(items_per_task, chunk->count-chunk_off); - P2R_BakeTypesStringsInNode *n = push_array(scratch.arena, P2R_BakeTypesStringsInNode, 1); - SLLQueuePush(in->first, in->last, n); - n->v = chunk->v + chunk_off; - n->count = items_in_this_chunk; - chunk_off += items_in_this_chunk; - items_left -= items_in_this_chunk; - if(chunk_off >= chunk->count) - { - chunk = chunk->next; - chunk_off = 0; - } - } - async_task_list_push(scratch.arena, &bake_string_map_build_tasks, async_task_launch(scratch.arena, p2r_bake_types_strings_work, .input = in)); - } - } - - // rjf: UDTs - ProfScope("kick off udts string map build tasks") - { - U64 items_per_task = 4096; - U64 num_tasks = (in_params->udts.total_count+items_per_task-1)/items_per_task; - RDIM_UDTChunkNode *chunk = in_params->udts.first; - U64 chunk_off = 0; - for(U64 task_idx = 0; task_idx < num_tasks; task_idx += 1) - { - P2R_BakeUDTsStringsIn *in = push_array(scratch.arena, P2R_BakeUDTsStringsIn, 1); - in->top = &bake_string_map_topology; - in->maps = bake_string_maps__in_progress; - U64 items_left = items_per_task; - for(;chunk != 0 && items_left > 0;) - { - U64 items_in_this_chunk = Min(items_per_task, chunk->count-chunk_off); - P2R_BakeUDTsStringsInNode *n = push_array(scratch.arena, P2R_BakeUDTsStringsInNode, 1); - SLLQueuePush(in->first, in->last, n); - n->v = chunk->v + chunk_off; - n->count = items_in_this_chunk; - chunk_off += items_in_this_chunk; - items_left -= items_in_this_chunk; - if(chunk_off >= chunk->count) - { - chunk = chunk->next; - chunk_off = 0; - } - } - async_task_list_push(scratch.arena, &bake_string_map_build_tasks, async_task_launch(scratch.arena, p2r_bake_udts_strings_work, .input = in)); - } - } - - // rjf: symbols - ProfScope("kick off symbols string map build tasks") - { - RDIM_SymbolChunkList *symbol_lists[] = - { - &in_params->global_variables, - &in_params->thread_variables, - &in_params->procedures, - }; - for(U64 list_idx = 0; list_idx < ArrayCount(symbol_lists); list_idx += 1) - { - U64 items_per_task = 4096; - U64 num_tasks = (symbol_lists[list_idx]->total_count+items_per_task-1)/items_per_task; - RDIM_SymbolChunkNode *chunk = symbol_lists[list_idx]->first; - U64 chunk_off = 0; - for(U64 task_idx = 0; task_idx < num_tasks; task_idx += 1) - { - P2R_BakeSymbolsStringsIn *in = push_array(scratch.arena, P2R_BakeSymbolsStringsIn, 1); - in->top = &bake_string_map_topology; - in->maps = bake_string_maps__in_progress; - U64 items_left = items_per_task; - for(;chunk != 0 && items_left > 0;) - { - U64 items_in_this_chunk = Min(items_per_task, chunk->count-chunk_off); - P2R_BakeSymbolsStringsInNode *n = push_array(scratch.arena, P2R_BakeSymbolsStringsInNode, 1); - SLLQueuePush(in->first, in->last, n); - n->v = chunk->v + chunk_off; - n->count = items_in_this_chunk; - chunk_off += items_in_this_chunk; - items_left -= items_in_this_chunk; - if(chunk_off >= chunk->count) - { - chunk = chunk->next; - chunk_off = 0; - } - } - async_task_list_push(scratch.arena, &bake_string_map_build_tasks, async_task_launch(scratch.arena, p2r_bake_symbols_strings_work, .input = in)); - } - } - } + RDIM_HelpState help_state = {0}; + help_state.arena = p2r_state->arena; + help_state.work_thread_arenas_count = p2r_state->work_thread_arenas_count; + help_state.work_thread_arenas = p2r_state->work_thread_arenas; - ProfScope("kick off inline site string map build task") - { - U64 items_per_task = 4096; - U64 num_tasks = CeilIntegerDiv(in_params->inline_sites.total_count, items_per_task); - RDIM_InlineSiteChunkNode *chunk = in_params->inline_sites.first; - U64 chunk_off = 0; - for(U64 task_idx = 0; task_idx < num_tasks; task_idx += 1) - { - P2R_BakeInlineSiteStringsIn *in = push_array(scratch.arena, P2R_BakeInlineSiteStringsIn, 1); - in->top = &bake_string_map_topology; - in->maps = bake_string_maps__in_progress; - U64 items_left = items_per_task; - for(;chunk != 0 && items_left > 0;) - { - U64 items_in_this_chunk = Min(items_per_task, chunk->count-chunk_off); - P2R_BakeInlineSiteStringsInNode *n = push_array(scratch.arena, P2R_BakeInlineSiteStringsInNode, 1); - SLLQueuePush(in->first, in->last, n); - n->v = chunk->v + chunk_off; - n->count = items_in_this_chunk; - chunk_off += items_in_this_chunk; - items_left -= items_in_this_chunk; - if(chunk_off >= chunk->count) - { - chunk = chunk->next; - chunk_off = 0; - } - } - async_task_list_push(scratch.arena, &bake_string_map_build_tasks, async_task_launch(scratch.arena, p2r_bake_inline_site_strings_work, .input = in)); - } - } - - // rjf: scope chunks - ProfScope("kick off scope chunks string map build tasks") - { - U64 items_per_task = 4096; - U64 num_tasks = (in_params->scopes.total_count+items_per_task-1)/items_per_task; - RDIM_ScopeChunkNode *chunk = in_params->scopes.first; - U64 chunk_off = 0; - for(U64 task_idx = 0; task_idx < num_tasks; task_idx += 1) - { - P2R_BakeScopesStringsIn *in = push_array(scratch.arena, P2R_BakeScopesStringsIn, 1); - in->top = &bake_string_map_topology; - in->maps = bake_string_maps__in_progress; - U64 items_left = items_per_task; - for(;chunk != 0 && items_left > 0;) - { - U64 items_in_this_chunk = Min(items_per_task, chunk->count-chunk_off); - P2R_BakeScopesStringsInNode *n = push_array(scratch.arena, P2R_BakeScopesStringsInNode, 1); - SLLQueuePush(in->first, in->last, n); - n->v = chunk->v + chunk_off; - n->count = items_in_this_chunk; - chunk_off += items_in_this_chunk; - items_left -= items_in_this_chunk; - if(chunk_off >= chunk->count) - { - chunk = chunk->next; - chunk_off = 0; - } - } - async_task_list_push(scratch.arena, &bake_string_map_build_tasks, async_task_launch(scratch.arena, p2r_bake_scopes_strings_work, .input = in)); - } - } - } - - ////////////////////////////// - //- rjf: kick off name map building tasks - // - P2R_BuildBakeNameMapIn build_bake_name_map_in[RDI_NameMapKind_COUNT] = {0}; - ASYNC_Task *build_bake_name_map_task[RDI_NameMapKind_COUNT] = {0}; - for(RDI_NameMapKind k = (RDI_NameMapKind)(RDI_NameMapKind_NULL+1); - k < RDI_NameMapKind_COUNT; - k = (RDI_NameMapKind)(k+1)) - { - build_bake_name_map_in[k].k = k; - build_bake_name_map_in[k].params = in_params; - build_bake_name_map_task[k] = async_task_launch(scratch.arena, p2r_build_bake_name_map_work, .input = &build_bake_name_map_in[k]); - } - - ////////////////////////////// - //- rjf: join string map building tasks - // - ProfScope("join string map building tasks") - { - for(ASYNC_TaskNode *n = bake_string_map_build_tasks.first; n != 0; n = n->next) - { - async_task_join(n->v); - } - } - - ////////////////////////////// - //- rjf: produce joined string map - // - RDIM_BakeStringMapLoose *unsorted_bake_string_map = rdim_bake_string_map_loose_make(arena, &bake_string_map_topology); - ProfScope("produce joined string map") - { - U64 slots_per_task = 16384; - U64 num_tasks = (bake_string_map_topology.slots_count+slots_per_task-1)/slots_per_task; - ASYNC_Task **tasks = push_array(scratch.arena, ASYNC_Task *, num_tasks); - - // rjf: kickoff tasks - for(U64 task_idx = 0; task_idx < num_tasks; task_idx += 1) - { - P2R_JoinBakeStringMapSlotsIn *in = push_array(scratch.arena, P2R_JoinBakeStringMapSlotsIn, 1); - in->top = &bake_string_map_topology; - in->src_maps = bake_string_maps__in_progress; - in->src_maps_count = async_thread_count(); - in->dst_map = unsorted_bake_string_map; - in->slot_idx_range = r1u64(task_idx*slots_per_task, task_idx*slots_per_task + slots_per_task); - in->slot_idx_range.max = Min(in->slot_idx_range.max, in->top->slots_count); - tasks[task_idx] = async_task_launch(scratch.arena, p2r_bake_string_map_join_work, .input = in); - } - - // rjf: join tasks - for(U64 task_idx = 0; task_idx < num_tasks; task_idx += 1) - { - async_task_join(tasks[task_idx]); - } - - // rjf: insert small top-level stuff - rdim_bake_string_map_loose_push_top_level_info(arena, &bake_string_map_topology, unsorted_bake_string_map, &in_params->top_level_info); - rdim_bake_string_map_loose_push_binary_sections(arena, &bake_string_map_topology, unsorted_bake_string_map, &in_params->binary_sections); - rdim_bake_string_map_loose_push_path_tree(arena, &bake_string_map_topology, unsorted_bake_string_map, path_tree); - } - - ////////////////////////////// - //- rjf: kick off string map sorting tasks - // - ASYNC_TaskList sort_bake_string_map_tasks = {0}; - RDIM_BakeStringMapLoose *sorted_bake_string_map__in_progress = rdim_bake_string_map_loose_make(arena, &bake_string_map_topology); - { - U64 slots_per_task = 4096; - U64 num_tasks = (bake_string_map_topology.slots_count+slots_per_task-1)/slots_per_task; - for(U64 task_idx = 0; task_idx < num_tasks; task_idx += 1) - { - P2R_SortBakeStringMapSlotsIn *in = push_array(scratch.arena, P2R_SortBakeStringMapSlotsIn, 1); - { - in->top = &bake_string_map_topology; - in->src_map = unsorted_bake_string_map; - in->dst_map = sorted_bake_string_map__in_progress; - in->slot_idx = task_idx*slots_per_task; - in->slot_count = slots_per_task; - if(in->slot_idx+in->slot_count > bake_string_map_topology.slots_count) - { - in->slot_count = bake_string_map_topology.slots_count - in->slot_idx; - } - } - async_task_list_push(scratch.arena, &sort_bake_string_map_tasks, async_task_launch(scratch.arena, p2r_bake_string_map_sort_work, .input = in)); - } - } - - ////////////////////////////// - //- rjf: join string map sorting tasks - // - ProfScope("join string map sorting tasks") - { - for(ASYNC_TaskNode *n = sort_bake_string_map_tasks.first; n != 0; n = n->next) - { - async_task_join(n->v); - } - } - RDIM_BakeStringMapLoose *sorted_bake_string_map = sorted_bake_string_map__in_progress; - - ////////////////////////////// - //- rjf: build finalized string map - // - ProfBegin("build finalized string map base indices"); - RDIM_BakeStringMapBaseIndices bake_string_map_base_idxes = rdim_bake_string_map_base_indices_from_map_loose(arena, &bake_string_map_topology, sorted_bake_string_map); - ProfEnd(); - ProfBegin("build finalized string map"); - RDIM_BakeStringMapTight bake_strings = rdim_bake_string_map_tight_from_loose(arena, &bake_string_map_topology, &bake_string_map_base_idxes, sorted_bake_string_map); - ProfEnd(); - - ////////////////////////////// - //- rjf: kick off pass 2 tasks - // - P2R_BakeUnitsIn bake_units_top_level_in = {&bake_strings, path_tree, &in_params->units}; - ASYNC_Task *bake_units_task = async_task_launch(scratch.arena, p2r_bake_units_work, .input = &bake_units_top_level_in); - P2R_BakeUnitVMapIn bake_unit_vmap_in = {&in_params->units}; - ASYNC_Task *bake_unit_vmap_task = async_task_launch(scratch.arena, p2r_bake_unit_vmap_work, .input = &bake_unit_vmap_in); - P2R_BakeSrcFilesIn bake_src_files_in = {&bake_strings, path_tree, &in_params->src_files}; - ASYNC_Task *bake_src_files_task = async_task_launch(scratch.arena, p2r_bake_src_files_work, .input = &bake_src_files_in); - P2R_BakeUDTsIn bake_udts_in = {&bake_strings, &in_params->udts}; - ASYNC_Task *bake_udts_task = async_task_launch(scratch.arena, p2r_bake_udts_work, .input = &bake_udts_in); - P2R_BakeGlobalVariablesIn bake_global_variables_in = {&bake_strings, &in_params->global_variables}; - ASYNC_Task *bake_global_variables_task = async_task_launch(scratch.arena, p2r_bake_global_variables_work, .input = &bake_global_variables_in); - P2R_BakeGlobalVMapIn bake_global_vmap_in = {&in_params->global_variables}; - ASYNC_Task *bake_global_vmap_task = async_task_launch(scratch.arena, p2r_bake_global_vmap_work, .input = &bake_global_vmap_in); - P2R_BakeThreadVariablesIn bake_thread_variables_in = {&bake_strings, &in_params->thread_variables}; - ASYNC_Task *bake_thread_variables_task = async_task_launch(scratch.arena, p2r_bake_thread_variables_work, .input = &bake_thread_variables_in); - P2R_BakeProceduresIn bake_procedures_in = {&bake_strings, &in_params->procedures}; - ASYNC_Task *bake_procedures_task = async_task_launch(scratch.arena, p2r_bake_procedures_work, .input = &bake_procedures_in); - P2R_BakeScopesIn bake_scopes_in = {&bake_strings, &in_params->scopes}; - ASYNC_Task *bake_scopes_task = async_task_launch(scratch.arena, p2r_bake_scopes_work, .input = &bake_scopes_in); - P2R_BakeScopeVMapIn bake_scope_vmap_in = {&in_params->scopes}; - ASYNC_Task *bake_scope_vmap_task = async_task_launch(scratch.arena, p2r_bake_scope_vmap_work, .input = &bake_scope_vmap_in); - P2R_BakeInlineSitesIn bake_inline_sites_in = {&bake_strings, &in_params->inline_sites}; - ASYNC_Task *bake_inline_sites_task = async_task_launch(scratch.arena, p2r_bake_inline_sites_work, .input = &bake_inline_sites_in); - P2R_BakeFilePathsIn bake_file_paths_in = {&bake_strings, path_tree}; - ASYNC_Task *bake_file_paths_task = async_task_launch(scratch.arena, p2r_bake_file_paths_work, .input = &bake_file_paths_in); - P2R_BakeStringsIn bake_strings_in = {&bake_strings}; - ASYNC_Task *bake_strings_task = async_task_launch(scratch.arena, p2r_bake_strings_work, .input = &bake_strings_in); - - ////////////////////////////// - //- rjf: join name map building tasks - // - RDIM_BakeNameMap *name_maps[RDI_NameMapKind_COUNT] = {0}; - ProfScope("join name map building tasks") - { - for(RDI_NameMapKind k = (RDI_NameMapKind)(RDI_NameMapKind_NULL+1); - k < RDI_NameMapKind_COUNT; - k = (RDI_NameMapKind)(k+1)) - { - name_maps[k] = async_task_join_struct(build_bake_name_map_task[k], RDIM_BakeNameMap); - } - } - - ////////////////////////////// - //- rjf: build interned idx run map - // - RDIM_BakeIdxRunMap *idx_runs = 0; - ProfScope("build interned idx run map") - { - idx_runs = rdim_bake_idx_run_map_from_params(arena, name_maps, in_params); - } - - ////////////////////////////// - //- rjf: do small top-level bakes - // - ProfScope("top level info") out_results->top_level_info = rdim_bake_top_level_info(arena, &bake_strings, &in_params->top_level_info); - ProfScope("binary sections") out_results->binary_sections = rdim_bake_binary_sections(arena, &bake_strings, &in_params->binary_sections); - ProfScope("top level name maps section") out_results->top_level_name_maps = rdim_bake_name_maps_top_level(arena, &bake_strings, idx_runs, name_maps); - - ////////////////////////////// - //- rjf: kick off pass 3 tasks - // - P2R_BakeTypeNodesIn bake_type_nodes_in = {&bake_strings, idx_runs, &in_params->types}; - ASYNC_Task *bake_type_nodes_task = async_task_launch(scratch.arena, p2r_bake_type_nodes_work, .input = &bake_type_nodes_in); - ASYNC_Task *bake_name_maps_tasks[RDI_NameMapKind_COUNT] = {0}; - { - for EachNonZeroEnumVal(RDI_NameMapKind, k) - { - if(name_maps[k] == 0 || name_maps[k]->name_count == 0) - { - continue; - } - P2R_BakeNameMapIn *in = push_array(scratch.arena, P2R_BakeNameMapIn, 1); - in->strings = &bake_strings; - in->idx_runs = idx_runs; - in->map = name_maps[k]; - in->kind = k; - bake_name_maps_tasks[k] = async_task_launch(scratch.arena, p2r_bake_name_map_work, .input = in); - } - } - P2R_BakeIdxRunsIn bake_idx_runs_in = {idx_runs}; - ASYNC_Task *bake_idx_runs_task = async_task_launch(scratch.arena, p2r_bake_idx_runs_work, .input = &bake_idx_runs_in); - - ////////////////////////////// - //- rjf: join remaining completed bakes - // - ProfScope("top-level units info") out_results->units = *async_task_join_struct(bake_units_task, RDIM_UnitBakeResult); - ProfScope("unit vmap") out_results->unit_vmap = *async_task_join_struct(bake_unit_vmap_task, RDIM_UnitVMapBakeResult); - ProfScope("source files") out_results->src_files = *async_task_join_struct(bake_src_files_task, RDIM_SrcFileBakeResult); - ProfScope("UDTs") out_results->udts = *async_task_join_struct(bake_udts_task, RDIM_UDTBakeResult); - ProfScope("global variables") out_results->global_variables = *async_task_join_struct(bake_global_variables_task, RDIM_GlobalVariableBakeResult); - ProfScope("global vmap") out_results->global_vmap = *async_task_join_struct(bake_global_vmap_task, RDIM_GlobalVMapBakeResult); - ProfScope("thread variables") out_results->thread_variables = *async_task_join_struct(bake_thread_variables_task, RDIM_ThreadVariableBakeResult); - ProfScope("procedures") out_results->procedures = *async_task_join_struct(bake_procedures_task, RDIM_ProcedureBakeResult); - ProfScope("scopes") out_results->scopes = *async_task_join_struct(bake_scopes_task, RDIM_ScopeBakeResult); - ProfScope("scope vmap") out_results->scope_vmap = *async_task_join_struct(bake_scope_vmap_task, RDIM_ScopeVMapBakeResult); - ProfScope("inline sites") out_results->inline_sites = *async_task_join_struct(bake_inline_sites_task, RDIM_InlineSiteBakeResult); - ProfScope("file paths") out_results->file_paths = *async_task_join_struct(bake_file_paths_task, RDIM_FilePathBakeResult); - ProfScope("strings") out_results->strings = *async_task_join_struct(bake_strings_task, RDIM_StringBakeResult); - ProfScope("type nodes") out_results->type_nodes = *async_task_join_struct(bake_type_nodes_task, RDIM_TypeNodeBakeResult); - ProfScope("idx runs") out_results->idx_runs = *async_task_join_struct(bake_idx_runs_task, RDIM_IndexRunBakeResult); - ProfScope("line tables") out_results->line_tables = *async_task_join_struct(bake_line_tables_task, RDIM_LineTableBakeResult); - - ////////////////////////////// - //- rjf: join individual name map bakes - // - RDIM_NameMapBakeResult name_map_bakes[RDI_NameMapKind_COUNT] = {0}; - ProfScope("name maps") - { - for EachNonZeroEnumVal(RDI_NameMapKind, k) - { - RDIM_NameMapBakeResult *bake = async_task_join_struct(bake_name_maps_tasks[k], RDIM_NameMapBakeResult); - if(bake != 0) - { - name_map_bakes[k] = *bake; - } - } - } - - ////////////////////////////// - //- rjf: join all individual name map bakes - // - ProfScope("join all name map bakes into final name map bake") - { - out_results->name_maps = rdim_name_map_bake_results_combine(arena, name_map_bakes, ArrayCount(name_map_bakes)); - } - - scratch_end(scratch); - return out; + P2R_Bake2Serialize *result = push_array(arena, P2R_Bake2Serialize, 1); + result->bake_results = rdim_bake(&help_state, &in->bake_params); + return result; } //////////////////////////////// @@ -4852,33 +4017,7 @@ internal P2R_Serialize2File * p2r_compress(Arena *arena, P2R_Serialize2File *in) { P2R_Serialize2File *out = push_array(arena, P2R_Serialize2File, 1); - { - //- rjf: set up compression context - rr_lzb_simple_context ctx = {0}; - ctx.m_tableSizeBits = 14; - ctx.m_hashTable = push_array(arena, U16, 1<bundle.sections[k]; - RDIM_SerializedSection *dst = &out->bundle.sections[k]; - MemoryCopyStruct(dst, src); - - // rjf: determine if this section should be compressed - B32 should_compress = 1; - - // rjf: compress if needed - if(should_compress) - { - MemoryZero(ctx.m_hashTable, sizeof(U16)*(1<data = push_array_no_zero(arena, U8, src->encoded_size); - dst->encoded_size = rr_lzb_simple_encode_veryfast(&ctx, src->data, src->encoded_size, dst->data); - dst->unpacked_size = src->encoded_size; - dst->encoding = RDI_SectionEncoding_LZB; - } - } - } + out->bundle = rdim_compress(arena, &in->bundle); return out; } diff --git a/src/rdi_from_pdb/rdi_from_pdb.h b/src/rdi_from_pdb/rdi_from_pdb.h index ced21a66..03cae2d2 100644 --- a/src/rdi_from_pdb/rdi_from_pdb.h +++ b/src/rdi_from_pdb/rdi_from_pdb.h @@ -273,274 +273,6 @@ struct P2R_SymbolStreamConvertOut RDIM_InlineSiteChunkList inline_sites; }; -//////////////////////////////// -//~ rjf: Baking Task Types - -//- rjf: line table baking task types - -typedef struct P2R_BakeLineTablesIn P2R_BakeLineTablesIn; -struct P2R_BakeLineTablesIn -{ - RDIM_LineTableChunkList *line_tables; -}; - -//- rjf: string map baking task types - -typedef struct P2R_BakeSrcFilesStringsIn P2R_BakeSrcFilesStringsIn; -struct P2R_BakeSrcFilesStringsIn -{ - RDIM_BakeStringMapTopology *top; - RDIM_BakeStringMapLoose **maps; - RDIM_SrcFileChunkList *list; -}; - -typedef struct P2R_BakeUnitsStringsIn P2R_BakeUnitsStringsIn; -struct P2R_BakeUnitsStringsIn -{ - RDIM_BakeStringMapTopology *top; - RDIM_BakeStringMapLoose **maps; - RDIM_UnitChunkList *list; -}; - -typedef struct P2R_BakeTypesStringsInNode P2R_BakeTypesStringsInNode; -struct P2R_BakeTypesStringsInNode -{ - P2R_BakeTypesStringsInNode *next; - RDIM_Type *v; - RDI_U64 count; -}; - -typedef struct P2R_BakeTypesStringsIn P2R_BakeTypesStringsIn; -struct P2R_BakeTypesStringsIn -{ - RDIM_BakeStringMapTopology *top; - RDIM_BakeStringMapLoose **maps; - P2R_BakeTypesStringsInNode *first; - P2R_BakeTypesStringsInNode *last; -}; - -typedef struct P2R_BakeUDTsStringsInNode P2R_BakeUDTsStringsInNode; -struct P2R_BakeUDTsStringsInNode -{ - P2R_BakeUDTsStringsInNode *next; - RDIM_UDT *v; - RDI_U64 count; -}; - -typedef struct P2R_BakeUDTsStringsIn P2R_BakeUDTsStringsIn; -struct P2R_BakeUDTsStringsIn -{ - RDIM_BakeStringMapTopology *top; - RDIM_BakeStringMapLoose **maps; - P2R_BakeUDTsStringsInNode *first; - P2R_BakeUDTsStringsInNode *last; -}; - -typedef struct P2R_BakeSymbolsStringsInNode P2R_BakeSymbolsStringsInNode; -struct P2R_BakeSymbolsStringsInNode -{ - P2R_BakeSymbolsStringsInNode *next; - RDIM_Symbol *v; - RDI_U64 count; -}; - -typedef struct P2R_BakeSymbolsStringsIn P2R_BakeSymbolsStringsIn; -struct P2R_BakeSymbolsStringsIn -{ - RDIM_BakeStringMapTopology *top; - RDIM_BakeStringMapLoose **maps; - P2R_BakeSymbolsStringsInNode *first; - P2R_BakeSymbolsStringsInNode *last; -}; - -typedef struct P2R_BakeInlineSiteStringsInNode P2R_BakeInlineSiteStringsInNode; -struct P2R_BakeInlineSiteStringsInNode -{ - P2R_BakeInlineSiteStringsInNode *next; - RDIM_InlineSite *v; - RDI_U64 count; -}; - -typedef struct P2R_BakeInlineSiteStringsIn P2R_BakeInlineSiteStringsIn; -struct P2R_BakeInlineSiteStringsIn -{ - RDIM_BakeStringMapTopology *top; - RDIM_BakeStringMapLoose **maps; - P2R_BakeInlineSiteStringsInNode *first; - P2R_BakeInlineSiteStringsInNode *last; -}; - -typedef struct P2R_BakeScopesStringsInNode P2R_BakeScopesStringsInNode; -struct P2R_BakeScopesStringsInNode -{ - P2R_BakeScopesStringsInNode *next; - RDIM_Scope *v; - RDI_U64 count; -}; - -typedef struct P2R_BakeScopesStringsIn P2R_BakeScopesStringsIn; -struct P2R_BakeScopesStringsIn -{ - RDIM_BakeStringMapTopology *top; - RDIM_BakeStringMapLoose **maps; - P2R_BakeScopesStringsInNode *first; - P2R_BakeScopesStringsInNode *last; -}; - -//- rjf: string map joining task types - -typedef struct P2R_JoinBakeStringMapSlotsIn P2R_JoinBakeStringMapSlotsIn; -struct P2R_JoinBakeStringMapSlotsIn -{ - RDIM_BakeStringMapTopology *top; - RDIM_BakeStringMapLoose **src_maps; - U64 src_maps_count; - RDIM_BakeStringMapLoose *dst_map; - Rng1U64 slot_idx_range; -}; - -//- rjf: string map sorting task types - -typedef struct P2R_SortBakeStringMapSlotsIn P2R_SortBakeStringMapSlotsIn; -struct P2R_SortBakeStringMapSlotsIn -{ - RDIM_BakeStringMapTopology *top; - RDIM_BakeStringMapLoose *src_map; - RDIM_BakeStringMapLoose *dst_map; - U64 slot_idx; - U64 slot_count; -}; - -//- rjf: OLD string map baking types - -typedef struct P2R_BuildBakeStringMapIn P2R_BuildBakeStringMapIn; -struct P2R_BuildBakeStringMapIn -{ - RDIM_BakePathTree *path_tree; - RDIM_BakeParams *params; -}; - -typedef struct P2R_BuildBakeNameMapIn P2R_BuildBakeNameMapIn; -struct P2R_BuildBakeNameMapIn -{ - RDI_NameMapKind k; - RDIM_BakeParams *params; -}; - -//- rjf: debug info baking task types - -typedef struct P2R_BakeUnitsIn P2R_BakeUnitsIn; -struct P2R_BakeUnitsIn -{ - RDIM_BakeStringMapTight *strings; - RDIM_BakePathTree *path_tree; - RDIM_UnitChunkList *units; -}; - -typedef struct P2R_BakeUnitVMapIn P2R_BakeUnitVMapIn; -struct P2R_BakeUnitVMapIn -{ - RDIM_UnitChunkList *units; -}; - -typedef struct P2R_BakeSrcFilesIn P2R_BakeSrcFilesIn; -struct P2R_BakeSrcFilesIn -{ - RDIM_BakeStringMapTight *strings; - RDIM_BakePathTree *path_tree; - RDIM_SrcFileChunkList *src_files; -}; - -typedef struct P2R_BakeUDTsIn P2R_BakeUDTsIn; -struct P2R_BakeUDTsIn -{ - RDIM_BakeStringMapTight *strings; - RDIM_UDTChunkList *udts; -}; - -typedef struct P2R_BakeGlobalVariablesIn P2R_BakeGlobalVariablesIn; -struct P2R_BakeGlobalVariablesIn -{ - RDIM_BakeStringMapTight *strings; - RDIM_SymbolChunkList *global_variables; -}; - -typedef struct P2R_BakeGlobalVMapIn P2R_BakeGlobalVMapIn; -struct P2R_BakeGlobalVMapIn -{ - RDIM_SymbolChunkList *global_variables; -}; - -typedef struct P2R_BakeThreadVariablesIn P2R_BakeThreadVariablesIn; -struct P2R_BakeThreadVariablesIn -{ - RDIM_BakeStringMapTight *strings; - RDIM_SymbolChunkList *thread_variables; -}; - -typedef struct P2R_BakeProceduresIn P2R_BakeProceduresIn; -struct P2R_BakeProceduresIn -{ - RDIM_BakeStringMapTight *strings; - RDIM_SymbolChunkList *procedures; -}; - -typedef struct P2R_BakeScopesIn P2R_BakeScopesIn; -struct P2R_BakeScopesIn -{ - RDIM_BakeStringMapTight *strings; - RDIM_ScopeChunkList *scopes; -}; - -typedef struct P2R_BakeScopeVMapIn P2R_BakeScopeVMapIn; -struct P2R_BakeScopeVMapIn -{ - RDIM_ScopeChunkList *scopes; -}; - -typedef struct P2R_BakeInlineSitesIn P2R_BakeInlineSitesIn; -struct P2R_BakeInlineSitesIn -{ - RDIM_BakeStringMapTight *strings; - RDIM_InlineSiteChunkList *inline_sites; -}; - -typedef struct P2R_BakeFilePathsIn P2R_BakeFilePathsIn; -struct P2R_BakeFilePathsIn -{ - RDIM_BakeStringMapTight *strings; - RDIM_BakePathTree *path_tree; -}; - -typedef struct P2R_BakeStringsIn P2R_BakeStringsIn; -struct P2R_BakeStringsIn -{ - RDIM_BakeStringMapTight *strings; -}; - -typedef struct P2R_BakeTypeNodesIn P2R_BakeTypeNodesIn; -struct P2R_BakeTypeNodesIn -{ - RDIM_BakeStringMapTight *strings; - RDIM_BakeIdxRunMap *idx_runs; - RDIM_TypeChunkList *types; -}; - -typedef struct P2R_BakeNameMapIn P2R_BakeNameMapIn; -struct P2R_BakeNameMapIn -{ - RDIM_BakeStringMapTight *strings; - RDIM_BakeIdxRunMap *idx_runs; - RDIM_BakeNameMap *map; - RDI_NameMapKind kind; -}; - -typedef struct P2R_BakeIdxRunsIn P2R_BakeIdxRunsIn; -struct P2R_BakeIdxRunsIn -{ - RDIM_BakeIdxRunMap *idx_runs; -}; - //////////////////////////////// //~ rjf: Top-Level State @@ -630,46 +362,6 @@ ASYNC_WORK_DEF(p2r_symbol_stream_convert_work); internal P2R_Convert2Bake *p2r_convert(Arena *arena, P2R_User2Convert *in); -//////////////////////////////// -//~ rjf: Baking Stage Tasks - -//- rjf: unsorted bake string map building -ASYNC_WORK_DEF(p2r_bake_src_files_strings_work); -ASYNC_WORK_DEF(p2r_bake_units_strings_work); -ASYNC_WORK_DEF(p2r_bake_types_strings_work); -ASYNC_WORK_DEF(p2r_bake_udts_strings_work); -ASYNC_WORK_DEF(p2r_bake_symbols_strings_work); -ASYNC_WORK_DEF(p2r_bake_scopes_strings_work); -ASYNC_WORK_DEF(p2r_bake_line_tables_work); - -//- rjf: bake string map joining -ASYNC_WORK_DEF(p2r_bake_string_map_join_work); - -//- rjf: bake string map sorting -ASYNC_WORK_DEF(p2r_bake_string_map_sort_work); - -//- rjf: pass 1: interner/deduper map builds -ASYNC_WORK_DEF(p2r_build_bake_name_map_work); - -//- rjf: pass 2: string-map-dependent debug info stream builds -ASYNC_WORK_DEF(p2r_bake_units_work); -ASYNC_WORK_DEF(p2r_bake_unit_vmap_work); -ASYNC_WORK_DEF(p2r_bake_src_files_work); -ASYNC_WORK_DEF(p2r_bake_udts_work); -ASYNC_WORK_DEF(p2r_bake_global_variables_work); -ASYNC_WORK_DEF(p2r_bake_global_vmap_work); -ASYNC_WORK_DEF(p2r_bake_thread_variables_work); -ASYNC_WORK_DEF(p2r_bake_procedures_work); -ASYNC_WORK_DEF(p2r_bake_scopes_work); -ASYNC_WORK_DEF(p2r_bake_scope_vmap_work); -ASYNC_WORK_DEF(p2r_bake_file_paths_work); -ASYNC_WORK_DEF(p2r_bake_strings_work); - -//- rjf: pass 3: idx-run-map-dependent debug info stream builds -ASYNC_WORK_DEF(p2r_bake_type_nodes_work); -ASYNC_WORK_DEF(p2r_bake_name_map_work); -ASYNC_WORK_DEF(p2r_bake_idx_runs_work); - //////////////////////////////// //~ rjf: Top-Level Initialization diff --git a/src/rdi_from_pdb/rdi_from_pdb_main.c b/src/rdi_from_pdb/rdi_from_pdb_main.c index 9b2e49b5..8b24e350 100644 --- a/src/rdi_from_pdb/rdi_from_pdb_main.c +++ b/src/rdi_from_pdb/rdi_from_pdb_main.c @@ -21,6 +21,7 @@ #include "os/os_inc.h" #include "async/async.h" #include "rdi_make/rdi_make_local.h" +#include "rdi_make/rdi_make_help.h" #include "coff/coff.h" #include "coff/coff_parse.h" #include "codeview/codeview.h" @@ -37,6 +38,7 @@ #include "os/os_inc.c" #include "async/async.c" #include "rdi_make/rdi_make_local.c" +#include "rdi_make/rdi_make_help.c" #include "coff/coff.c" #include "coff/coff_parse.c" #include "codeview/codeview.c" diff --git a/src/rdi_make/rdi_make_help.c b/src/rdi_make/rdi_make_help.c new file mode 100644 index 00000000..488030c0 --- /dev/null +++ b/src/rdi_make/rdi_make_help.c @@ -0,0 +1,901 @@ +// Copyright (c) 2024 Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +//////////////////////////////// +//~ rjf: Baking Stage Tasks + +//- rjf: bake string map building + +#define rdim_make_string_map_if_needed() do {if(in->maps[thread_idx] == 0) ProfScope("make map") {in->maps[thread_idx] = rdim_bake_string_map_loose_make(arena, in->top);}} while(0) + +ASYNC_WORK_DEF(rdim_bake_src_files_strings_work) +{ + ProfBeginFunction(); + Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; + RDIM_BakeSrcFilesStringsIn *in = (RDIM_BakeSrcFilesStringsIn *)input; + rdim_make_string_map_if_needed(); + ProfScope("bake src file strings") rdim_bake_string_map_loose_push_src_files(arena, in->top, in->maps[thread_idx], in->list); + ProfEnd(); + return 0; +} + +ASYNC_WORK_DEF(rdim_bake_units_strings_work) +{ + ProfBeginFunction(); + Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; + RDIM_BakeUnitsStringsIn *in = (RDIM_BakeUnitsStringsIn *)input; + rdim_make_string_map_if_needed(); + ProfScope("bake unit strings") rdim_bake_string_map_loose_push_units(arena, in->top, in->maps[thread_idx], in->list); + ProfEnd(); + return 0; +} + +ASYNC_WORK_DEF(rdim_bake_types_strings_work) +{ + ProfBeginFunction(); + Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; + RDIM_BakeTypesStringsIn *in = (RDIM_BakeTypesStringsIn *)input; + rdim_make_string_map_if_needed(); + ProfScope("bake type strings") + { + for(RDIM_BakeTypesStringsInNode *n = in->first; n != 0; n = n->next) + { + rdim_bake_string_map_loose_push_type_slice(arena, in->top, in->maps[thread_idx], n->v, n->count); + } + } + ProfEnd(); + return 0; +} + +ASYNC_WORK_DEF(rdim_bake_udts_strings_work) +{ + ProfBeginFunction(); + Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; + RDIM_BakeUDTsStringsIn *in = (RDIM_BakeUDTsStringsIn *)input; + rdim_make_string_map_if_needed(); + ProfScope("bake udt strings") + { + for(RDIM_BakeUDTsStringsInNode *n = in->first; n != 0; n = n->next) + { + rdim_bake_string_map_loose_push_udt_slice(arena, in->top, in->maps[thread_idx], n->v, n->count); + } + } + ProfEnd(); + return 0; +} + +ASYNC_WORK_DEF(rdim_bake_symbols_strings_work) +{ + ProfBeginFunction(); + Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; + RDIM_BakeSymbolsStringsIn *in = (RDIM_BakeSymbolsStringsIn *)input; + rdim_make_string_map_if_needed(); + ProfScope("bake symbol strings") + { + for(RDIM_BakeSymbolsStringsInNode *n = in->first; n != 0; n = n->next) + { + rdim_bake_string_map_loose_push_symbol_slice(arena, in->top, in->maps[thread_idx], n->v, n->count); + } + } + ProfEnd(); + return 0; +} + +ASYNC_WORK_DEF(rdim_bake_inline_site_strings_work) +{ + ProfBeginFunction(); + Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; + RDIM_BakeInlineSiteStringsIn *in = input; + rdim_make_string_map_if_needed(); + ProfScope("bake inline site strings") + { + for(RDIM_BakeInlineSiteStringsInNode *n = in->first; n != 0; n = n->next) + { + rdim_bake_string_map_loose_push_inline_site_slice(arena, in->top, in->maps[thread_idx], n->v, n->count); + } + } + ProfEnd(); + return 0; +} + +ASYNC_WORK_DEF(rdim_bake_scopes_strings_work) +{ + ProfBeginFunction(); + Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; + RDIM_BakeScopesStringsIn *in = (RDIM_BakeScopesStringsIn *)input; + rdim_make_string_map_if_needed(); + ProfScope("bake scope strings") + { + for(RDIM_BakeScopesStringsInNode *n = in->first; n != 0; n = n->next) + { + rdim_bake_string_map_loose_push_scope_slice(arena, in->top, in->maps[thread_idx], n->v, n->count); + } + } + ProfEnd(); + return 0; +} + +ASYNC_WORK_DEF(rdim_bake_line_tables_work) +{ + ProfBeginFunction(); + Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; + RDIM_BakeLineTablesIn *in = (RDIM_BakeLineTablesIn *)input; + RDIM_LineTableBakeResult *out = push_array(arena, RDIM_LineTableBakeResult, 1); + ProfScope("bake line tables") *out = rdim_bake_line_tables(arena, in->line_tables); + ProfEnd(); + return out; +} + +#undef rdim_make_string_map_if_needed + +//- rjf: bake string map joining + +ASYNC_WORK_DEF(rdim_bake_string_map_join_work) +{ + ProfBeginFunction(); + Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; + RDIM_JoinBakeStringMapSlotsIn *in = (RDIM_JoinBakeStringMapSlotsIn *)input; + ProfScope("join bake string maps") + { + for(U64 src_map_idx = 0; src_map_idx < in->src_maps_count; src_map_idx += 1) + { + for(U64 slot_idx = in->slot_idx_range.min; slot_idx < in->slot_idx_range.max; slot_idx += 1) + { + B32 src_slots_good = (in->src_maps[src_map_idx] != 0 && in->src_maps[src_map_idx]->slots != 0); + B32 dst_slot_is_zero = (in->dst_map->slots[slot_idx] == 0); + if(src_slots_good && dst_slot_is_zero) + { + in->dst_map->slots[slot_idx] = in->src_maps[src_map_idx]->slots[slot_idx]; + } + else if(src_slots_good && in->src_maps[src_map_idx]->slots[slot_idx] != 0) + { + rdim_bake_string_chunk_list_concat_in_place(in->dst_map->slots[slot_idx], in->src_maps[src_map_idx]->slots[slot_idx]); + } + } + } + } + ProfEnd(); + return 0; +} + +//- rjf: bake string map sorting + +ASYNC_WORK_DEF(rdim_bake_string_map_sort_work) +{ + ProfBeginFunction(); + Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; + RDIM_SortBakeStringMapSlotsIn *in = (RDIM_SortBakeStringMapSlotsIn *)input; + ProfScope("sort bake string chunk list map range") + { + for(U64 slot_idx = in->slot_idx; + slot_idx < in->slot_idx+in->slot_count; + slot_idx += 1) + { + if(in->src_map->slots[slot_idx] != 0) + { + if(in->src_map->slots[slot_idx]->total_count > 1) + { + in->dst_map->slots[slot_idx] = push_array(arena, RDIM_BakeStringChunkList, 1); + *in->dst_map->slots[slot_idx] = rdim_bake_string_chunk_list_sorted_from_unsorted(arena, in->src_map->slots[slot_idx]); + } + else + { + in->dst_map->slots[slot_idx] = in->src_map->slots[slot_idx]; + } + } + } + } + ProfEnd(); + return 0; +} + +//- rjf: pass 1: interner/deduper map builds + +ASYNC_WORK_DEF(rdim_build_bake_name_map_work) +{ + ProfBeginFunction(); + Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; + RDIM_BuildBakeNameMapIn *in = (RDIM_BuildBakeNameMapIn *)input; + RDIM_BakeNameMap *name_map = 0; + ProfScope("build name map %i", in->k) name_map = rdim_bake_name_map_from_kind_params(arena, in->k, in->params); + ProfEnd(); + return name_map; +} + +//- rjf: pass 2: string-map-dependent debug info stream builds + +ASYNC_WORK_DEF(rdim_bake_units_work) +{ + ProfBeginFunction(); + Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; + RDIM_BakeUnitsIn *in = (RDIM_BakeUnitsIn *)input; + RDIM_UnitBakeResult *out = push_array(arena, RDIM_UnitBakeResult, 1); + ProfScope("bake units") *out = rdim_bake_units(arena, in->strings, in->path_tree, in->units); + ProfEnd(); + return out; +} + +ASYNC_WORK_DEF(rdim_bake_unit_vmap_work) +{ + ProfBeginFunction(); + Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; + RDIM_BakeUnitVMapIn *in = (RDIM_BakeUnitVMapIn *)input; + RDIM_UnitVMapBakeResult *out = push_array(arena, RDIM_UnitVMapBakeResult, 1); + ProfScope("bake unit vmap") *out = rdim_bake_unit_vmap(arena, in->units); + ProfEnd(); + return out; +} + +ASYNC_WORK_DEF(rdim_bake_src_files_work) +{ + ProfBeginFunction(); + Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; + RDIM_BakeSrcFilesIn *in = (RDIM_BakeSrcFilesIn *)input; + RDIM_SrcFileBakeResult *out = push_array(arena, RDIM_SrcFileBakeResult, 1); + ProfScope("bake src files") *out = rdim_bake_src_files(arena, in->strings, in->path_tree, in->src_files); + ProfEnd(); + return out; +} + +ASYNC_WORK_DEF(rdim_bake_udts_work) +{ + ProfBeginFunction(); + Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; + RDIM_BakeUDTsIn *in = (RDIM_BakeUDTsIn *)input; + RDIM_UDTBakeResult *out = push_array(arena, RDIM_UDTBakeResult, 1); + ProfScope("bake udts") *out = rdim_bake_udts(arena, in->strings, in->udts); + ProfEnd(); + return out; +} + +ASYNC_WORK_DEF(rdim_bake_global_variables_work) +{ + ProfBeginFunction(); + Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; + RDIM_BakeGlobalVariablesIn *in = (RDIM_BakeGlobalVariablesIn *)input; + RDIM_GlobalVariableBakeResult *out = push_array(arena, RDIM_GlobalVariableBakeResult, 1); + ProfScope("bake global variables") *out = rdim_bake_global_variables(arena, in->strings, in->global_variables); + ProfEnd(); + return out; +} + +ASYNC_WORK_DEF(rdim_bake_global_vmap_work) +{ + ProfBeginFunction(); + Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; + RDIM_BakeGlobalVMapIn *in = (RDIM_BakeGlobalVMapIn *)input; + RDIM_GlobalVMapBakeResult *out = push_array(arena, RDIM_GlobalVMapBakeResult, 1); + ProfScope("bake global vmap") *out = rdim_bake_global_vmap(arena, in->global_variables); + ProfEnd(); + return out; +} + +ASYNC_WORK_DEF(rdim_bake_thread_variables_work) +{ + ProfBeginFunction(); + Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; + RDIM_BakeThreadVariablesIn *in = (RDIM_BakeThreadVariablesIn *)input; + RDIM_ThreadVariableBakeResult *out = push_array(arena, RDIM_ThreadVariableBakeResult, 1); + ProfScope("bake thread variables") *out = rdim_bake_thread_variables(arena, in->strings, in->thread_variables); + ProfEnd(); + return out; +} + +ASYNC_WORK_DEF(rdim_bake_procedures_work) +{ + ProfBeginFunction(); + Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; + RDIM_BakeProceduresIn *in = (RDIM_BakeProceduresIn *)input; + RDIM_ProcedureBakeResult *out = push_array(arena, RDIM_ProcedureBakeResult, 1); + ProfScope("bake procedures") *out = rdim_bake_procedures(arena, in->strings, in->procedures); + ProfEnd(); + return out; +} + +ASYNC_WORK_DEF(rdim_bake_scopes_work) +{ + ProfBeginFunction(); + Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; + RDIM_BakeScopesIn *in = (RDIM_BakeScopesIn *)input; + RDIM_ScopeBakeResult *out = push_array(arena, RDIM_ScopeBakeResult, 1); + ProfScope("bake scopes") *out = rdim_bake_scopes(arena, in->strings, in->scopes); + ProfEnd(); + return out; +} + +ASYNC_WORK_DEF(rdim_bake_scope_vmap_work) +{ + ProfBeginFunction(); + Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; + RDIM_BakeScopeVMapIn *in = (RDIM_BakeScopeVMapIn *)input; + RDIM_ScopeVMapBakeResult *out = push_array(arena, RDIM_ScopeVMapBakeResult, 1); + ProfScope("bake scope vmap") *out = rdim_bake_scope_vmap(arena, in->scopes); + ProfEnd(); + return out; +} + +ASYNC_WORK_DEF(rdim_bake_inline_sites_work) +{ + ProfBeginFunction(); + Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; + RDIM_BakeInlineSitesIn *in = (RDIM_BakeInlineSitesIn *)input; + RDIM_InlineSiteBakeResult *out = push_array(arena, RDIM_InlineSiteBakeResult, 1); + ProfScope("bake inline sites") *out = rdim_bake_inline_sites(arena, in->strings, in->inline_sites); + ProfEnd(); + return out; +} + +ASYNC_WORK_DEF(rdim_bake_file_paths_work) +{ + ProfBeginFunction(); + Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; + RDIM_BakeFilePathsIn *in = (RDIM_BakeFilePathsIn *)input; + RDIM_FilePathBakeResult *out = push_array(arena, RDIM_FilePathBakeResult, 1); + ProfScope("bake file paths") *out = rdim_bake_file_paths(arena, in->strings, in->path_tree); + ProfEnd(); + return out; +} + +ASYNC_WORK_DEF(rdim_bake_strings_work) +{ + ProfBeginFunction(); + Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; + RDIM_BakeStringsIn *in = (RDIM_BakeStringsIn *)input; + RDIM_StringBakeResult *out = push_array(arena, RDIM_StringBakeResult, 1); + ProfScope("bake strings") *out = rdim_bake_strings(arena, in->strings); + ProfEnd(); + return out; +} + +//- rjf: pass 3: idx-run-map-dependent debug info stream builds + +ASYNC_WORK_DEF(rdim_bake_type_nodes_work) +{ + ProfBeginFunction(); + Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; + RDIM_BakeTypeNodesIn *in = (RDIM_BakeTypeNodesIn *)input; + RDIM_TypeNodeBakeResult *out = push_array(arena, RDIM_TypeNodeBakeResult, 1); + ProfScope("bake type nodes") *out = rdim_bake_types(arena, in->strings, in->idx_runs, in->types); + ProfEnd(); + return out; +} + +ASYNC_WORK_DEF(rdim_bake_name_map_work) +{ + ProfBeginFunction(); + Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; + RDIM_BakeNameMapIn *in = (RDIM_BakeNameMapIn *)input; + RDIM_NameMapBakeResult *out = push_array(arena, RDIM_NameMapBakeResult, 1); + ProfScope("bake name map %i", in->kind) *out = rdim_bake_name_map(arena, in->strings, in->idx_runs, in->map); + ProfEnd(); + return out; +} + +ASYNC_WORK_DEF(rdim_bake_idx_runs_work) +{ + ProfBeginFunction(); + Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; + RDIM_BakeIdxRunsIn *in = (RDIM_BakeIdxRunsIn *)input; + RDIM_IndexRunBakeResult *out = push_array(arena, RDIM_IndexRunBakeResult, 1); + ProfScope("bake idx runs") *out = rdim_bake_index_runs(arena, in->idx_runs); + ProfEnd(); + return out; +} + +internal RDIM_HelpState * +rdim_help_init(void) +{ + Arena *arena = arena_alloc(); + RDIM_HelpState *state = push_array(arena, RDIM_HelpState, 1); + state->arena = arena; + state->work_thread_arenas_count = async_thread_count(); + state->work_thread_arenas = push_array(arena, Arena *, state->work_thread_arenas_count); + for EachIndex(idx, state->work_thread_arenas_count) + { + state->work_thread_arenas[idx] = arena_alloc(); + } + return state; +} + +internal RDIM_BakeResults +rdim_bake(RDIM_HelpState *state, RDIM_BakeParams *in_params) +{ + Temp scratch = scratch_begin(0,0); + RDIM_BakeResults out = {0}; + + rdim_help_state = state; + + ////////////////////////////// + //- rjf: kick off line tables baking + // + ASYNC_Task *bake_line_tables_task = 0; + { + RDIM_BakeLineTablesIn *in = push_array(scratch.arena, RDIM_BakeLineTablesIn, 1); + in->line_tables = &in_params->line_tables; + bake_line_tables_task = async_task_launch(scratch.arena, rdim_bake_line_tables_work, .input = in); + } + + ////////////////////////////// + //- rjf: build interned path tree + // + RDIM_BakePathTree *path_tree = 0; + ProfScope("build interned path tree") + { + path_tree = rdim_bake_path_tree_from_params(state->work_thread_arenas[0], in_params); + } + + ////////////////////////////// + //- rjf: kick off string map building tasks + // + RDIM_BakeStringMapTopology bake_string_map_topology = {(64 + + in_params->procedures.total_count*1 + + in_params->global_variables.total_count*1 + + in_params->thread_variables.total_count*1 + + in_params->types.total_count/2)}; + RDIM_BakeStringMapLoose **bake_string_maps__in_progress = push_array(scratch.arena, RDIM_BakeStringMapLoose *, async_thread_count()); + ASYNC_TaskList bake_string_map_build_tasks = {0}; + { + // rjf: src files + ProfScope("kick off src files string map build task") + { + RDIM_BakeSrcFilesStringsIn *in = push_array(scratch.arena, RDIM_BakeSrcFilesStringsIn, 1); + in->top = &bake_string_map_topology; + in->maps = bake_string_maps__in_progress; + in->list = &in_params->src_files; + async_task_list_push(scratch.arena, &bake_string_map_build_tasks, async_task_launch(scratch.arena, rdim_bake_src_files_strings_work, .input = in)); + } + + // rjf: units + ProfScope("kick off units string map build task") + { + RDIM_BakeUnitsStringsIn *in = push_array(scratch.arena, RDIM_BakeUnitsStringsIn, 1); + in->top = &bake_string_map_topology; + in->maps = bake_string_maps__in_progress; + in->list = &in_params->units; + async_task_list_push(scratch.arena, &bake_string_map_build_tasks, async_task_launch(scratch.arena, rdim_bake_units_strings_work, .input = in)); + } + + // rjf: types + ProfScope("kick off types string map build tasks") + { + U64 items_per_task = 4096; + U64 num_tasks = (in_params->types.total_count+items_per_task-1)/items_per_task; + RDIM_TypeChunkNode *chunk = in_params->types.first; + U64 chunk_off = 0; + for(U64 task_idx = 0; task_idx < num_tasks; task_idx += 1) + { + RDIM_BakeTypesStringsIn *in = push_array(scratch.arena, RDIM_BakeTypesStringsIn, 1); + in->top = &bake_string_map_topology; + in->maps = bake_string_maps__in_progress; + U64 items_left = items_per_task; + for(;chunk != 0 && items_left > 0;) + { + U64 items_in_this_chunk = Min(items_per_task, chunk->count-chunk_off); + RDIM_BakeTypesStringsInNode *n = push_array(scratch.arena, RDIM_BakeTypesStringsInNode, 1); + SLLQueuePush(in->first, in->last, n); + n->v = chunk->v + chunk_off; + n->count = items_in_this_chunk; + chunk_off += items_in_this_chunk; + items_left -= items_in_this_chunk; + if(chunk_off >= chunk->count) + { + chunk = chunk->next; + chunk_off = 0; + } + } + async_task_list_push(scratch.arena, &bake_string_map_build_tasks, async_task_launch(scratch.arena, rdim_bake_types_strings_work, .input = in)); + } + } + + // rjf: UDTs + ProfScope("kick off udts string map build tasks") + { + U64 items_per_task = 4096; + U64 num_tasks = (in_params->udts.total_count+items_per_task-1)/items_per_task; + RDIM_UDTChunkNode *chunk = in_params->udts.first; + U64 chunk_off = 0; + for(U64 task_idx = 0; task_idx < num_tasks; task_idx += 1) + { + RDIM_BakeUDTsStringsIn *in = push_array(scratch.arena, RDIM_BakeUDTsStringsIn, 1); + in->top = &bake_string_map_topology; + in->maps = bake_string_maps__in_progress; + U64 items_left = items_per_task; + for(;chunk != 0 && items_left > 0;) + { + U64 items_in_this_chunk = Min(items_per_task, chunk->count-chunk_off); + RDIM_BakeUDTsStringsInNode *n = push_array(scratch.arena, RDIM_BakeUDTsStringsInNode, 1); + SLLQueuePush(in->first, in->last, n); + n->v = chunk->v + chunk_off; + n->count = items_in_this_chunk; + chunk_off += items_in_this_chunk; + items_left -= items_in_this_chunk; + if(chunk_off >= chunk->count) + { + chunk = chunk->next; + chunk_off = 0; + } + } + async_task_list_push(scratch.arena, &bake_string_map_build_tasks, async_task_launch(scratch.arena, rdim_bake_udts_strings_work, .input = in)); + } + } + + // rjf: symbols + ProfScope("kick off symbols string map build tasks") + { + RDIM_SymbolChunkList *symbol_lists[] = + { + &in_params->global_variables, + &in_params->thread_variables, + &in_params->procedures, + }; + for(U64 list_idx = 0; list_idx < ArrayCount(symbol_lists); list_idx += 1) + { + U64 items_per_task = 4096; + U64 num_tasks = (symbol_lists[list_idx]->total_count+items_per_task-1)/items_per_task; + RDIM_SymbolChunkNode *chunk = symbol_lists[list_idx]->first; + U64 chunk_off = 0; + for(U64 task_idx = 0; task_idx < num_tasks; task_idx += 1) + { + RDIM_BakeSymbolsStringsIn *in = push_array(scratch.arena, RDIM_BakeSymbolsStringsIn, 1); + in->top = &bake_string_map_topology; + in->maps = bake_string_maps__in_progress; + U64 items_left = items_per_task; + for(;chunk != 0 && items_left > 0;) + { + U64 items_in_this_chunk = Min(items_per_task, chunk->count-chunk_off); + RDIM_BakeSymbolsStringsInNode *n = push_array(scratch.arena, RDIM_BakeSymbolsStringsInNode, 1); + SLLQueuePush(in->first, in->last, n); + n->v = chunk->v + chunk_off; + n->count = items_in_this_chunk; + chunk_off += items_in_this_chunk; + items_left -= items_in_this_chunk; + if(chunk_off >= chunk->count) + { + chunk = chunk->next; + chunk_off = 0; + } + } + async_task_list_push(scratch.arena, &bake_string_map_build_tasks, async_task_launch(scratch.arena, rdim_bake_symbols_strings_work, .input = in)); + } + } + } + + ProfScope("kick off inline site string map build task") + { + U64 items_per_task = 4096; + U64 num_tasks = CeilIntegerDiv(in_params->inline_sites.total_count, items_per_task); + RDIM_InlineSiteChunkNode *chunk = in_params->inline_sites.first; + U64 chunk_off = 0; + for(U64 task_idx = 0; task_idx < num_tasks; task_idx += 1) + { + RDIM_BakeInlineSiteStringsIn *in = push_array(scratch.arena, RDIM_BakeInlineSiteStringsIn, 1); + in->top = &bake_string_map_topology; + in->maps = bake_string_maps__in_progress; + U64 items_left = items_per_task; + for(;chunk != 0 && items_left > 0;) + { + U64 items_in_this_chunk = Min(items_per_task, chunk->count-chunk_off); + RDIM_BakeInlineSiteStringsInNode *n = push_array(scratch.arena, RDIM_BakeInlineSiteStringsInNode, 1); + SLLQueuePush(in->first, in->last, n); + n->v = chunk->v + chunk_off; + n->count = items_in_this_chunk; + chunk_off += items_in_this_chunk; + items_left -= items_in_this_chunk; + if(chunk_off >= chunk->count) + { + chunk = chunk->next; + chunk_off = 0; + } + } + async_task_list_push(scratch.arena, &bake_string_map_build_tasks, async_task_launch(scratch.arena, rdim_bake_inline_site_strings_work, .input = in)); + } + } + + // rjf: scope chunks + ProfScope("kick off scope chunks string map build tasks") + { + U64 items_per_task = 4096; + U64 num_tasks = (in_params->scopes.total_count+items_per_task-1)/items_per_task; + RDIM_ScopeChunkNode *chunk = in_params->scopes.first; + U64 chunk_off = 0; + for(U64 task_idx = 0; task_idx < num_tasks; task_idx += 1) + { + RDIM_BakeScopesStringsIn *in = push_array(scratch.arena, RDIM_BakeScopesStringsIn, 1); + in->top = &bake_string_map_topology; + in->maps = bake_string_maps__in_progress; + U64 items_left = items_per_task; + for(;chunk != 0 && items_left > 0;) + { + U64 items_in_this_chunk = Min(items_per_task, chunk->count-chunk_off); + RDIM_BakeScopesStringsInNode *n = push_array(scratch.arena, RDIM_BakeScopesStringsInNode, 1); + SLLQueuePush(in->first, in->last, n); + n->v = chunk->v + chunk_off; + n->count = items_in_this_chunk; + chunk_off += items_in_this_chunk; + items_left -= items_in_this_chunk; + if(chunk_off >= chunk->count) + { + chunk = chunk->next; + chunk_off = 0; + } + } + async_task_list_push(scratch.arena, &bake_string_map_build_tasks, async_task_launch(scratch.arena, rdim_bake_scopes_strings_work, .input = in)); + } + } + } + + ////////////////////////////// + //- rjf: kick off name map building tasks + // + RDIM_BuildBakeNameMapIn build_bake_name_map_in[RDI_NameMapKind_COUNT] = {0}; + ASYNC_Task *build_bake_name_map_task[RDI_NameMapKind_COUNT] = {0}; + for(RDI_NameMapKind k = (RDI_NameMapKind)(RDI_NameMapKind_NULL+1); + k < RDI_NameMapKind_COUNT; + k = (RDI_NameMapKind)(k+1)) + { + build_bake_name_map_in[k].k = k; + build_bake_name_map_in[k].params = in_params; + build_bake_name_map_task[k] = async_task_launch(scratch.arena, rdim_build_bake_name_map_work, .input = &build_bake_name_map_in[k]); + } + + ////////////////////////////// + //- rjf: join string map building tasks + // + ProfScope("join string map building tasks") + { + for(ASYNC_TaskNode *n = bake_string_map_build_tasks.first; n != 0; n = n->next) + { + async_task_join(n->v); + } + } + + ////////////////////////////// + //- rjf: produce joined string map + // + RDIM_BakeStringMapLoose *unsorted_bake_string_map = rdim_bake_string_map_loose_make(state->work_thread_arenas[0], &bake_string_map_topology); + ProfScope("produce joined string map") + { + U64 slots_per_task = 16384; + U64 num_tasks = (bake_string_map_topology.slots_count+slots_per_task-1)/slots_per_task; + ASYNC_Task **tasks = push_array(scratch.arena, ASYNC_Task *, num_tasks); + + // rjf: kickoff tasks + for(U64 task_idx = 0; task_idx < num_tasks; task_idx += 1) + { + RDIM_JoinBakeStringMapSlotsIn *in = push_array(scratch.arena, RDIM_JoinBakeStringMapSlotsIn, 1); + in->top = &bake_string_map_topology; + in->src_maps = bake_string_maps__in_progress; + in->src_maps_count = async_thread_count(); + in->dst_map = unsorted_bake_string_map; + in->slot_idx_range = r1u64(task_idx*slots_per_task, task_idx*slots_per_task + slots_per_task); + in->slot_idx_range.max = Min(in->slot_idx_range.max, in->top->slots_count); + tasks[task_idx] = async_task_launch(scratch.arena, rdim_bake_string_map_join_work, .input = in); + } + + // rjf: join tasks + for(U64 task_idx = 0; task_idx < num_tasks; task_idx += 1) + { + async_task_join(tasks[task_idx]); + } + + // rjf: insert small top-level stuff + rdim_bake_string_map_loose_push_top_level_info(state->work_thread_arenas[0], &bake_string_map_topology, unsorted_bake_string_map, &in_params->top_level_info); + rdim_bake_string_map_loose_push_binary_sections(state->work_thread_arenas[0], &bake_string_map_topology, unsorted_bake_string_map, &in_params->binary_sections); + rdim_bake_string_map_loose_push_path_tree(state->work_thread_arenas[0], &bake_string_map_topology, unsorted_bake_string_map, path_tree); + } + + ////////////////////////////// + //- rjf: kick off string map sorting tasks + // + ASYNC_TaskList sort_bake_string_map_tasks = {0}; + RDIM_BakeStringMapLoose *sorted_bake_string_map__in_progress = rdim_bake_string_map_loose_make(state->work_thread_arenas[0], &bake_string_map_topology); + { + U64 slots_per_task = 4096; + U64 num_tasks = (bake_string_map_topology.slots_count+slots_per_task-1)/slots_per_task; + for(U64 task_idx = 0; task_idx < num_tasks; task_idx += 1) + { + RDIM_SortBakeStringMapSlotsIn *in = push_array(scratch.arena, RDIM_SortBakeStringMapSlotsIn, 1); + { + in->top = &bake_string_map_topology; + in->src_map = unsorted_bake_string_map; + in->dst_map = sorted_bake_string_map__in_progress; + in->slot_idx = task_idx*slots_per_task; + in->slot_count = slots_per_task; + if(in->slot_idx+in->slot_count > bake_string_map_topology.slots_count) + { + in->slot_count = bake_string_map_topology.slots_count - in->slot_idx; + } + } + async_task_list_push(scratch.arena, &sort_bake_string_map_tasks, async_task_launch(scratch.arena, rdim_bake_string_map_sort_work, .input = in)); + } + } + + ////////////////////////////// + //- rjf: join string map sorting tasks + // + ProfScope("join string map sorting tasks") + { + for(ASYNC_TaskNode *n = sort_bake_string_map_tasks.first; n != 0; n = n->next) + { + async_task_join(n->v); + } + } + RDIM_BakeStringMapLoose *sorted_bake_string_map = sorted_bake_string_map__in_progress; + + ////////////////////////////// + //- rjf: build finalized string map + // + ProfBegin("build finalized string map base indices"); + RDIM_BakeStringMapBaseIndices bake_string_map_base_idxes = rdim_bake_string_map_base_indices_from_map_loose(state->work_thread_arenas[0], &bake_string_map_topology, sorted_bake_string_map); + ProfEnd(); + ProfBegin("build finalized string map"); + RDIM_BakeStringMapTight bake_strings = rdim_bake_string_map_tight_from_loose(state->work_thread_arenas[0], &bake_string_map_topology, &bake_string_map_base_idxes, sorted_bake_string_map); + ProfEnd(); + + ////////////////////////////// + //- rjf: kick off pass 2 tasks + // + RDIM_BakeUnitsIn bake_units_top_level_in = {&bake_strings, path_tree, &in_params->units}; + ASYNC_Task *bake_units_task = async_task_launch(scratch.arena, rdim_bake_units_work, .input = &bake_units_top_level_in); + RDIM_BakeUnitVMapIn bake_unit_vmap_in = {&in_params->units}; + ASYNC_Task *bake_unit_vmap_task = async_task_launch(scratch.arena, rdim_bake_unit_vmap_work, .input = &bake_unit_vmap_in); + RDIM_BakeSrcFilesIn bake_src_files_in = {&bake_strings, path_tree, &in_params->src_files}; + ASYNC_Task *bake_src_files_task = async_task_launch(scratch.arena, rdim_bake_src_files_work, .input = &bake_src_files_in); + RDIM_BakeUDTsIn bake_udts_in = {&bake_strings, &in_params->udts}; + ASYNC_Task *bake_udts_task = async_task_launch(scratch.arena, rdim_bake_udts_work, .input = &bake_udts_in); + RDIM_BakeGlobalVariablesIn bake_global_variables_in = {&bake_strings, &in_params->global_variables}; + ASYNC_Task *bake_global_variables_task = async_task_launch(scratch.arena, rdim_bake_global_variables_work, .input = &bake_global_variables_in); + RDIM_BakeGlobalVMapIn bake_global_vmap_in = {&in_params->global_variables}; + ASYNC_Task *bake_global_vmap_task = async_task_launch(scratch.arena, rdim_bake_global_vmap_work, .input = &bake_global_vmap_in); + RDIM_BakeThreadVariablesIn bake_thread_variables_in = {&bake_strings, &in_params->thread_variables}; + ASYNC_Task *bake_thread_variables_task = async_task_launch(scratch.arena, rdim_bake_thread_variables_work, .input = &bake_thread_variables_in); + RDIM_BakeProceduresIn bake_procedures_in = {&bake_strings, &in_params->procedures}; + ASYNC_Task *bake_procedures_task = async_task_launch(scratch.arena, rdim_bake_procedures_work, .input = &bake_procedures_in); + RDIM_BakeScopesIn bake_scopes_in = {&bake_strings, &in_params->scopes}; + ASYNC_Task *bake_scopes_task = async_task_launch(scratch.arena, rdim_bake_scopes_work, .input = &bake_scopes_in); + RDIM_BakeScopeVMapIn bake_scope_vmap_in = {&in_params->scopes}; + ASYNC_Task *bake_scope_vmap_task = async_task_launch(scratch.arena, rdim_bake_scope_vmap_work, .input = &bake_scope_vmap_in); + RDIM_BakeInlineSitesIn bake_inline_sites_in = {&bake_strings, &in_params->inline_sites}; + ASYNC_Task *bake_inline_sites_task = async_task_launch(scratch.arena, rdim_bake_inline_sites_work, .input = &bake_inline_sites_in); + RDIM_BakeFilePathsIn bake_file_paths_in = {&bake_strings, path_tree}; + ASYNC_Task *bake_file_paths_task = async_task_launch(scratch.arena, rdim_bake_file_paths_work, .input = &bake_file_paths_in); + RDIM_BakeStringsIn bake_strings_in = {&bake_strings}; + ASYNC_Task *bake_strings_task = async_task_launch(scratch.arena, rdim_bake_strings_work, .input = &bake_strings_in); + + ////////////////////////////// + //- rjf: join name map building tasks + // + RDIM_BakeNameMap *name_maps[RDI_NameMapKind_COUNT] = {0}; + ProfScope("join name map building tasks") + { + for(RDI_NameMapKind k = (RDI_NameMapKind)(RDI_NameMapKind_NULL+1); + k < RDI_NameMapKind_COUNT; + k = (RDI_NameMapKind)(k+1)) + { + name_maps[k] = async_task_join_struct(build_bake_name_map_task[k], RDIM_BakeNameMap); + } + } + + ////////////////////////////// + //- rjf: build interned idx run map + // + RDIM_BakeIdxRunMap *idx_runs = 0; + ProfScope("build interned idx run map") + { + idx_runs = rdim_bake_idx_run_map_from_params(state->work_thread_arenas[0], name_maps, in_params); + } + + ////////////////////////////// + //- rjf: do small top-level bakes + // + ProfScope("top level info") out.top_level_info = rdim_bake_top_level_info(state->work_thread_arenas[0], &bake_strings, &in_params->top_level_info); + ProfScope("binary sections") out.binary_sections = rdim_bake_binary_sections(state->work_thread_arenas[0], &bake_strings, &in_params->binary_sections); + ProfScope("top level name maps section") out.top_level_name_maps = rdim_bake_name_maps_top_level(state->work_thread_arenas[0], &bake_strings, idx_runs, name_maps); + + ////////////////////////////// + //- rjf: kick off pass 3 tasks + // + RDIM_BakeTypeNodesIn bake_type_nodes_in = {&bake_strings, idx_runs, &in_params->types}; + ASYNC_Task *bake_type_nodes_task = async_task_launch(scratch.arena, rdim_bake_type_nodes_work, .input = &bake_type_nodes_in); + ASYNC_Task *bake_name_maps_tasks[RDI_NameMapKind_COUNT] = {0}; + { + for EachNonZeroEnumVal(RDI_NameMapKind, k) + { + if(name_maps[k] == 0 || name_maps[k]->name_count == 0) + { + continue; + } + RDIM_BakeNameMapIn *in = push_array(scratch.arena, RDIM_BakeNameMapIn, 1); + in->strings = &bake_strings; + in->idx_runs = idx_runs; + in->map = name_maps[k]; + in->kind = k; + bake_name_maps_tasks[k] = async_task_launch(scratch.arena, rdim_bake_name_map_work, .input = in); + } + } + RDIM_BakeIdxRunsIn bake_idx_runs_in = {idx_runs}; + ASYNC_Task *bake_idx_runs_task = async_task_launch(scratch.arena, rdim_bake_idx_runs_work, .input = &bake_idx_runs_in); + + ////////////////////////////// + //- rjf: join remaining completed bakes + // + ProfScope("top-level units info") out.units = *async_task_join_struct(bake_units_task, RDIM_UnitBakeResult); + ProfScope("unit vmap") out.unit_vmap = *async_task_join_struct(bake_unit_vmap_task, RDIM_UnitVMapBakeResult); + ProfScope("source files") out.src_files = *async_task_join_struct(bake_src_files_task, RDIM_SrcFileBakeResult); + ProfScope("UDTs") out.udts = *async_task_join_struct(bake_udts_task, RDIM_UDTBakeResult); + ProfScope("global variables") out.global_variables = *async_task_join_struct(bake_global_variables_task, RDIM_GlobalVariableBakeResult); + ProfScope("global vmap") out.global_vmap = *async_task_join_struct(bake_global_vmap_task, RDIM_GlobalVMapBakeResult); + ProfScope("thread variables") out.thread_variables = *async_task_join_struct(bake_thread_variables_task, RDIM_ThreadVariableBakeResult); + ProfScope("procedures") out.procedures = *async_task_join_struct(bake_procedures_task, RDIM_ProcedureBakeResult); + ProfScope("scopes") out.scopes = *async_task_join_struct(bake_scopes_task, RDIM_ScopeBakeResult); + ProfScope("scope vmap") out.scope_vmap = *async_task_join_struct(bake_scope_vmap_task, RDIM_ScopeVMapBakeResult); + ProfScope("inline sites") out.inline_sites = *async_task_join_struct(bake_inline_sites_task, RDIM_InlineSiteBakeResult); + ProfScope("file paths") out.file_paths = *async_task_join_struct(bake_file_paths_task, RDIM_FilePathBakeResult); + ProfScope("strings") out.strings = *async_task_join_struct(bake_strings_task, RDIM_StringBakeResult); + ProfScope("type nodes") out.type_nodes = *async_task_join_struct(bake_type_nodes_task, RDIM_TypeNodeBakeResult); + ProfScope("idx runs") out.idx_runs = *async_task_join_struct(bake_idx_runs_task, RDIM_IndexRunBakeResult); + ProfScope("line tables") out.line_tables = *async_task_join_struct(bake_line_tables_task, RDIM_LineTableBakeResult); + + ////////////////////////////// + //- rjf: join individual name map bakes + // + RDIM_NameMapBakeResult name_map_bakes[RDI_NameMapKind_COUNT] = {0}; + ProfScope("name maps") + { + for EachNonZeroEnumVal(RDI_NameMapKind, k) + { + RDIM_NameMapBakeResult *bake = async_task_join_struct(bake_name_maps_tasks[k], RDIM_NameMapBakeResult); + if(bake != 0) + { + name_map_bakes[k] = *bake; + } + } + } + + ////////////////////////////// + //- rjf: join all individual name map bakes + // + ProfScope("join all name map bakes into final name map bake") + { + out.name_maps = rdim_name_map_bake_results_combine(state->work_thread_arenas[0], name_map_bakes, ArrayCount(name_map_bakes)); + } + + rdim_help_state = 0; + + scratch_end(scratch); + return out; +} + +internal RDIM_SerializedSectionBundle +rdim_compress(Arena *arena, RDIM_SerializedSectionBundle *in) +{ + RDIM_SerializedSectionBundle out = {0}; + + //- rjf: set up compression context + rr_lzb_simple_context ctx = {0}; + ctx.m_tableSizeBits = 14; + ctx.m_hashTable = push_array(arena, U16, 1<sections[k]; + RDIM_SerializedSection *dst = &out.sections[k]; + MemoryCopyStruct(dst, src); + + // rjf: determine if this section should be compressed + B32 should_compress = 1; + + // rjf: compress if needed + if(should_compress) + { + MemoryZero(ctx.m_hashTable, sizeof(U16)*(1<data = push_array_no_zero(arena, U8, src->encoded_size); + dst->encoded_size = rr_lzb_simple_encode_veryfast(&ctx, src->data, src->encoded_size, dst->data); + dst->unpacked_size = src->encoded_size; + dst->encoding = RDI_SectionEncoding_LZB; + } + } + + return out; +} diff --git a/src/rdi_make/rdi_make_help.h b/src/rdi_make/rdi_make_help.h new file mode 100644 index 00000000..159e3a3e --- /dev/null +++ b/src/rdi_make/rdi_make_help.h @@ -0,0 +1,330 @@ +// Copyright (c) 2024 Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +#ifndef RDIM_MAKE_HELP +#define RDIM_MAKE_HELP + +//- rjf: line table baking task types + +typedef struct RDIM_BakeLineTablesIn RDIM_BakeLineTablesIn; +struct RDIM_BakeLineTablesIn +{ + RDIM_LineTableChunkList *line_tables; +}; + +//- rjf: string map baking task types + +typedef struct RDIM_BakeSrcFilesStringsIn RDIM_BakeSrcFilesStringsIn; +struct RDIM_BakeSrcFilesStringsIn +{ + RDIM_BakeStringMapTopology *top; + RDIM_BakeStringMapLoose **maps; + RDIM_SrcFileChunkList *list; +}; + +typedef struct RDIM_BakeUnitsStringsIn RDIM_BakeUnitsStringsIn; +struct RDIM_BakeUnitsStringsIn +{ + RDIM_BakeStringMapTopology *top; + RDIM_BakeStringMapLoose **maps; + RDIM_UnitChunkList *list; +}; + +typedef struct RDIM_BakeUDTsStringsInNode RDIM_BakeUDTsStringsInNode; +struct RDIM_BakeUDTsStringsInNode +{ + RDIM_BakeUDTsStringsInNode *next; + RDIM_UDT *v; + RDI_U64 count; +}; + +typedef struct RDIM_BakeTypesStringsInNode RDIM_BakeTypesStringsInNode; +struct RDIM_BakeTypesStringsInNode +{ + RDIM_BakeTypesStringsInNode *next; + RDIM_Type *v; + RDI_U64 count; +}; + +typedef struct RDIM_BakeTypesStringsIn RDIM_BakeTypesStringsIn; +struct RDIM_BakeTypesStringsIn +{ + RDIM_BakeStringMapTopology *top; + RDIM_BakeStringMapLoose **maps; + RDIM_BakeTypesStringsInNode *first; + RDIM_BakeTypesStringsInNode *last; +}; + +typedef struct RDIM_BakeUDTsStringsIn RDIM_BakeUDTsStringsIn; +struct RDIM_BakeUDTsStringsIn +{ + RDIM_BakeStringMapTopology *top; + RDIM_BakeStringMapLoose **maps; + RDIM_BakeUDTsStringsInNode *first; + RDIM_BakeUDTsStringsInNode *last; +}; + +typedef struct RDIM_BakeSymbolsStringsInNode RDIM_BakeSymbolsStringsInNode; +struct RDIM_BakeSymbolsStringsInNode +{ + RDIM_BakeSymbolsStringsInNode *next; + RDIM_Symbol *v; + RDI_U64 count; +}; + +typedef struct RDIM_BakeSymbolsStringsIn RDIM_BakeSymbolsStringsIn; +struct RDIM_BakeSymbolsStringsIn +{ + RDIM_BakeStringMapTopology *top; + RDIM_BakeStringMapLoose **maps; + RDIM_BakeSymbolsStringsInNode *first; + RDIM_BakeSymbolsStringsInNode *last; +}; + +typedef struct RDIM_BakeInlineSiteStringsInNode RDIM_BakeInlineSiteStringsInNode; +struct RDIM_BakeInlineSiteStringsInNode +{ + RDIM_BakeInlineSiteStringsInNode *next; + RDIM_InlineSite *v; + RDI_U64 count; +}; + +typedef struct RDIM_BakeInlineSiteStringsIn RDIM_BakeInlineSiteStringsIn; +struct RDIM_BakeInlineSiteStringsIn +{ + RDIM_BakeStringMapTopology *top; + RDIM_BakeStringMapLoose **maps; + RDIM_BakeInlineSiteStringsInNode *first; + RDIM_BakeInlineSiteStringsInNode *last; +}; + +typedef struct RDIM_BakeScopesStringsInNode RDIM_BakeScopesStringsInNode; +struct RDIM_BakeScopesStringsInNode +{ + RDIM_BakeScopesStringsInNode *next; + RDIM_Scope *v; + RDI_U64 count; +}; + +typedef struct RDIM_BakeScopesStringsIn RDIM_BakeScopesStringsIn; +struct RDIM_BakeScopesStringsIn +{ + RDIM_BakeStringMapTopology *top; + RDIM_BakeStringMapLoose **maps; + RDIM_BakeScopesStringsInNode *first; + RDIM_BakeScopesStringsInNode *last; +}; + +//- rjf: OLD string map baking types + +typedef struct RDIM_BuildBakeStringMapIn RDIM_BuildBakeStringMapIn; +struct RDIM_BuildBakeStringMapIn +{ + RDIM_BakePathTree *path_tree; + RDIM_BakeParams *params; +}; + +typedef struct RDIM_BuildBakeNameMapIn RDIM_BuildBakeNameMapIn; +struct RDIM_BuildBakeNameMapIn +{ + RDI_NameMapKind k; + RDIM_BakeParams *params; +}; + +//- rjf: string map joining task types + +typedef struct RDIM_JoinBakeStringMapSlotsIn RDIM_JoinBakeStringMapSlotsIn; +struct RDIM_JoinBakeStringMapSlotsIn +{ + RDIM_BakeStringMapTopology *top; + RDIM_BakeStringMapLoose **src_maps; + U64 src_maps_count; + RDIM_BakeStringMapLoose *dst_map; + Rng1U64 slot_idx_range; +}; + +//- rjf: string map sorting task types + +typedef struct RDIM_SortBakeStringMapSlotsIn RDIM_SortBakeStringMapSlotsIn; +struct RDIM_SortBakeStringMapSlotsIn +{ + RDIM_BakeStringMapTopology *top; + RDIM_BakeStringMapLoose *src_map; + RDIM_BakeStringMapLoose *dst_map; + U64 slot_idx; + U64 slot_count; +}; + +//- rjf: debug info baking task types + +typedef struct RDIM_BakeUnitsIn RDIM_BakeUnitsIn; +struct RDIM_BakeUnitsIn +{ + RDIM_BakeStringMapTight *strings; + RDIM_BakePathTree *path_tree; + RDIM_UnitChunkList *units; +}; + +typedef struct RDIM_BakeUnitVMapIn RDIM_BakeUnitVMapIn; +struct RDIM_BakeUnitVMapIn +{ + RDIM_UnitChunkList *units; +}; + +typedef struct RDIM_BakeSrcFilesIn RDIM_BakeSrcFilesIn; +struct RDIM_BakeSrcFilesIn +{ + RDIM_BakeStringMapTight *strings; + RDIM_BakePathTree *path_tree; + RDIM_SrcFileChunkList *src_files; +}; + +typedef struct RDIM_BakeUDTsIn RDIM_BakeUDTsIn; +struct RDIM_BakeUDTsIn +{ + RDIM_BakeStringMapTight *strings; + RDIM_UDTChunkList *udts; +}; + +typedef struct RDIM_BakeGlobalVariablesIn RDIM_BakeGlobalVariablesIn; +struct RDIM_BakeGlobalVariablesIn +{ + RDIM_BakeStringMapTight *strings; + RDIM_SymbolChunkList *global_variables; +}; + +typedef struct RDIM_BakeGlobalVMapIn RDIM_BakeGlobalVMapIn; +struct RDIM_BakeGlobalVMapIn +{ + RDIM_SymbolChunkList *global_variables; +}; + +typedef struct RDIM_BakeThreadVariablesIn RDIM_BakeThreadVariablesIn; +struct RDIM_BakeThreadVariablesIn +{ + RDIM_BakeStringMapTight *strings; + RDIM_SymbolChunkList *thread_variables; +}; + +typedef struct RDIM_BakeProceduresIn RDIM_BakeProceduresIn; +struct RDIM_BakeProceduresIn +{ + RDIM_BakeStringMapTight *strings; + RDIM_SymbolChunkList *procedures; +}; + +typedef struct RDIM_BakeScopesIn RDIM_BakeScopesIn; +struct RDIM_BakeScopesIn +{ + RDIM_BakeStringMapTight *strings; + RDIM_ScopeChunkList *scopes; +}; + +typedef struct RDIM_BakeScopeVMapIn RDIM_BakeScopeVMapIn; +struct RDIM_BakeScopeVMapIn +{ + RDIM_ScopeChunkList *scopes; +}; + +typedef struct RDIM_BakeInlineSitesIn RDIM_BakeInlineSitesIn; +struct RDIM_BakeInlineSitesIn +{ + RDIM_BakeStringMapTight *strings; + RDIM_InlineSiteChunkList *inline_sites; +}; + +typedef struct RDIM_BakeFilePathsIn RDIM_BakeFilePathsIn; +struct RDIM_BakeFilePathsIn +{ + RDIM_BakeStringMapTight *strings; + RDIM_BakePathTree *path_tree; +}; + +typedef struct RDIM_BakeStringsIn RDIM_BakeStringsIn; +struct RDIM_BakeStringsIn +{ + RDIM_BakeStringMapTight *strings; +}; + +typedef struct RDIM_BakeTypeNodesIn RDIM_BakeTypeNodesIn; +struct RDIM_BakeTypeNodesIn +{ + RDIM_BakeStringMapTight *strings; + RDIM_BakeIdxRunMap *idx_runs; + RDIM_TypeChunkList *types; +}; + +typedef struct RDIM_BakeNameMapIn RDIM_BakeNameMapIn; +struct RDIM_BakeNameMapIn +{ + RDIM_BakeStringMapTight *strings; + RDIM_BakeIdxRunMap *idx_runs; + RDIM_BakeNameMap *map; + RDI_NameMapKind kind; +}; + +typedef struct RDIM_BakeIdxRunsIn RDIM_BakeIdxRunsIn; +struct RDIM_BakeIdxRunsIn +{ + RDIM_BakeIdxRunMap *idx_runs; +}; + +//////////////////////////////// +//~ rjf: Baking Stage Tasks + +//- rjf: unsorted bake string map building +ASYNC_WORK_DEF(p2r_bake_src_files_strings_work); +ASYNC_WORK_DEF(p2r_bake_units_strings_work); +ASYNC_WORK_DEF(p2r_bake_types_strings_work); +ASYNC_WORK_DEF(p2r_bake_udts_strings_work); +ASYNC_WORK_DEF(p2r_bake_symbols_strings_work); +ASYNC_WORK_DEF(p2r_bake_scopes_strings_work); +ASYNC_WORK_DEF(p2r_bake_line_tables_work); + +//- rjf: bake string map joining +ASYNC_WORK_DEF(p2r_bake_string_map_join_work); + +//- rjf: bake string map sorting +ASYNC_WORK_DEF(p2r_bake_string_map_sort_work); + +//- rjf: pass 1: interner/deduper map builds +ASYNC_WORK_DEF(p2r_build_bake_name_map_work); + +//- rjf: pass 2: string-map-dependent debug info stream builds +ASYNC_WORK_DEF(p2r_bake_units_work); +ASYNC_WORK_DEF(p2r_bake_unit_vmap_work); +ASYNC_WORK_DEF(p2r_bake_src_files_work); +ASYNC_WORK_DEF(p2r_bake_udts_work); +ASYNC_WORK_DEF(p2r_bake_global_variables_work); +ASYNC_WORK_DEF(p2r_bake_global_vmap_work); +ASYNC_WORK_DEF(p2r_bake_thread_variables_work); +ASYNC_WORK_DEF(p2r_bake_procedures_work); +ASYNC_WORK_DEF(p2r_bake_scopes_work); +ASYNC_WORK_DEF(p2r_bake_scope_vmap_work); +ASYNC_WORK_DEF(p2r_bake_file_paths_work); +ASYNC_WORK_DEF(p2r_bake_strings_work); + +//- rjf: pass 3: idx-run-map-dependent debug info stream builds +ASYNC_WORK_DEF(p2r_bake_type_nodes_work); +ASYNC_WORK_DEF(p2r_bake_name_map_work); +ASYNC_WORK_DEF(p2r_bake_idx_runs_work); + +typedef struct RDIM_HelpState RDIM_HelpState; +struct RDIM_HelpState +{ + Arena *arena; + U64 work_thread_arenas_count; + Arena **work_thread_arenas; +}; + +//////////////////////////////// + +global RDIM_HelpState *rdim_help_state = 0; + +//////////////////////////////// + +internal RDIM_HelpState * rdim_help_init(void); +internal RDIM_BakeResults rdim_bake(RDIM_HelpState *state, RDIM_BakeParams *in); +internal RDIM_SerializedSectionBundle rdim_compress(Arena *arena, RDIM_SerializedSectionBundle *in); + +#endif // RDIM_MAKE_HELP