From 2ff0abb91ea36b13b57d97ff496b4079c557c278 Mon Sep 17 00:00:00 2001 From: Ryan Fleury Date: Fri, 7 Jun 2024 11:03:00 -0700 Subject: [PATCH] rdi_make, rdi_from_pdb: complete next pass over new serialization/compression interface, including for new top-level-only name map data sections; eliminate determinism issues from string map build (task dispatch was busted) --- project.4coder | 2 +- src/lib_rdi_make/rdi_make.c | 147 ++++++ src/lib_rdi_make/rdi_make.h | 59 ++- src/raddbg/raddbg_main.c | 17 +- .../rdi_breakpad_from_pdb_main.c | 26 +- src/rdi_dump/rdi_dump_main.c | 5 +- src/rdi_from_pdb/rdi_from_pdb.c | 489 ++++++++---------- src/rdi_from_pdb/rdi_from_pdb.h | 34 +- src/rdi_from_pdb/rdi_from_pdb_main.c | 17 +- 9 files changed, 483 insertions(+), 313 deletions(-) diff --git a/project.4coder b/project.4coder index 5c0032f8..28e00d14 100644 --- a/project.4coder +++ b/project.4coder @@ -56,7 +56,7 @@ commands = }, .rjf_f2 = { - .win = "build rdi_from_pdb rdi_dump && pushd build && rdi_from_pdb --pdb:mule_main.pdb --out:mule_main.rdi && rdi_dump mule_main.rdi > mule_main.dump && popd", + .win = "build rdi_from_pdb rdi_dump && pushd build && rdi_from_pdb --pdb:mule_main.pdb --out:mule_main_1.rdi && rdi_dump --only:strings mule_main_1.rdi > mule_main_1.dump && rdi_from_pdb --pdb:mule_main.pdb --out:mule_main_2.rdi && rdi_dump --only:strings mule_main_2.rdi > mule_main_2.dump && diff mule_main_1.dump mule_main_2.dump && popd", .linux = "", .out = "*compilation*", .footer_panel = true, diff --git a/src/lib_rdi_make/rdi_make.c b/src/lib_rdi_make/rdi_make.c index 17130e91..4e507fd2 100644 --- a/src/lib_rdi_make/rdi_make.c +++ b/src/lib_rdi_make/rdi_make.c @@ -2292,6 +2292,44 @@ rdim_bake_name_map(RDIM_Arena *arena, RDIM_BakeStringMapTight *strings, RDIM_Bak return result; } +//- rjf: partial bakes -> final bake functions + +RDI_PROC RDIM_NameMapBakeResult +rdim_name_map_bake_results_combine(RDIM_Arena *arena, RDIM_NameMapBakeResult *results, RDI_U64 results_count) +{ + RDIM_NameMapBakeResult result = {0}; + { + //- rjf: count needed # of buckets/nodes + RDI_U64 all_buckets_count = 0; + RDI_U64 all_nodes_count = 0; + for(RDI_U64 idx = 0; idx < results_count; idx += 1) + { + all_buckets_count += results[idx].buckets_count; + all_nodes_count += results[idx].nodes_count; + } + + //- rjf: allocate outputs + result.buckets_count = all_buckets_count; + result.buckets = rdim_push_array_no_zero(arena, RDI_NameMapBucket, result.buckets_count); + result.nodes_count = all_nodes_count; + result.nodes = rdim_push_array_no_zero(arena, RDI_NameMapNode, result.nodes_count); + + //- rjf: fill outputs + { + RDI_U64 buckets_off = 0; + RDI_U64 nodes_off = 0; + for(RDI_U64 idx = 0; idx < results_count; idx += 1) + { + rdim_memcpy(result.buckets + buckets_off, results[idx].buckets, sizeof(result.buckets[0])*results[idx].buckets_count); + rdim_memcpy(result.nodes + nodes_off, results[idx].nodes, sizeof(result.nodes[0])*results[idx].nodes_count); + buckets_off += results[idx].buckets_count; + nodes_off += results[idx].nodes_count; + } + } + } + return result; +} + //- rjf: independent (top-level, global) baking functions RDI_PROC RDIM_TopLevelInfoBakeResult @@ -3515,9 +3553,114 @@ rdim_bake_index_runs(RDIM_Arena *arena, RDIM_BakeIdxRunMap *idx_runs) return result; } +//////////////////////////////// +//~ rjf: [Serializing] Bake Results -> String Blobs + +RDI_PROC RDIM_SerializedSection +rdim_serialized_section_make_unpacked(void *data, RDI_U64 size) +{ + RDIM_SerializedSection s; + rdim_memzero_struct(&s); + s.data = data; + s.encoded_size = s.unpacked_size = size; + s.encoding = RDI_SectionEncoding_Unpacked; + return s; +} + +RDI_PROC RDIM_SerializedSectionBundle +rdim_serialized_section_bundle_from_bake_results(RDIM_BakeResults *results) +{ + RDIM_SerializedSectionBundle bundle; + rdim_memzero_struct(&bundle); + bundle.sections[RDI_SectionKind_TopLevelInfo] = rdim_serialized_section_make_unpacked_struct(results->top_level_info.top_level_info); + bundle.sections[RDI_SectionKind_StringData] = rdim_serialized_section_make_unpacked_array(results->strings.string_data, results->strings.string_data_size); + bundle.sections[RDI_SectionKind_StringTable] = rdim_serialized_section_make_unpacked_array(results->strings.string_offs, results->strings.string_offs_count); + bundle.sections[RDI_SectionKind_IndexRuns] = rdim_serialized_section_make_unpacked_array(results->idx_runs.idx_runs, results->idx_runs.idx_count); + bundle.sections[RDI_SectionKind_BinarySections] = rdim_serialized_section_make_unpacked_array(results->binary_sections.binary_sections, results->binary_sections.binary_sections_count); + bundle.sections[RDI_SectionKind_FilePathNodes] = rdim_serialized_section_make_unpacked_array(results->file_paths.nodes, results->file_paths.nodes_count); + bundle.sections[RDI_SectionKind_SourceFiles] = rdim_serialized_section_make_unpacked_array(results->src_files.source_files, results->src_files.source_files_count); + bundle.sections[RDI_SectionKind_LineTables] = rdim_serialized_section_make_unpacked_array(results->line_tables.line_tables, results->line_tables.line_tables_count); + bundle.sections[RDI_SectionKind_LineInfoVOffs] = rdim_serialized_section_make_unpacked_array(results->line_tables.line_table_voffs, results->line_tables.line_table_voffs_count); + bundle.sections[RDI_SectionKind_LineInfoLines] = rdim_serialized_section_make_unpacked_array(results->line_tables.line_table_lines, results->line_tables.line_table_lines_count); + bundle.sections[RDI_SectionKind_LineInfoColumns] = rdim_serialized_section_make_unpacked_array(results->line_tables.line_table_columns, results->line_tables.line_table_columns_count); + bundle.sections[RDI_SectionKind_SourceLineMaps] = rdim_serialized_section_make_unpacked_array(results->src_files.source_line_maps, results->src_files.source_line_maps_count); + bundle.sections[RDI_SectionKind_SourceLineMapNumbers] = rdim_serialized_section_make_unpacked_array(results->src_files.source_line_map_nums, results->src_files.source_line_map_nums_count); + bundle.sections[RDI_SectionKind_SourceLineMapRanges] = rdim_serialized_section_make_unpacked_array(results->src_files.source_line_map_rngs, results->src_files.source_line_map_rngs_count); + bundle.sections[RDI_SectionKind_SourceLineMapVOffs] = rdim_serialized_section_make_unpacked_array(results->src_files.source_line_map_voffs, results->src_files.source_line_map_voffs_count); + bundle.sections[RDI_SectionKind_Units] = rdim_serialized_section_make_unpacked_array(results->units.units, results->units.units_count); + bundle.sections[RDI_SectionKind_UnitVMap] = rdim_serialized_section_make_unpacked_array(results->unit_vmap.vmap.vmap, results->unit_vmap.vmap.count+1); + bundle.sections[RDI_SectionKind_TypeNodes] = rdim_serialized_section_make_unpacked_array(results->type_nodes.type_nodes, results->type_nodes.type_nodes_count); + bundle.sections[RDI_SectionKind_UDTs] = rdim_serialized_section_make_unpacked_array(results->udts.udts, results->udts.udts_count); + bundle.sections[RDI_SectionKind_Members] = rdim_serialized_section_make_unpacked_array(results->udts.members, results->udts.members_count); + bundle.sections[RDI_SectionKind_EnumMembers] = rdim_serialized_section_make_unpacked_array(results->udts.enum_members, results->udts.enum_members_count); + bundle.sections[RDI_SectionKind_GlobalVariables] = rdim_serialized_section_make_unpacked_array(results->global_variables.global_variables, results->global_variables.global_variables_count); + bundle.sections[RDI_SectionKind_GlobalVMap] = rdim_serialized_section_make_unpacked_array(results->global_vmap.vmap.vmap, results->global_vmap.vmap.count+1); + bundle.sections[RDI_SectionKind_ThreadVariables] = rdim_serialized_section_make_unpacked_array(results->thread_variables.thread_variables, results->thread_variables.thread_variables_count); + bundle.sections[RDI_SectionKind_Procedures] = rdim_serialized_section_make_unpacked_array(results->procedures.procedures, results->procedures.procedures_count); + bundle.sections[RDI_SectionKind_Scopes] = rdim_serialized_section_make_unpacked_array(results->scopes.scopes, results->scopes.scopes_count); + bundle.sections[RDI_SectionKind_ScopeVOffData] = rdim_serialized_section_make_unpacked_array(results->scopes.scope_voffs, results->scopes.scope_voffs_count); + bundle.sections[RDI_SectionKind_ScopeVMap] = rdim_serialized_section_make_unpacked_array(results->scope_vmap.vmap.vmap, results->scope_vmap.vmap.count+1); + bundle.sections[RDI_SectionKind_InlineSites] = rdim_serialized_section_make_unpacked(0, 0); + bundle.sections[RDI_SectionKind_Locals] = rdim_serialized_section_make_unpacked_array(results->scopes.locals, results->scopes.locals_count); + bundle.sections[RDI_SectionKind_LocationBlocks] = rdim_serialized_section_make_unpacked_array(results->scopes.location_blocks, results->scopes.location_blocks_count); + bundle.sections[RDI_SectionKind_LocationData] = rdim_serialized_section_make_unpacked_array(results->scopes.location_data, results->scopes.location_data_size); + bundle.sections[RDI_SectionKind_NameMaps] = rdim_serialized_section_make_unpacked_array(results->top_level_name_maps.name_maps, results->top_level_name_maps.name_maps_count); + bundle.sections[RDI_SectionKind_NameMapBuckets] = rdim_serialized_section_make_unpacked_array(results->name_maps.buckets, results->name_maps.buckets_count); + bundle.sections[RDI_SectionKind_NameMapNodes] = rdim_serialized_section_make_unpacked_array(results->name_maps.nodes, results->name_maps.nodes_count); + return bundle; +} + +RDI_PROC RDIM_String8List +rdim_file_blobs_from_section_bundle(RDIM_Arena *arena, RDIM_SerializedSectionBundle *bundle) +{ + RDIM_String8List strings; + rdim_memzero_struct(&strings); + { + RDIM_Temp scratch = rdim_scratch_begin(&arena, 1); + + // rjf: push empty header & data section table + RDI_Header *rdi_header = rdim_push_array(arena, RDI_Header, 1); + RDI_Section *rdi_sections = rdim_push_array(arena, RDI_Section, RDI_SectionKind_COUNT); + rdim_str8_list_push(arena, &strings, rdim_str8_struct(rdi_header)); + rdim_str8_list_push_align(arena, &strings, 8); + U32 data_section_off = (U32)strings.total_size; + rdim_str8_list_push(arena, &strings, rdim_str8((RDI_U8 *)rdi_sections, sizeof(RDI_Section)*RDI_SectionKind_COUNT)); + + // rjf: fill baked header + { + rdi_header->magic = RDI_MAGIC_CONSTANT; + rdi_header->encoding_version = RDI_ENCODING_VERSION; + rdi_header->data_section_off = data_section_off; + rdi_header->data_section_count = RDI_SectionKind_COUNT; + } + + // rjf: fill baked data section table + for(RDI_SectionKind k = RDI_SectionKind_NULL; k < RDI_SectionKind_COUNT; k += 1) + { + RDI_Section *dst = rdi_sections+k; + U64 data_section_off = 0; + if(bundle->sections[k].encoded_size != 0) + { + rdim_str8_list_push_align(arena, &strings, 8); + data_section_off = strings.total_size; + rdim_str8_list_push(arena, &strings, rdim_str8((RDI_U8 *)bundle->sections[k].data, bundle->sections[k].encoded_size)); + } + dst->encoding = bundle->sections[k].encoding; + dst->off = data_section_off; + dst->encoded_size = bundle->sections[k].encoded_size; + dst->unpacked_size = bundle->sections[k].unpacked_size; + } + + rdim_scratch_end(scratch); + } + return strings; +} + //////////////////////////////// //~ rjf: [Baking] Build Artifacts -> Data Section Lists +#if 0 + //- rjf: top-level info RDI_PROC RDIM_BakeSectionList @@ -4876,9 +5019,12 @@ rdim_bake_idx_run_section_list_from_idx_run_map(RDIM_Arena *arena, RDIM_BakeIdxR return sections; } +#endif + //////////////////////////////// //~ rjf: [Serializing] Baked Data Section List -> Serialized Binary Strings +#if 0 RDI_PROC RDIM_String8List rdim_serialized_strings_from_params_bake_section_list(RDIM_Arena *arena, RDIM_BakeParams *params, RDIM_BakeSectionList *sections) { @@ -4947,3 +5093,4 @@ rdim_serialized_strings_from_params_bake_section_list(RDIM_Arena *arena, RDIM_Ba } return strings; } +#endif diff --git a/src/lib_rdi_make/rdi_make.h b/src/lib_rdi_make/rdi_make.h index 04e12a75..328a9793 100644 --- a/src/lib_rdi_make/rdi_make.h +++ b/src/lib_rdi_make/rdi_make.h @@ -1219,13 +1219,46 @@ struct RDIM_IndexRunBakeResult RDI_U64 idx_count; }; -//////////////////////////////// -//~ rjf: Serializing Types - -typedef struct RDIM_SerializeParams RDIM_SerializeParams; -struct RDIM_SerializeParams +typedef struct RDIM_BakeResults RDIM_BakeResults; +struct RDIM_BakeResults { - RDIM_String8 section_data[RDI_SectionKind_COUNT]; + RDIM_TopLevelInfoBakeResult top_level_info; + RDIM_BinarySectionBakeResult binary_sections; + RDIM_UnitBakeResult units; + RDIM_UnitVMapBakeResult unit_vmap; + RDIM_SrcFileBakeResult src_files; + RDIM_LineTableBakeResult line_tables; + RDIM_TypeNodeBakeResult type_nodes; + RDIM_UDTBakeResult udts; + RDIM_GlobalVariableBakeResult global_variables; + RDIM_GlobalVMapBakeResult global_vmap; + RDIM_ThreadVariableBakeResult thread_variables; + RDIM_ProcedureBakeResult procedures; + RDIM_ScopeBakeResult scopes; + RDIM_ScopeVMapBakeResult scope_vmap; + RDIM_TopLevelNameMapBakeResult top_level_name_maps; + RDIM_NameMapBakeResult name_maps; + RDIM_FilePathBakeResult file_paths; + RDIM_StringBakeResult strings; + RDIM_IndexRunBakeResult idx_runs; +}; + +//////////////////////////////// +//~ rjf: Serialization Types + +typedef struct RDIM_SerializedSection RDIM_SerializedSection; +struct RDIM_SerializedSection +{ + void *data; + RDI_U64 encoded_size; + RDI_U64 unpacked_size; + RDI_SectionEncoding encoding; +}; + +typedef struct RDIM_SerializedSectionBundle RDIM_SerializedSectionBundle; +struct RDIM_SerializedSectionBundle +{ + RDIM_SerializedSection sections[RDI_SectionKind_COUNT]; }; //////////////////////////////// @@ -1443,6 +1476,9 @@ RDI_PROC RDIM_BakePathTree *rdim_bake_path_tree_from_params(RDIM_Arena *arena, R //- rjf: partial/joinable baking functions RDI_PROC RDIM_NameMapBakeResult rdim_bake_name_map(RDIM_Arena *arena, RDIM_BakeStringMapTight *strings, RDIM_BakeIdxRunMap *idx_runs, RDIM_BakeNameMap *src, RDI_U64 base_node_idx); +//- rjf: partial bakes -> final bake functions +RDI_PROC RDIM_NameMapBakeResult rdim_name_map_bake_results_combine(RDIM_Arena *arena, RDIM_NameMapBakeResult *results, RDI_U64 results_count); + //- rjf: independent (top-level, global) baking functions RDI_PROC RDIM_TopLevelInfoBakeResult rdim_bake_top_level_info(RDIM_Arena *arena, RDIM_BakeStringMapTight *strings, RDIM_TopLevelInfo *src); RDI_PROC RDIM_BinarySectionBakeResult rdim_bake_binary_sections(RDIM_Arena *arena, RDIM_BakeStringMapTight *strings, RDIM_BinarySectionList *src); @@ -1463,6 +1499,15 @@ RDI_PROC RDIM_FilePathBakeResult rdim_bake_file_paths(RDIM_Arena *arena, RDI_PROC RDIM_StringBakeResult rdim_bake_strings(RDIM_Arena *arena, RDIM_BakeStringMapTight *strings); RDI_PROC RDIM_IndexRunBakeResult rdim_bake_index_runs(RDIM_Arena *arena, RDIM_BakeIdxRunMap *idx_runs); +//////////////////////////////// +//~ rjf: [Serializing] Bake Results -> String Blobs + +RDI_PROC RDIM_SerializedSection rdim_serialized_section_make_unpacked(void *data, RDI_U64 size); +#define rdim_serialized_section_make_unpacked_struct(ptr) rdim_serialized_section_make_unpacked((ptr), sizeof(*(ptr))) +#define rdim_serialized_section_make_unpacked_array(ptr, count) rdim_serialized_section_make_unpacked((ptr), sizeof(*(ptr))*(count)) +RDI_PROC RDIM_SerializedSectionBundle rdim_serialized_section_bundle_from_bake_results(RDIM_BakeResults *results); +RDI_PROC RDIM_String8List rdim_file_blobs_from_section_bundle(RDIM_Arena *arena, RDIM_SerializedSectionBundle *bundle); + //////////////////////////////// //~ rjf: [Baking] Build Artifacts -> Data Section Lists @@ -1526,6 +1571,8 @@ RDI_PROC RDIM_BakeSectionList rdim_bake_idx_run_section_list_from_idx_run_map(RD //////////////////////////////// //~ rjf: [Serializing] Baked Data Section List -> Serialized Binary Strings +#if 0 RDI_PROC RDIM_String8List rdim_serialized_strings_from_params_bake_section_list(RDIM_Arena *arena, RDIM_BakeParams *params, RDIM_BakeSectionList *sections); +#endif #endif // RDI_MAKE_H diff --git a/src/raddbg/raddbg_main.c b/src/raddbg/raddbg_main.c index d9892de7..9db07fce 100644 --- a/src/raddbg/raddbg_main.c +++ b/src/raddbg/raddbg_main.c @@ -515,21 +515,30 @@ entry_point(CmdLine *cmd_line) bake2srlz = p2r_bake(scratch.arena, convert2bake); } + //- rjf: serialize + P2R_Serialize2File *srlz2file = 0; + ProfScope("serialize") + { + srlz2file = push_array(scratch.arena, P2R_Serialize2File, 1); + srlz2file->bundle = rdim_serialized_section_bundle_from_bake_results(&bake2srlz->bake_results); + } + //- rjf: compress - P2R_Bake2Serialize *bake2srlz_compressed = bake2srlz; + P2R_Serialize2File *srlz2file_compressed = srlz2file; if(cmd_line_has_flag(cmd_line, str8_lit("compress"))) ProfScope("compress") { - bake2srlz_compressed = p2r_compress(scratch.arena, bake2srlz); + srlz2file_compressed = push_array(scratch.arena, P2R_Serialize2File, 1); + srlz2file_compressed = p2r_compress(scratch.arena, srlz2file); } //- rjf: serialize - String8List serialize_out = rdim_serialized_strings_from_params_bake_section_list(scratch.arena, &convert2bake->bake_params, &bake2srlz_compressed->sections); + String8List blobs = rdim_file_blobs_from_section_bundle(scratch.arena, &srlz2file_compressed->bundle); //- rjf: write if(out_file_is_good) { U64 off = 0; - for(String8Node *n = serialize_out.first; n != 0; n = n->next) + for(String8Node *n = blobs.first; n != 0; n = n->next) { os_file_write(out_file, r1u64(off, off+n->string.size), n->string.str); off += n->string.size; diff --git a/src/rdi_breakpad_from_pdb/rdi_breakpad_from_pdb_main.c b/src/rdi_breakpad_from_pdb/rdi_breakpad_from_pdb_main.c index 8a125b47..b1578399 100644 --- a/src/rdi_breakpad_from_pdb/rdi_breakpad_from_pdb_main.c +++ b/src/rdi_breakpad_from_pdb/rdi_breakpad_from_pdb_main.c @@ -26,7 +26,7 @@ #include "base/base_inc.h" #include "os/os_inc.h" #include "task_system/task_system.h" -#include "rdi_make_local/rdi_make_local.h" +#include "rdi_make/rdi_make_local.h" #include "coff/coff.h" #include "codeview/codeview.h" #include "codeview/codeview_stringize.h" @@ -39,7 +39,7 @@ #include "base/base_inc.c" #include "os/os_inc.c" #include "task_system/task_system.c" -#include "rdi_make_local/rdi_make_local.c" +#include "rdi_make/rdi_make_local.c" #include "coff/coff.c" #include "codeview/codeview.c" #include "codeview/codeview_stringize.c" @@ -56,7 +56,7 @@ typedef struct P2B_BakeUnitVMapIn P2B_BakeUnitVMapIn; struct P2B_BakeUnitVMapIn { - RDIM_BakeParams *params; + RDIM_UnitChunkList *units; }; typedef struct P2B_BakeUnitVMapOut P2B_BakeUnitVMapOut; @@ -70,21 +70,9 @@ internal TS_TASK_FUNCTION_DEF(p2b_bake_unit_vmap_task__entry_point) { P2B_BakeUnitVMapIn *in = (P2B_BakeUnitVMapIn *)p; P2B_BakeUnitVMapOut *out = push_array(arena, P2B_BakeUnitVMapOut, 1); - RDIM_BakeSectionList sections = rdim_bake_unit_vmap_section_list_from_params(arena, in->params); - RDIM_BakeSection *vmap_section = 0; - for(RDIM_BakeSectionNode *n = sections.first; n != 0 && vmap_section == 0; n = n->next) - { - switch(n->v.tag) - { - default:{}break; - case RDI_SectionKind_UnitVmap:{vmap_section = &n->v;}break; - } - } - if(vmap_section != 0) - { - out->vmap_entries = (RDI_VMapEntry *)vmap_section->data; - out->vmap_entries_count = vmap_section->unpacked_size/sizeof(RDI_VMapEntry); - } + RDIM_UnitVMapBakeResult bake = rdim_bake_unit_vmap(arena, in->units); + out->vmap_entries = bake.vmap.vmap; + out->vmap_entries_count = bake.vmap.count+1; return out; } @@ -251,7 +239,7 @@ entry_point(CmdLine *cmdline) P2B_BakeUnitVMapIn bake_unit_vmap_in = {params}; TS_Ticket bake_unit_vmap_ticket = ts_kickoff(p2b_bake_unit_vmap_task__entry_point, 0, &bake_unit_vmap_in); - //- rjf: kick off per-unit baking + //- rjf: kick off per-line-table baking P2B_BakeUnitIn *bake_units_in = push_array(arena, P2B_BakeUnitIn, params->units.total_count+1); TS_Ticket *bake_units_tickets = push_array(arena, TS_Ticket, params->units.total_count+1); { diff --git a/src/rdi_dump/rdi_dump_main.c b/src/rdi_dump/rdi_dump_main.c index 3e26bdb6..3aa06373 100644 --- a/src/rdi_dump/rdi_dump_main.c +++ b/src/rdi_dump/rdi_dump_main.c @@ -74,9 +74,9 @@ entry_point(CmdLine *cmd_line) // rjf: extract input file path input_name = str8_list_first(&cmd_line->inputs); - // rjf: extract dump options + // rjf: extract "only" options { - String8List dump_options = cmd_line_strings(cmd_line, str8_lit("dump")); + String8List dump_options = cmd_line_strings(cmd_line, str8_lit("only")); if(dump_options.first != 0) { dump_flags = 0; @@ -101,6 +101,7 @@ entry_point(CmdLine *cmd_line) else if(str8_match(n->string, str8_lit("scopes"), StringMatchFlag_CaseInsensitive)) { dump_flags |= DumpFlag_Scopes; } else if(str8_match(n->string, str8_lit("scope_vmap"), StringMatchFlag_CaseInsensitive)) { dump_flags |= DumpFlag_ScopeVMap; } else if(str8_match(n->string, str8_lit("name_maps"), StringMatchFlag_CaseInsensitive)) { dump_flags |= DumpFlag_NameMaps; } + else if(str8_match(n->string, str8_lit("strings"), StringMatchFlag_CaseInsensitive)) { dump_flags |= DumpFlag_Strings; } } } } diff --git a/src/rdi_from_pdb/rdi_from_pdb.c b/src/rdi_from_pdb/rdi_from_pdb.c index d0c8afc4..41591105 100644 --- a/src/rdi_from_pdb/rdi_from_pdb.c +++ b/src/rdi_from_pdb/rdi_from_pdb.c @@ -3628,9 +3628,9 @@ internal TS_TASK_FUNCTION_DEF(p2r_bake_scopes_strings_task__entry_point) internal TS_TASK_FUNCTION_DEF(p2r_bake_line_tables_task__entry_point) { P2R_BakeLineTablesIn *in = (P2R_BakeLineTablesIn *)p; - RDIM_BakeSectionList *s = push_array(arena, RDIM_BakeSectionList, 1); - ProfScope("bake line tables") *s = rdim_bake_line_table_section_list_from_params(arena, in->params); - return s; + RDIM_LineTableBakeResult *out = push_array(arena, RDIM_LineTableBakeResult, 1); + ProfScope("bake line tables") *out = rdim_bake_line_tables(arena, in->line_tables); + return out; } #undef p2r_make_string_map_if_needed @@ -3721,81 +3721,81 @@ internal TS_TASK_FUNCTION_DEF(p2r_bake_unit_vmap_task__entry_point) internal TS_TASK_FUNCTION_DEF(p2r_bake_src_files_task__entry_point) { P2R_BakeSrcFilesIn *in = (P2R_BakeSrcFilesIn *)p; - RDIM_BakeSectionList *s = push_array(arena, RDIM_BakeSectionList, 1); - ProfScope("bake src files") *s = rdim_bake_src_file_section_list_from_params(arena, in->strings, in->path_tree, in->params); - return s; + RDIM_SrcFileBakeResult *out = push_array(arena, RDIM_SrcFileBakeResult, 1); + ProfScope("bake src files") *out = rdim_bake_src_files(arena, in->strings, in->path_tree, in->src_files); + return out; } internal TS_TASK_FUNCTION_DEF(p2r_bake_udts_task__entry_point) { P2R_BakeUDTsIn *in = (P2R_BakeUDTsIn *)p; - RDIM_BakeSectionList *s = push_array(arena, RDIM_BakeSectionList, 1); - ProfScope("bake udts") *s = rdim_bake_udt_section_list_from_params(arena, in->strings, in->params); - return s; + RDIM_UDTBakeResult *out = push_array(arena, RDIM_UDTBakeResult, 1); + ProfScope("bake udts") *out = rdim_bake_udts(arena, in->strings, in->udts); + return out; } internal TS_TASK_FUNCTION_DEF(p2r_bake_global_variables_task__entry_point) { P2R_BakeGlobalVariablesIn *in = (P2R_BakeGlobalVariablesIn *)p; - RDIM_BakeSectionList *s = push_array(arena, RDIM_BakeSectionList, 1); - ProfScope("bake global variables") *s = rdim_bake_global_variable_section_list_from_params(arena, in->strings, in->params); - return s; + RDIM_GlobalVariableBakeResult *out = push_array(arena, RDIM_GlobalVariableBakeResult, 1); + ProfScope("bake global variables") *out = rdim_bake_global_variables(arena, in->strings, in->global_variables); + return out; } internal TS_TASK_FUNCTION_DEF(p2r_bake_global_vmap_task__entry_point) { P2R_BakeGlobalVMapIn *in = (P2R_BakeGlobalVMapIn *)p; - RDIM_BakeSectionList *s = push_array(arena, RDIM_BakeSectionList, 1); - ProfScope("bake global vmap") *s = rdim_bake_global_vmap_section_list_from_params(arena, in->params); - return s; + RDIM_GlobalVMapBakeResult *out = push_array(arena, RDIM_GlobalVMapBakeResult, 1); + ProfScope("bake global vmap") *out = rdim_bake_global_vmap(arena, in->global_variables); + return out; } internal TS_TASK_FUNCTION_DEF(p2r_bake_thread_variables_task__entry_point) { P2R_BakeThreadVariablesIn *in = (P2R_BakeThreadVariablesIn *)p; - RDIM_BakeSectionList *s = push_array(arena, RDIM_BakeSectionList, 1); - ProfScope("bake thread variables") *s = rdim_bake_thread_variable_section_list_from_params(arena, in->strings, in->params); - return s; + RDIM_ThreadVariableBakeResult *out = push_array(arena, RDIM_ThreadVariableBakeResult, 1); + ProfScope("bake thread variables") *out = rdim_bake_thread_variables(arena, in->strings, in->thread_variables); + return out; } internal TS_TASK_FUNCTION_DEF(p2r_bake_procedures_task__entry_point) { P2R_BakeProceduresIn *in = (P2R_BakeProceduresIn *)p; - RDIM_BakeSectionList *s = push_array(arena, RDIM_BakeSectionList, 1); - ProfScope("bake procedures") *s = rdim_bake_procedure_section_list_from_params(arena, in->strings, in->params); - return s; + RDIM_ProcedureBakeResult *out = push_array(arena, RDIM_ProcedureBakeResult, 1); + ProfScope("bake procedures") *out = rdim_bake_procedures(arena, in->strings, in->procedures); + return out; } internal TS_TASK_FUNCTION_DEF(p2r_bake_scopes_task__entry_point) { P2R_BakeScopesIn *in = (P2R_BakeScopesIn *)p; - RDIM_BakeSectionList *s = push_array(arena, RDIM_BakeSectionList, 1); - ProfScope("bake scopes") *s = rdim_bake_scope_section_list_from_params(arena, in->strings, in->params); - return s; + RDIM_ScopeBakeResult *out = push_array(arena, RDIM_ScopeBakeResult, 1); + ProfScope("bake scopes") *out = rdim_bake_scopes(arena, in->strings, in->scopes); + return out; } internal TS_TASK_FUNCTION_DEF(p2r_bake_scope_vmap_task__entry_point) { P2R_BakeScopeVMapIn *in = (P2R_BakeScopeVMapIn *)p; - RDIM_BakeSectionList *s = push_array(arena, RDIM_BakeSectionList, 1); - ProfScope("bake scope vmap") *s = rdim_bake_scope_vmap_section_list_from_params(arena, in->params); - return s; + RDIM_ScopeVMapBakeResult *out = push_array(arena, RDIM_ScopeVMapBakeResult, 1); + ProfScope("bake scope vmap") *out = rdim_bake_scope_vmap(arena, in->scopes); + return out; } internal TS_TASK_FUNCTION_DEF(p2r_bake_file_paths_task__entry_point) { P2R_BakeFilePathsIn *in = (P2R_BakeFilePathsIn *)p; - RDIM_BakeSectionList *s = push_array(arena, RDIM_BakeSectionList, 1); - ProfScope("bake file paths") *s = rdim_bake_file_path_section_list_from_path_tree(arena, in->strings, in->path_tree); - return s; + RDIM_FilePathBakeResult *out = push_array(arena, RDIM_FilePathBakeResult, 1); + ProfScope("bake file paths") *out = rdim_bake_file_paths(arena, in->strings, in->path_tree); + return out; } internal TS_TASK_FUNCTION_DEF(p2r_bake_strings_task__entry_point) { P2R_BakeStringsIn *in = (P2R_BakeStringsIn *)p; - RDIM_BakeSectionList *s = push_array(arena, RDIM_BakeSectionList, 1); - ProfScope("bake strings") *s = rdim_bake_string_section_list_from_string_map(arena, in->strings); - return s; + RDIM_StringBakeResult *out = push_array(arena, RDIM_StringBakeResult, 1); + ProfScope("bake strings") *out = rdim_bake_strings(arena, in->strings); + return out; } //- rjf: pass 3: idx-run-map-dependent debug info stream builds @@ -3803,25 +3803,25 @@ internal TS_TASK_FUNCTION_DEF(p2r_bake_strings_task__entry_point) internal TS_TASK_FUNCTION_DEF(p2r_bake_type_nodes_task__entry_point) { P2R_BakeTypeNodesIn *in = (P2R_BakeTypeNodesIn *)p; - RDIM_BakeSectionList *s = push_array(arena, RDIM_BakeSectionList, 1); - ProfScope("bake type nodes") *s = rdim_bake_type_node_section_list_from_params(arena, in->strings, in->idx_runs, in->params); - return s; + RDIM_TypeNodeBakeResult *out = push_array(arena, RDIM_TypeNodeBakeResult, 1); + ProfScope("bake type nodes") *out = rdim_bake_types(arena, in->strings, in->idx_runs, in->types); + return out; } internal TS_TASK_FUNCTION_DEF(p2r_bake_name_map_task__entry_point) { P2R_BakeNameMapIn *in = (P2R_BakeNameMapIn *)p; - RDIM_BakeSectionList *s = push_array(arena, RDIM_BakeSectionList, 1); - ProfScope("bake name map %i", in->kind) *s = rdim_bake_name_map_section_list_from_params_kind_map(arena, in->strings, in->idx_runs, in->params, in->kind, in->map); - return s; + RDIM_NameMapBakeResult *out = push_array(arena, RDIM_NameMapBakeResult, 1); + ProfScope("bake name map %i", in->kind) *out = rdim_bake_name_map(arena, in->strings, in->idx_runs, in->map, in->base_node_idx); + return out; } internal TS_TASK_FUNCTION_DEF(p2r_bake_idx_runs_task__entry_point) { P2R_BakeIdxRunsIn *in = (P2R_BakeIdxRunsIn *)p; - RDIM_BakeSectionList *s = push_array(arena, RDIM_BakeSectionList, 1); - ProfScope("bake idx runs") *s = rdim_bake_idx_run_section_list_from_idx_run_map(arena, in->idx_runs); - return s; + RDIM_IndexRunBakeResult *out = push_array(arena, RDIM_IndexRunBakeResult, 1); + ProfScope("bake idx runs") *out = rdim_bake_index_runs(arena, in->idx_runs); + return out; } //////////////////////////////// @@ -3833,24 +3833,30 @@ p2r_bake(Arena *arena, P2R_Convert2Bake *in) Temp scratch = scratch_begin(&arena, 1); RDIM_BakeParams *in_params = &in->bake_params; P2R_Bake2Serialize *out = push_array(arena, P2R_Bake2Serialize, 1); - RDIM_SerializeParams *out_params = &out->serialize_params; + RDIM_BakeResults *out_results = &out->bake_results; + ////////////////////////////// //- rjf: kick off line tables baking + // TS_Ticket bake_line_tables_ticket = {0}; { P2R_BakeLineTablesIn *in = push_array(scratch.arena, P2R_BakeLineTablesIn, 1); - in->params = in_params; + in->line_tables = &in_params->line_tables; bake_line_tables_ticket = ts_kickoff(p2r_bake_line_tables_task__entry_point, 0, in); } + ////////////////////////////// //- rjf: build interned path tree + // RDIM_BakePathTree *path_tree = 0; ProfScope("build interned path tree") { path_tree = rdim_bake_path_tree_from_params(arena, in_params); } + ////////////////////////////// //- rjf: kick off string map building tasks + // RDIM_BakeStringMapTopology bake_string_map_topology = {(in_params->procedures.total_count*1 + in_params->global_variables.total_count*1 + in_params->thread_variables.total_count*1 + @@ -3881,42 +3887,64 @@ p2r_bake(Arena *arena, P2R_Convert2Bake *in) // rjf: types ProfScope("kick off types string map build tasks") { - for(RDIM_TypeChunkNode *chunk = in_params->types.first; chunk != 0; chunk = chunk->next) + U64 items_per_task = 4096; + U64 num_tasks = (in_params->types.total_count+items_per_task-1)/items_per_task; + RDIM_TypeChunkNode *chunk = in_params->types.first; + U64 chunk_off = 0; + for(U64 task_idx = 0; task_idx < num_tasks; task_idx += 1) { - U64 items_per_task = Min(4096, chunk->count); - U64 tasks_per_this_chunk = (chunk->count+items_per_task-1)/items_per_task; - for(U64 task_idx = 0; task_idx < tasks_per_this_chunk; task_idx += 1) + P2R_BakeTypesStringsIn *in = push_array(scratch.arena, P2R_BakeTypesStringsIn, 1); + in->top = &bake_string_map_topology; + in->maps = bake_string_maps__in_progress; + U64 items_left = items_per_task; + for(;chunk != 0 && items_left > 0;) { - P2R_BakeTypesStringsIn *in = push_array(scratch.arena, P2R_BakeTypesStringsIn, 1); - in->top = &bake_string_map_topology; - in->maps = bake_string_maps__in_progress; + U64 items_in_this_chunk = Min(items_per_task, chunk->count-chunk_off); P2R_BakeTypesStringsInNode *n = push_array(scratch.arena, P2R_BakeTypesStringsInNode, 1); SLLQueuePush(in->first, in->last, n); - n->v = chunk->v + task_idx*items_per_task; - n->count = Min(items_per_task, chunk->count - task_idx*items_per_task); - ts_ticket_list_push(scratch.arena, &bake_string_map_build_tickets, ts_kickoff(p2r_bake_types_strings_task__entry_point, 0, in)); + n->v = chunk->v + chunk_off; + n->count = items_in_this_chunk; + chunk_off += items_in_this_chunk; + items_left -= items_in_this_chunk; + if(chunk_off >= chunk->count) + { + chunk = chunk->next; + chunk_off = 0; + } } + ts_ticket_list_push(scratch.arena, &bake_string_map_build_tickets, ts_kickoff(p2r_bake_types_strings_task__entry_point, 0, in)); } } // rjf: UDTs ProfScope("kick off udts string map build tasks") { - for(RDIM_UDTChunkNode *chunk = in_params->udts.first; chunk != 0; chunk = chunk->next) + U64 items_per_task = 4096; + U64 num_tasks = (in_params->udts.total_count+items_per_task-1)/items_per_task; + RDIM_UDTChunkNode *chunk = in_params->udts.first; + U64 chunk_off = 0; + for(U64 task_idx = 0; task_idx < num_tasks; task_idx += 1) { - U64 items_per_task = Min(4096, chunk->count); - U64 tasks_per_this_chunk = (chunk->count+items_per_task-1)/items_per_task; - for(U64 task_idx = 0; task_idx < tasks_per_this_chunk; task_idx += 1) + P2R_BakeUDTsStringsIn *in = push_array(scratch.arena, P2R_BakeUDTsStringsIn, 1); + in->top = &bake_string_map_topology; + in->maps = bake_string_maps__in_progress; + U64 items_left = items_per_task; + for(;chunk != 0 && items_left > 0;) { - P2R_BakeUDTsStringsIn *in = push_array(scratch.arena, P2R_BakeUDTsStringsIn, 1); - in->top = &bake_string_map_topology; - in->maps = bake_string_maps__in_progress; + U64 items_in_this_chunk = Min(items_per_task, chunk->count-chunk_off); P2R_BakeUDTsStringsInNode *n = push_array(scratch.arena, P2R_BakeUDTsStringsInNode, 1); SLLQueuePush(in->first, in->last, n); - n->v = chunk->v + task_idx*items_per_task; - n->count = Min(items_per_task, chunk->count - task_idx*items_per_task); - ts_ticket_list_push(scratch.arena, &bake_string_map_build_tickets, ts_kickoff(p2r_bake_udts_strings_task__entry_point, 0, in)); + n->v = chunk->v + chunk_off; + n->count = items_in_this_chunk; + chunk_off += items_in_this_chunk; + items_left -= items_in_this_chunk; + if(chunk_off >= chunk->count) + { + chunk = chunk->next; + chunk_off = 0; + } } + ts_ticket_list_push(scratch.arena, &bake_string_map_build_tickets, ts_kickoff(p2r_bake_udts_strings_task__entry_point, 0, in)); } } @@ -3931,21 +3959,32 @@ p2r_bake(Arena *arena, P2R_Convert2Bake *in) }; for(U64 list_idx = 0; list_idx < ArrayCount(symbol_lists); list_idx += 1) { - for(RDIM_SymbolChunkNode *chunk = symbol_lists[list_idx]->first; chunk != 0; chunk = chunk->next) + U64 items_per_task = 4096; + U64 num_tasks = (symbol_lists[list_idx]->total_count+items_per_task-1)/items_per_task; + RDIM_SymbolChunkNode *chunk = symbol_lists[list_idx]->first; + U64 chunk_off = 0; + for(U64 task_idx = 0; task_idx < num_tasks; task_idx += 1) { - U64 items_per_task = Min(4096, chunk->count); - U64 tasks_per_this_chunk = (chunk->count+items_per_task-1)/items_per_task; - for(U64 task_idx = 0; task_idx < tasks_per_this_chunk; task_idx += 1) + P2R_BakeSymbolsStringsIn *in = push_array(scratch.arena, P2R_BakeSymbolsStringsIn, 1); + in->top = &bake_string_map_topology; + in->maps = bake_string_maps__in_progress; + U64 items_left = items_per_task; + for(;chunk != 0 && items_left > 0;) { - P2R_BakeSymbolsStringsIn *in = push_array(scratch.arena, P2R_BakeSymbolsStringsIn, 1); - in->top = &bake_string_map_topology; - in->maps = bake_string_maps__in_progress; + U64 items_in_this_chunk = Min(items_per_task, chunk->count-chunk_off); P2R_BakeSymbolsStringsInNode *n = push_array(scratch.arena, P2R_BakeSymbolsStringsInNode, 1); SLLQueuePush(in->first, in->last, n); - n->v = chunk->v + task_idx*items_per_task; - n->count = Min(items_per_task, chunk->count - task_idx*items_per_task); - ts_ticket_list_push(scratch.arena, &bake_string_map_build_tickets, ts_kickoff(p2r_bake_symbols_strings_task__entry_point, 0, in)); + n->v = chunk->v + chunk_off; + n->count = items_in_this_chunk; + chunk_off += items_in_this_chunk; + items_left -= items_in_this_chunk; + if(chunk_off >= chunk->count) + { + chunk = chunk->next; + chunk_off = 0; + } } + ts_ticket_list_push(scratch.arena, &bake_string_map_build_tickets, ts_kickoff(p2r_bake_symbols_strings_task__entry_point, 0, in)); } } } @@ -3953,26 +3992,39 @@ p2r_bake(Arena *arena, P2R_Convert2Bake *in) // rjf: scope chunks ProfScope("kick off scope chunks string map build tasks") { - for(RDIM_ScopeChunkNode *chunk = in_params->scopes.first; chunk != 0; chunk = chunk->next) + U64 items_per_task = 4096; + U64 num_tasks = (in_params->scopes.total_count+items_per_task-1)/items_per_task; + RDIM_ScopeChunkNode *chunk = in_params->scopes.first; + U64 chunk_off = 0; + for(U64 task_idx = 0; task_idx < num_tasks; task_idx += 1) { - U64 items_per_task = Min(4096, chunk->count); - U64 tasks_per_this_chunk = (chunk->count+items_per_task-1)/items_per_task; - for(U64 task_idx = 0; task_idx < tasks_per_this_chunk; task_idx += 1) + P2R_BakeScopesStringsIn *in = push_array(scratch.arena, P2R_BakeScopesStringsIn, 1); + in->top = &bake_string_map_topology; + in->maps = bake_string_maps__in_progress; + U64 items_left = items_per_task; + for(;chunk != 0 && items_left > 0;) { - P2R_BakeScopesStringsIn *in = push_array(scratch.arena, P2R_BakeScopesStringsIn, 1); - in->top = &bake_string_map_topology; - in->maps = bake_string_maps__in_progress; + U64 items_in_this_chunk = Min(items_per_task, chunk->count-chunk_off); P2R_BakeScopesStringsInNode *n = push_array(scratch.arena, P2R_BakeScopesStringsInNode, 1); SLLQueuePush(in->first, in->last, n); - n->v = chunk->v + task_idx*items_per_task; - n->count = Min(items_per_task, chunk->count - task_idx*items_per_task); - ts_ticket_list_push(scratch.arena, &bake_string_map_build_tickets, ts_kickoff(p2r_bake_scopes_strings_task__entry_point, 0, in)); + n->v = chunk->v + chunk_off; + n->count = items_in_this_chunk; + chunk_off += items_in_this_chunk; + items_left -= items_in_this_chunk; + if(chunk_off >= chunk->count) + { + chunk = chunk->next; + chunk_off = 0; + } } + ts_ticket_list_push(scratch.arena, &bake_string_map_build_tickets, ts_kickoff(p2r_bake_scopes_strings_task__entry_point, 0, in)); } } } + ////////////////////////////// //- rjf: kick off name map building tasks + // P2R_BuildBakeNameMapIn build_bake_name_map_in[RDI_NameMapKind_COUNT] = {0}; TS_Ticket build_bake_name_map_ticket[RDI_NameMapKind_COUNT] = {0}; for(RDI_NameMapKind k = (RDI_NameMapKind)(RDI_NameMapKind_NULL+1); @@ -3984,7 +4036,9 @@ p2r_bake(Arena *arena, P2R_Convert2Bake *in) build_bake_name_map_ticket[k] = ts_kickoff(p2r_build_bake_name_map_task__entry_point, 0, &build_bake_name_map_in[k]); } + ////////////////////////////// //- rjf: join string map building tasks + // ProfScope("join string map building tasks") { for(TS_TicketNode *n = bake_string_map_build_tickets.first; n != 0; n = n->next) @@ -3993,7 +4047,9 @@ p2r_bake(Arena *arena, P2R_Convert2Bake *in) } } + ////////////////////////////// //- rjf: produce joined string map + // RDIM_BakeStringMapLoose *unsorted_bake_string_map = rdim_bake_string_map_loose_make(arena, &bake_string_map_topology); ProfScope("produce joined string map") { @@ -4026,7 +4082,9 @@ p2r_bake(Arena *arena, P2R_Convert2Bake *in) rdim_bake_string_map_loose_push_path_tree(arena, &bake_string_map_topology, unsorted_bake_string_map, path_tree); } + ////////////////////////////// //- rjf: kick off string map sorting tasks + // TS_TicketList sort_bake_string_map_task_tickets = {0}; RDIM_BakeStringMapLoose *sorted_bake_string_map__in_progress = rdim_bake_string_map_loose_make(arena, &bake_string_map_topology); { @@ -4050,7 +4108,9 @@ p2r_bake(Arena *arena, P2R_Convert2Bake *in) } } + ////////////////////////////// //- rjf: join string map sorting tasks + // ProfScope("join string map sorting tasks") { for(TS_TicketNode *n = sort_bake_string_map_task_tickets.first; n != 0; n = n->next) @@ -4060,7 +4120,9 @@ p2r_bake(Arena *arena, P2R_Convert2Bake *in) } RDIM_BakeStringMapLoose *sorted_bake_string_map = sorted_bake_string_map__in_progress; + ////////////////////////////// //- rjf: build finalized string map + // ProfBegin("build finalized string map base indices"); RDIM_BakeStringMapBaseIndices bake_string_map_base_idxes = rdim_bake_string_map_base_indices_from_map_loose(arena, &bake_string_map_topology, sorted_bake_string_map); ProfEnd(); @@ -4068,47 +4130,37 @@ p2r_bake(Arena *arena, P2R_Convert2Bake *in) RDIM_BakeStringMapTight bake_strings = rdim_bake_string_map_tight_from_loose(arena, &bake_string_map_topology, &bake_string_map_base_idxes, sorted_bake_string_map); ProfEnd(); + ////////////////////////////// //- rjf: kick off pass 2 tasks + // P2R_BakeUnitsIn bake_units_top_level_in = {&bake_strings, path_tree, &in_params->units}; TS_Ticket bake_units_ticket = ts_kickoff(p2r_bake_units_task__entry_point, 0, &bake_units_top_level_in); P2R_BakeUnitVMapIn bake_unit_vmap_in = {&in_params->units}; TS_Ticket bake_unit_vmap_ticket = ts_kickoff(p2r_bake_unit_vmap_task__entry_point, 0, &bake_unit_vmap_in); - P2R_BakeSrcFilesIn bake_src_files_in = {&bake_strings, path_tree, in_params}; + P2R_BakeSrcFilesIn bake_src_files_in = {&bake_strings, path_tree, &in_params->src_files}; TS_Ticket bake_src_files_ticket = ts_kickoff(p2r_bake_src_files_task__entry_point, 0, &bake_src_files_in); - P2R_BakeUDTsIn bake_udts_in = {&bake_strings, in_params}; + P2R_BakeUDTsIn bake_udts_in = {&bake_strings, &in_params->udts}; TS_Ticket bake_udts_ticket = ts_kickoff(p2r_bake_udts_task__entry_point, 0, &bake_udts_in); - P2R_BakeGlobalVariablesIn bake_global_variables_in = {&bake_strings, in_params}; + P2R_BakeGlobalVariablesIn bake_global_variables_in = {&bake_strings, &in_params->global_variables}; TS_Ticket bake_global_variables_ticket = ts_kickoff(p2r_bake_global_variables_task__entry_point, 0, &bake_global_variables_in); - P2R_BakeGlobalVMapIn bake_global_vmap_in = {in_params}; + P2R_BakeGlobalVMapIn bake_global_vmap_in = {&in_params->global_variables}; TS_Ticket bake_global_vmap_ticket = ts_kickoff(p2r_bake_global_vmap_task__entry_point, 0, &bake_global_vmap_in); - P2R_BakeThreadVariablesIn bake_thread_variables_in = {&bake_strings, in_params}; + P2R_BakeThreadVariablesIn bake_thread_variables_in = {&bake_strings, &in_params->thread_variables}; TS_Ticket bake_thread_variables_ticket = ts_kickoff(p2r_bake_thread_variables_task__entry_point, 0, &bake_thread_variables_in); - P2R_BakeProceduresIn bake_procedures_in = {&bake_strings, in_params}; + P2R_BakeProceduresIn bake_procedures_in = {&bake_strings, &in_params->procedures}; TS_Ticket bake_procedures_ticket = ts_kickoff(p2r_bake_procedures_task__entry_point, 0, &bake_procedures_in); - P2R_BakeScopesIn bake_scopes_in = {&bake_strings, in_params}; + P2R_BakeScopesIn bake_scopes_in = {&bake_strings, &in_params->scopes}; TS_Ticket bake_scopes_ticket = ts_kickoff(p2r_bake_scopes_task__entry_point, 0, &bake_scopes_in); - P2R_BakeScopeVMapIn bake_scope_vmap_in = {in_params}; + P2R_BakeScopeVMapIn bake_scope_vmap_in = {&in_params->scopes}; TS_Ticket bake_scope_vmap_ticket = ts_kickoff(p2r_bake_scope_vmap_task__entry_point, 0, &bake_scope_vmap_in); P2R_BakeFilePathsIn bake_file_paths_in = {&bake_strings, path_tree}; TS_Ticket bake_file_paths_ticket = ts_kickoff(p2r_bake_file_paths_task__entry_point, 0, &bake_file_paths_in); P2R_BakeStringsIn bake_strings_in = {&bake_strings}; TS_Ticket bake_strings_ticket = ts_kickoff(p2r_bake_strings_task__entry_point, 0, &bake_strings_in); - //- rjf: top-level info - ProfScope("top level info") - { - RDIM_TopLevelInfoBakeResult bake = rdim_bake_top_level_info(arena, &bake_strings, &in_params->top_level_info); - out_params->section_data[RDI_SectionKind_TopLevelInfo] = rdim_str8_struct(bake.top_level_info); - } - - //- rjf: binary sections - ProfScope("binary sections") - { - RDIM_BinarySectionBakeResult bake = rdim_bake_binary_sections(arena, &bake_strings, &in_params->binary_sections); - out_params->section_data[RDI_SectionKind_BinarySections] = rdim_str8_struct_array(bake.binary_sections, bake.binary_sections_count); - } - + ////////////////////////////// //- rjf: join name map building tasks + // RDIM_BakeNameMap *name_maps[RDI_NameMapKind_COUNT] = {0}; ProfScope("join name map building tasks") { @@ -4120,159 +4172,88 @@ p2r_bake(Arena *arena, P2R_Convert2Bake *in) } } + ////////////////////////////// //- rjf: build interned idx run map + // RDIM_BakeIdxRunMap *idx_runs = 0; ProfScope("build interned idx run map") { idx_runs = rdim_bake_idx_run_map_from_params(arena, name_maps, in_params); } + ////////////////////////////// + //- rjf: do small top-level bakes + // + ProfScope("top level info") out_results->top_level_info = rdim_bake_top_level_info(arena, &bake_strings, &in_params->top_level_info); + ProfScope("binary sections") out_results->binary_sections = rdim_bake_binary_sections(arena, &bake_strings, &in_params->binary_sections); + ProfScope("top level name maps section") out_results->top_level_name_maps = rdim_bake_name_maps_top_level(arena, &bake_strings, idx_runs, name_maps); + + ////////////////////////////// //- rjf: kick off pass 3 tasks - P2R_BakeTypeNodesIn bake_type_nodes_in = {&bake_strings, idx_runs, in_params}; + // + P2R_BakeTypeNodesIn bake_type_nodes_in = {&bake_strings, idx_runs, &in_params->types}; TS_Ticket bake_type_nodes_ticket = ts_kickoff(p2r_bake_type_nodes_task__entry_point, 0, &bake_type_nodes_in); - TS_TicketList bake_name_maps_tickets = {0}; - for(RDI_NameMapKind k = (RDI_NameMapKind)(RDI_NameMapKind_NULL+1); - k < RDI_NameMapKind_COUNT; - k = (RDI_NameMapKind)(k+1)) + TS_Ticket bake_name_maps_tickets[RDI_NameMapKind_COUNT] = {0}; { - if(name_maps[k] == 0 || name_maps[k]->name_count == 0) + U64 base_node_idx = 0; + for(EachNonZeroEnumVal(RDI_NameMapKind, k)) { - continue; + if(name_maps[k] == 0 || name_maps[k]->name_count == 0) + { + continue; + } + P2R_BakeNameMapIn *in = push_array(scratch.arena, P2R_BakeNameMapIn, 1); + in->strings = &bake_strings; + in->idx_runs = idx_runs; + in->map = name_maps[k]; + in->base_node_idx = base_node_idx; + in->kind = k; + bake_name_maps_tickets[k] = ts_kickoff(p2r_bake_name_map_task__entry_point, 0, in); + base_node_idx += name_maps[k]->name_count; } - P2R_BakeNameMapIn *in = push_array(scratch.arena, P2R_BakeNameMapIn, 1); - in->strings = &bake_strings; - in->idx_runs = idx_runs; - in->params = in_params; - in->kind = k; - in->map = name_maps[k]; - ts_ticket_list_push(scratch.arena, &bake_name_maps_tickets, ts_kickoff(p2r_bake_name_map_task__entry_point, 0, in)); } P2R_BakeIdxRunsIn bake_idx_runs_in = {idx_runs}; TS_Ticket bake_idx_runs_ticket = ts_kickoff(p2r_bake_idx_runs_task__entry_point, 0, &bake_idx_runs_in); - //- rjf: bake top-level name maps section - ProfScope("top level name maps section") - { - RDIM_TopLevelNameMapBakeResult bake = rdim_bake_name_maps_top_level(arena, &bake_strings, idx_runs, name_maps); - out_params->section_data[RDI_SectionKind_NameMaps] = rdim_str8_struct_array(bake.name_maps, bake.name_maps_count); - } + ////////////////////////////// + //- rjf: join remaining completed bakes + // + ProfScope("top-level units info") out_results->units = *ts_join_struct(bake_units_ticket, max_U64, RDIM_UnitBakeResult); + ProfScope("unit vmap") out_results->unit_vmap = *ts_join_struct(bake_unit_vmap_ticket, max_U64, RDIM_UnitVMapBakeResult); + ProfScope("source files") out_results->src_files = *ts_join_struct(bake_src_files_ticket, max_U64, RDIM_SrcFileBakeResult); + ProfScope("UDTs") out_results->udts = *ts_join_struct(bake_udts_ticket, max_U64, RDIM_UDTBakeResult); + ProfScope("global variables") out_results->global_variables = *ts_join_struct(bake_global_variables_ticket, max_U64, RDIM_GlobalVariableBakeResult); + ProfScope("global vmap") out_results->global_vmap = *ts_join_struct(bake_global_vmap_ticket, max_U64, RDIM_GlobalVMapBakeResult); + ProfScope("thread variables") out_results->thread_variables = *ts_join_struct(bake_thread_variables_ticket, max_U64, RDIM_ThreadVariableBakeResult); + ProfScope("procedures") out_results->procedures = *ts_join_struct(bake_procedures_ticket, max_U64, RDIM_ProcedureBakeResult); + ProfScope("scopes") out_results->scopes = *ts_join_struct(bake_scopes_ticket, max_U64, RDIM_ScopeBakeResult); + ProfScope("scope vmap") out_results->scope_vmap = *ts_join_struct(bake_scope_vmap_ticket, max_U64, RDIM_ScopeVMapBakeResult); + ProfScope("file paths") out_results->file_paths = *ts_join_struct(bake_file_paths_ticket, max_U64, RDIM_FilePathBakeResult); + ProfScope("strings") out_results->strings = *ts_join_struct(bake_strings_ticket, max_U64, RDIM_StringBakeResult); + ProfScope("type nodes") out_results->type_nodes = *ts_join_struct(bake_type_nodes_ticket, max_U64, RDIM_TypeNodeBakeResult); + ProfScope("idx runs") out_results->idx_runs = *ts_join_struct(bake_idx_runs_ticket, max_U64, RDIM_IndexRunBakeResult); + ProfScope("line tables") out_results->line_tables = *ts_join_struct(bake_line_tables_ticket, max_U64, RDIM_LineTableBakeResult); - //- rjf: join top-level units info - ProfScope("top-level units info") + ////////////////////////////// + //- rjf: join individual name map bakes + // + RDIM_NameMapBakeResult name_map_bakes[RDI_NameMapKind_COUNT] = {0}; + ProfScope("name maps") { - RDIM_UnitBakeResult *bake = ts_join_struct(bake_units_ticket, max_U64, RDIM_UnitBakeResult); - out_params->section_data[RDI_SectionKind_Units] = rdim_str8_struct_array(bake->units, bake->units_count); - } - - //- rjf: join unit vmap - ProfScope("unit vmap") - { - RDIM_UnitVMapBakeResult *bake = ts_join_struct(bake_unit_vmap_ticket, max_U64, RDIM_UnitVMapBakeResult); - out_params->section_data[RDI_SectionKind_UnitVMap] = rdim_str8_struct_array(bake->vmap.vmap, bake->vmap.count+1); - } - - //- rjf: join source files - ProfScope("source files") - { - RDIM_BakeSectionList *s = ts_join_struct(bake_src_files_ticket, max_U64, RDIM_BakeSectionList); - rdim_bake_section_list_concat_in_place(§ions, s); - } - - //- rjf: join UDTs - ProfScope("UDTs") - { - RDIM_BakeSectionList *s = ts_join_struct(bake_udts_ticket, max_U64, RDIM_BakeSectionList); - rdim_bake_section_list_concat_in_place(§ions, s); - } - - //- rjf: join global variables - ProfScope("global variables") - { - RDIM_BakeSectionList *s = ts_join_struct(bake_global_variables_ticket, max_U64, RDIM_BakeSectionList); - rdim_bake_section_list_concat_in_place(§ions, s); - } - - //- rjf: join global vmap - ProfScope("global vmap") - { - RDIM_BakeSectionList *s = ts_join_struct(bake_global_vmap_ticket, max_U64, RDIM_BakeSectionList); - rdim_bake_section_list_concat_in_place(§ions, s); - } - - //- rjf: join thread variables - ProfScope("thread variables") - { - RDIM_BakeSectionList *s = ts_join_struct(bake_thread_variables_ticket, max_U64, RDIM_BakeSectionList); - rdim_bake_section_list_concat_in_place(§ions, s); - } - - //- rjf: join procedures - ProfScope("procedures") - { - RDIM_BakeSectionList *s = ts_join_struct(bake_procedures_ticket, max_U64, RDIM_BakeSectionList); - rdim_bake_section_list_concat_in_place(§ions, s); - } - - //- rjf: join scopes - ProfScope("scopes") - { - RDIM_BakeSectionList *s = ts_join_struct(bake_scopes_ticket, max_U64, RDIM_BakeSectionList); - rdim_bake_section_list_concat_in_place(§ions, s); - } - - //- rjf: join scope vmap - ProfScope("scope vmap") - { - RDIM_BakeSectionList *s = ts_join_struct(bake_scope_vmap_ticket, max_U64, RDIM_BakeSectionList); - rdim_bake_section_list_concat_in_place(§ions, s); - } - - //- rjf: join file paths - ProfScope("file paths") - { - RDIM_BakeSectionList *s = ts_join_struct(bake_file_paths_ticket, max_U64, RDIM_BakeSectionList); - rdim_bake_section_list_concat_in_place(§ions, s); - } - - //- rjf: join strings - ProfScope("strings") - { - RDIM_BakeSectionList *s = ts_join_struct(bake_strings_ticket, max_U64, RDIM_BakeSectionList); - rdim_bake_section_list_concat_in_place(§ions, s); - } - - //- rjf: join type nodes - ProfScope("type nodes") - { - RDIM_BakeSectionList *s = ts_join_struct(bake_type_nodes_ticket, max_U64, RDIM_BakeSectionList); - rdim_bake_section_list_concat_in_place(§ions, s); - } - - //- rjf: join name maps - ProfScope("name map") - { - for(TS_TicketNode *n = bake_name_maps_tickets.first; n != 0; n = n->next) + for(EachNonZeroEnumVal(RDI_NameMapKind, k)) { - RDIM_BakeSectionList *s = ts_join_struct(n->v, max_U64, RDIM_BakeSectionList); - rdim_bake_section_list_concat_in_place(§ions, s); + name_map_bakes[k] = *ts_join_struct(bake_name_maps_tickets[k], max_U64, RDIM_NameMapBakeResult); } } - //- rjf: join index runs - ProfScope("idx runs") + ////////////////////////////// + //- rjf: join all individual name map bakes + // + ProfScope("join all name map bakes into final name map bake") { - RDIM_BakeSectionList *s = ts_join_struct(bake_idx_runs_ticket, max_U64, RDIM_BakeSectionList); - rdim_bake_section_list_concat_in_place(§ions, s); + out_results->name_maps = rdim_name_map_bake_results_combine(arena, name_map_bakes, ArrayCount(name_map_bakes)); } - //- rjf: join line tables - ProfScope("line tables") - { - RDIM_BakeSectionList *s = ts_join_struct(bake_line_tables_ticket, max_U64, RDIM_BakeSectionList); - rdim_bake_section_list_concat_in_place(§ions, s); - } - - //- rjf: fill & return scratch_end(scratch); return out; } @@ -4280,11 +4261,10 @@ p2r_bake(Arena *arena, P2R_Convert2Bake *in) //////////////////////////////// //~ rjf: Top-Level Compression Entry Point -internal P2R_Bake2Serialize * -p2r_compress(Arena *arena, P2R_Bake2Serialize *in) +internal P2R_Serialize2File * +p2r_compress(Arena *arena, P2R_Serialize2File *in) { - RDIM_BakeSectionList prepack_sections = in->sections; - RDIM_BakeSectionList postpack_sections = {0}; + P2R_Serialize2File *out = push_array(arena, P2R_Serialize2File, 1); { //- rjf: set up compression context rr_lzb_simple_context ctx = {0}; @@ -4292,18 +4272,11 @@ p2r_compress(Arena *arena, P2R_Bake2Serialize *in) ctx.m_hashTable = push_array(arena, U16, 1<next) + for(EachEnumVal(RDI_SectionKind, k)) { - RDIM_BakeSection *src = &src_n->v; - - // rjf: push new section - RDIM_BakeSection *dst = rdim_bake_section_list_push(arena, &postpack_sections); - - // rjf: unpack uncompressed section info - void *data = src->data; - RDI_SectionEncoding encoding = src->encoding; - RDI_U64 encoded_size = src->encoded_size; - RDI_U64 unpacked_size = src->unpacked_size; + RDIM_SerializedSection *src = &in->bundle.sections[k]; + RDIM_SerializedSection *dst = &out->bundle.sections[k]; + MemoryCopyStruct(dst, src); // rjf: determine if this section should be compressed B32 should_compress = 1; @@ -4312,22 +4285,12 @@ p2r_compress(Arena *arena, P2R_Bake2Serialize *in) if(should_compress) { MemoryZero(ctx.m_hashTable, sizeof(U16)*(1<data = push_array_no_zero(arena, U8, src->encoded_size); + dst->encoded_size = rr_lzb_simple_encode_veryfast(&ctx, src->data, src->encoded_size, dst->data); + dst->unpacked_size = src->encoded_size; + dst->encoding = RDI_SectionEncoding_LZB; } - - // rjf: fill - dst->data = data; - dst->encoding = encoding; - dst->encoded_size = encoded_size; - dst->unpacked_size = unpacked_size; - dst->tag = src->tag; - dst->tag_idx = src->tag_idx; } } - P2R_Bake2Serialize *out = push_array(arena, P2R_Bake2Serialize, 1); - out->sections = postpack_sections; return out; } diff --git a/src/rdi_from_pdb/rdi_from_pdb.h b/src/rdi_from_pdb/rdi_from_pdb.h index e0e40a7d..bcbf6cd3 100644 --- a/src/rdi_from_pdb/rdi_from_pdb.h +++ b/src/rdi_from_pdb/rdi_from_pdb.h @@ -55,7 +55,13 @@ struct P2R_Convert2Bake typedef struct P2R_Bake2Serialize P2R_Bake2Serialize; struct P2R_Bake2Serialize { - RDIM_SerializeParams serialize_params; + RDIM_BakeResults bake_results; +}; + +typedef struct P2R_Serialize2File P2R_Serialize2File; +struct P2R_Serialize2File +{ + RDIM_SerializedSectionBundle bundle; }; //////////////////////////////// @@ -269,7 +275,7 @@ struct P2R_SymbolStreamConvertOut typedef struct P2R_BakeLineTablesIn P2R_BakeLineTablesIn; struct P2R_BakeLineTablesIn { - RDIM_BakeParams *params; + RDIM_LineTableChunkList *line_tables; }; //- rjf: string map baking task types @@ -419,54 +425,54 @@ struct P2R_BakeSrcFilesIn { RDIM_BakeStringMapTight *strings; RDIM_BakePathTree *path_tree; - RDIM_BakeParams *params; + RDIM_SrcFileChunkList *src_files; }; typedef struct P2R_BakeUDTsIn P2R_BakeUDTsIn; struct P2R_BakeUDTsIn { RDIM_BakeStringMapTight *strings; - RDIM_BakeParams *params; + RDIM_UDTChunkList *udts; }; typedef struct P2R_BakeGlobalVariablesIn P2R_BakeGlobalVariablesIn; struct P2R_BakeGlobalVariablesIn { RDIM_BakeStringMapTight *strings; - RDIM_BakeParams *params; + RDIM_SymbolChunkList *global_variables; }; typedef struct P2R_BakeGlobalVMapIn P2R_BakeGlobalVMapIn; struct P2R_BakeGlobalVMapIn { - RDIM_BakeParams *params; + RDIM_SymbolChunkList *global_variables; }; typedef struct P2R_BakeThreadVariablesIn P2R_BakeThreadVariablesIn; struct P2R_BakeThreadVariablesIn { RDIM_BakeStringMapTight *strings; - RDIM_BakeParams *params; + RDIM_SymbolChunkList *thread_variables; }; typedef struct P2R_BakeProceduresIn P2R_BakeProceduresIn; struct P2R_BakeProceduresIn { RDIM_BakeStringMapTight *strings; - RDIM_BakeParams *params; + RDIM_SymbolChunkList *procedures; }; typedef struct P2R_BakeScopesIn P2R_BakeScopesIn; struct P2R_BakeScopesIn { RDIM_BakeStringMapTight *strings; - RDIM_BakeParams *params; + RDIM_ScopeChunkList *scopes; }; typedef struct P2R_BakeScopeVMapIn P2R_BakeScopeVMapIn; struct P2R_BakeScopeVMapIn { - RDIM_BakeParams *params; + RDIM_ScopeChunkList *scopes; }; typedef struct P2R_BakeFilePathsIn P2R_BakeFilePathsIn; @@ -487,7 +493,7 @@ struct P2R_BakeTypeNodesIn { RDIM_BakeStringMapTight *strings; RDIM_BakeIdxRunMap *idx_runs; - RDIM_BakeParams *params; + RDIM_TypeChunkList *types; }; typedef struct P2R_BakeNameMapIn P2R_BakeNameMapIn; @@ -495,9 +501,9 @@ struct P2R_BakeNameMapIn { RDIM_BakeStringMapTight *strings; RDIM_BakeIdxRunMap *idx_runs; - RDIM_BakeParams *params; - RDI_NameMapKind kind; RDIM_BakeNameMap *map; + U64 base_node_idx; + RDI_NameMapKind kind; }; typedef struct P2R_BakeIdxRunsIn P2R_BakeIdxRunsIn; @@ -628,6 +634,6 @@ internal P2R_Bake2Serialize *p2r_bake(Arena *arena, P2R_Convert2Bake *in); //////////////////////////////// //~ rjf: Top-Level Compression Entry Point -internal P2R_Bake2Serialize *p2r_compress(Arena *arena, P2R_Bake2Serialize *in); +internal P2R_Serialize2File *p2r_compress(Arena *arena, P2R_Serialize2File *in); #endif // RDI_FROM_PDB_H diff --git a/src/rdi_from_pdb/rdi_from_pdb_main.c b/src/rdi_from_pdb/rdi_from_pdb_main.c index e3c72c9a..50fd0a88 100644 --- a/src/rdi_from_pdb/rdi_from_pdb_main.c +++ b/src/rdi_from_pdb/rdi_from_pdb_main.c @@ -98,22 +98,31 @@ entry_point(CmdLine *cmdline) bake2srlz = p2r_bake(arena, convert2bake); } + //- rjf: serialize + P2R_Serialize2File *srlz2file = 0; + ProfScope("serialize") + { + srlz2file = push_array(arena, P2R_Serialize2File, 1); + srlz2file->bundle = rdim_serialized_section_bundle_from_bake_results(&bake2srlz->bake_results); + } + //- rjf: compress - P2R_Bake2Serialize *bake2srlz_compressed = bake2srlz; + P2R_Serialize2File *srlz2file_compressed = srlz2file; if(cmd_line_has_flag(cmdline, str8_lit("compress"))) ProfScope("compress") { - bake2srlz_compressed = p2r_compress(arena, bake2srlz); + srlz2file_compressed = push_array(arena, P2R_Serialize2File, 1); + srlz2file_compressed = p2r_compress(arena, srlz2file); } //- rjf: serialize - String8List serialize_out = rdim_serialized_strings_from_params_bake_section_list(arena, &convert2bake->bake_params, &bake2srlz_compressed->sections); + String8List blobs = rdim_file_blobs_from_section_bundle(arena, &srlz2file_compressed->bundle); //- rjf: write ProfScope("write") { OS_Handle output_file = os_file_open(OS_AccessFlag_Read|OS_AccessFlag_Write, user2convert->output_name); U64 off = 0; - for(String8Node *n = serialize_out.first; n != 0; n = n->next) + for(String8Node *n = blobs.first; n != 0; n = n->next) { os_file_write(output_file, r1u64(off, off+n->string.size), n->string.str); off += n->string.size;