From 79bf3d86395ec9ae17ea747195e1d664819568f4 Mon Sep 17 00:00:00 2001 From: Ryan Fleury Date: Mon, 9 Jun 2025 15:40:48 -0700 Subject: [PATCH] eliminate standalone programs for pdb -> rdi; dwarf -> rdi; pdb -> breakpad; sketch out 'radbin', unified cli frontend for dumping/converting/etc., plug in pdb -> rdi, pdb -> breakpad, & stub out dumper/dwarf --- .github/workflows/builds.yml | 4 +- build.bat | 12 +- project.4coder | 5 +- src/async/async.c | 33 + src/async/async.h | 16 + src/base/base_entry_point.c | 3 - src/base/base_log.h | 4 +- src/dbgi/dbgi.c | 3 +- src/lib_rdi_make/rdi_make.h | 40 + src/radbin/generated/radbin.meta.c | 21 + src/radbin/generated/radbin.meta.h | 28 + src/radbin/radbin.c | 686 +++++++++++++ src/radbin/radbin.h | 17 + src/radbin/radbin.mdesk | 28 + src/radbin/radbin_main.c | 70 ++ src/radcon/radcon_pdb.c | 2 +- src/radcon/radcon_pdb.h | 2 +- src/raddbg/raddbg_main.c | 75 +- src/raddump/raddump_main.c | 1 - .../rdi_breakpad_from_pdb.c | 106 ++ .../rdi_breakpad_from_pdb.h | 47 + .../rdi_breakpad_from_pdb_main.c | 331 ------- src/rdi_format/rdi_format.mdesk | 8 +- src/rdi_from_dwarf/rdi_from_dwarf_main.c | 121 --- src/rdi_from_pdb/rdi_from_pdb.c | 137 +-- src/rdi_from_pdb/rdi_from_pdb.h | 89 +- src/rdi_from_pdb/rdi_from_pdb_main.c | 136 --- src/rdi_make/rdi_make_help.c | 927 ------------------ src/rdi_make/rdi_make_help.h | 342 ------- src/rdi_make/rdi_make_local.c | 113 +-- src/rdi_make/rdi_make_local.h | 27 +- src/tester/tester_main.c | 8 +- 32 files changed, 1241 insertions(+), 2201 deletions(-) create mode 100644 src/radbin/generated/radbin.meta.c create mode 100644 src/radbin/generated/radbin.meta.h create mode 100644 src/radbin/radbin.c create mode 100644 src/radbin/radbin.h create mode 100644 src/radbin/radbin.mdesk create mode 100644 src/radbin/radbin_main.c create mode 100644 src/rdi_breakpad_from_pdb/rdi_breakpad_from_pdb.c create mode 100644 src/rdi_breakpad_from_pdb/rdi_breakpad_from_pdb.h delete mode 100644 src/rdi_breakpad_from_pdb/rdi_breakpad_from_pdb_main.c delete mode 100644 src/rdi_from_dwarf/rdi_from_dwarf_main.c delete mode 100644 src/rdi_from_pdb/rdi_from_pdb_main.c delete mode 100644 src/rdi_make/rdi_make_help.c delete mode 100644 src/rdi_make/rdi_make_help.h diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index b2de41ba..2c30d5fc 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -17,9 +17,7 @@ jobs: target: - raddbg - radlink - - rdi_from_pdb - - raddump - - rdi_breakpad_from_pdb + - radbin compiler: - msvc - clang diff --git a/build.bat b/build.bat index b1fbb145..e608d679 100644 --- a/build.bat +++ b/build.bat @@ -46,16 +46,16 @@ if "%opengl%"=="1" set auto_compile_flags=%auto_compile_flags% -DR_BACKEND=R_ :: --- Compile/Link Line Definitions ------------------------------------------ set cl_common= /I..\src\ /I..\local\ /nologo /FC /Z7 -set clang_common= -I..\src\ -I..\local\ -gcodeview -fdiagnostics-absolute-paths -Wall -Wno-unknown-warning-option -Wno-missing-braces -Wno-unused-function -Wno-unused-parameter -Wno-writable-strings -Wno-missing-field-initializers -Wno-unused-value -Wno-unused-variable -Wno-unused-local-typedef -Wno-deprecated-register -Wno-deprecated-declarations -Wno-unused-but-set-variable -Wno-single-bit-bitfield-constant-conversion -Wno-compare-distinct-pointer-types -Wno-initializer-overrides -Wno-incompatible-pointer-types-discards-qualifiers -Xclang -flto-visibility-public-std -D_USE_MATH_DEFINES -Dstrdup=_strdup -Dgnu_printf=printf -ferror-limit=10000 set cl_debug= call cl /Od /Ob1 /DBUILD_DEBUG=1 %cl_common% %auto_compile_flags% set cl_release= call cl /O2 /DBUILD_DEBUG=0 %cl_common% %auto_compile_flags% +set cl_link= /link /MANIFEST:EMBED /INCREMENTAL:NO /pdbaltpath:%%%%_PDB%%%% /NATVIS:"%~dp0\src\natvis\base.natvis" /noexp +set cl_out= /out: +set cl_linker= +set clang_common= -I..\src\ -I..\local\ -gcodeview -fdiagnostics-absolute-paths -Wall -Wno-unknown-warning-option -Wno-missing-braces -Wno-unused-function -Wno-unused-parameter -Wno-writable-strings -Wno-missing-field-initializers -Wno-unused-value -Wno-unused-variable -Wno-unused-local-typedef -Wno-deprecated-register -Wno-deprecated-declarations -Wno-unused-but-set-variable -Wno-single-bit-bitfield-constant-conversion -Wno-compare-distinct-pointer-types -Wno-initializer-overrides -Wno-incompatible-pointer-types-discards-qualifiers -Xclang -flto-visibility-public-std -D_USE_MATH_DEFINES -Dstrdup=_strdup -Dgnu_printf=printf -ferror-limit=10000 set clang_debug= call clang -g -O0 -DBUILD_DEBUG=1 %clang_common% %auto_compile_flags% set clang_release= call clang -g -O2 -DBUILD_DEBUG=0 %clang_common% %auto_compile_flags% -set cl_link= /link /MANIFEST:EMBED /INCREMENTAL:NO /pdbaltpath:%%%%_PDB%%%% /NATVIS:"%~dp0\src\natvis\base.natvis" /noexp set clang_link= -fuse-ld=lld -Xlinker /MANIFEST:EMBED -Xlinker /pdbaltpath:%%%%_PDB%%%% -Xlinker /NATVIS:"%~dp0\src\natvis\base.natvis" -set cl_out= /out: set clang_out= -o -set cl_linker= set clang_linker= -Xlinker :: --- Per-Build Settings ----------------------------------------------------- @@ -110,11 +110,9 @@ if not "%no_meta%"=="1" ( pushd build if "%raddbg%"=="1" set didbuild=1 && %compile% ..\src\raddbg\raddbg_main.c %compile_link% %link_icon% %out%raddbg.exe || exit /b 1 if "%radlink%"=="1" set didbuild=1 && %compile% ..\src\linker\lnk.c %compile_link% %linker% /NOIMPLIB %linker% /NATVIS:"%~dp0\src\linker\linker.natvis" %out%radlink.exe || exit /b 1 +if "%radbin%"=="1" set didbuild=1 && %compile% ..\src\radbin\radbin_main.c %compile_link% %out%radbin.exe || exit /b 1 if "%radcon%"=="1" set didbuild=1 && %compile% ..\src\radcon\radcon_main.c %compile_link% %out%radcon.exe || exit /b 1 if "%raddump%"=="1" set didbuild=1 && %compile% ..\src\raddump\raddump_main.c %compile_link% %out%raddump.exe || exit /b 1 -if "%rdi_from_pdb%"=="1" set didbuild=1 && %compile% ..\src\rdi_from_pdb\rdi_from_pdb_main.c %compile_link% %out%rdi_from_pdb.exe || exit /b 1 -if "%rdi_from_dwarf%"=="1" set didbuild=1 && %compile% ..\src\rdi_from_dwarf\rdi_from_dwarf_main.c %compile_link% %out%rdi_from_dwarf.exe || exit /b 1 -if "%rdi_breakpad_from_pdb%"=="1" set didbuild=1 && %compile% ..\src\rdi_breakpad_from_pdb\rdi_breakpad_from_pdb_main.c %compile_link% %out%rdi_breakpad_from_pdb.exe || exit /b 1 if "%tester%"=="1" set didbuild=1 && %compile% ..\src\tester\tester_main.c %compile_link% %out%tester.exe || exit /b 1 if "%ryan_scratch%"=="1" set didbuild=1 && %compile% ..\src\scratch\ryan_scratch.c %compile_link% %out%ryan_scratch.exe || exit /b 1 if "%eval_scratch%"=="1" set didbuild=1 && %compile% ..\src\scratch\eval_scratch.c %compile_link% %out%eval_scratch.exe || exit /b 1 diff --git a/project.4coder b/project.4coder index 32a1e737..da1093ae 100644 --- a/project.4coder +++ b/project.4coder @@ -46,7 +46,7 @@ load_paths = commands = { //- rjf: [raddbg] - .f1 = { .win = "raddbg_stable --ipc kill_all && build raddbg telemetry", .linux = "", .out = "*compilation*", .footer_panel = true, .save_dirty_files = true, .cursor_at_end = false, }, + // .f1 = { .win = "raddbg_stable --ipc kill_all && build raddbg telemetry", .linux = "", .out = "*compilation*", .footer_panel = true, .save_dirty_files = true, .cursor_at_end = false, }, //- rjf: [textperf] // .f1 = { .win = "raddbg_stable --ipc kill_all && build no_meta telemetry textperf && raddbg_stable --ipc bring_to_front && raddbg_stable --ipc run", .linux = "", .out = "*compilation*", .footer_panel = true, .save_dirty_files = true, .cursor_at_end = false, }, @@ -54,6 +54,9 @@ commands = //- rjf: [tester] // .f1 = { .win = "raddbg_stable --ipc kill_all && build no_meta tester", .linux = "", .out = "*compilation*", .footer_panel = true, .save_dirty_files = true, .cursor_at_end = false, }, + //- rjf: [radbin] + .f1 = { .win = "raddbg_stable --ipc kill_all && build radbin", .linux = "", .out = "*compilation*", .footer_panel = true, .save_dirty_files = true, .cursor_at_end = false, }, + //- rjf: running target .f3 = { .win = "raddbg_stable --ipc run", .linux = "", .out = "*compilation*", .footer_panel = true, .save_dirty_files = true, .cursor_at_end = false, }, // .f3 = { .win = "pushd build && raddbg --user:dev.raddbg_user && popd", .linux = "", .out = "*compilation*", .footer_panel = true, .save_dirty_files = true, .cursor_at_end = false, }, diff --git a/src/async/async.c b/src/async/async.c index c9af67f1..85ff743b 100644 --- a/src/async/async.c +++ b/src/async/async.c @@ -234,6 +234,39 @@ async_execute_work(ASYNC_Work work) } } +//////////////////////////////// +//~ rjf: Root Allocation/Deallocation + +internal ASYNC_Root * +async_root_alloc(void) +{ + Arena *arena = arena_alloc(); + ASYNC_Root *root = push_array(arena, ASYNC_Root, 1); + root->arenas = push_array(arena, Arena *, async_thread_count()); + root->arenas[0] = arena; + for(U64 idx = 1; idx < async_thread_count(); idx += 1) + { + root->arenas[idx] = arena_alloc(); + } + return root; +} + +internal void +async_root_release(ASYNC_Root *root) +{ + for(U64 idx = 1; idx < async_thread_count(); idx += 1) + { + arena_release(root->arenas[idx]); + } + arena_release(root->arenas[0]); +} + +internal Arena * +async_root_thread_arena(ASYNC_Root *root) +{ + return root->arenas[async_work_thread_idx]; +} + //////////////////////////////// //~ rjf: Work Thread Entry Point diff --git a/src/async/async.h b/src/async/async.h index f2f31d2a..a25492cb 100644 --- a/src/async/async.h +++ b/src/async/async.h @@ -70,6 +70,15 @@ struct ASYNC_TaskList U64 count; }; +//////////////////////////////// +//~ rjf: Root (Per-Worker-Thread Arena Bundle) + +typedef struct ASYNC_Root ASYNC_Root; +struct ASYNC_Root +{ + Arena **arenas; +}; + //////////////////////////////// //~ rjf: Shared State Bundle @@ -138,6 +147,13 @@ internal void *async_task_join(ASYNC_Task *task); internal ASYNC_Work async_pop_work(void); internal void async_execute_work(ASYNC_Work work); +//////////////////////////////// +//~ rjf: Root Allocation/Deallocation + +internal ASYNC_Root *async_root_alloc(void); +internal void async_root_release(ASYNC_Root *root); +internal Arena *async_root_thread_arena(ASYNC_Root *root); + //////////////////////////////// //~ rjf: Work Thread Entry Point diff --git a/src/base/base_entry_point.c b/src/base/base_entry_point.c index bd121af7..feaa45e1 100644 --- a/src/base/base_entry_point.c +++ b/src/base/base_entry_point.c @@ -41,9 +41,6 @@ main_thread_base_entry_point(int arguments_count, char **arguments) #if defined(ASYNC_H) && !defined(ASYNC_INIT_MANUAL) async_init(&cmdline); #endif -#if defined(RDI_FROM_PDB_H) && !defined(P2R_INIT_MANUAL) - p2r_init(); -#endif #if defined(HASH_STORE_H) && !defined(HS_INIT_MANUAL) hs_init(); #endif diff --git a/src/base/base_log.h b/src/base/base_log.h index e8d3d50b..1cc58ff0 100644 --- a/src/base/base_log.h +++ b/src/base/base_log.h @@ -48,9 +48,9 @@ internal void log_select(Log *log); internal void log_msg(LogMsgKind kind, String8 string); internal void log_msgf(LogMsgKind kind, char *fmt, ...); -#define log_info(s) log_msg(LogMsgKind_Info, (s)) +#define log_info(s) log_msg(LogMsgKind_Info, (s)) #define log_infof(...) log_msgf(LogMsgKind_Info, __VA_ARGS__) -#define log_user_error(s) log_msg(LogMsgKind_UserError, (s)) +#define log_user_error(s) log_msg(LogMsgKind_UserError, (s)) #define log_user_errorf(...) log_msgf(LogMsgKind_UserError, __VA_ARGS__) #define LogInfoNamedBlock(s) DeferLoop(log_infof("%S:\n{\n", (s)), log_infof("}\n")) diff --git a/src/dbgi/dbgi.c b/src/dbgi/dbgi.c index 941449bc..ef92f3ba 100644 --- a/src/dbgi/dbgi.c +++ b/src/dbgi/dbgi.c @@ -1033,8 +1033,9 @@ ASYNC_WORK_DEF(di_parse_work) str8_list_pushf(scratch.arena, ¶ms.cmd_line, "--compress"); } // str8_list_pushf(scratch.arena, ¶ms.cmd_line, "--capture"); - str8_list_pushf(scratch.arena, ¶ms.cmd_line, "--pdb:%S", og_path); + str8_list_pushf(scratch.arena, ¶ms.cmd_line, "--rdi"); str8_list_pushf(scratch.arena, ¶ms.cmd_line, "--out:%S", rdi_path); + str8_list_pushf(scratch.arena, ¶ms.cmd_line, "%S", og_path); process = os_process_launch(¶ms); } diff --git a/src/lib_rdi_make/rdi_make.h b/src/lib_rdi_make/rdi_make.h index 64f8c243..743aaff7 100644 --- a/src/lib_rdi_make/rdi_make.h +++ b/src/lib_rdi_make/rdi_make.h @@ -309,6 +309,46 @@ RDIM_CheckNil(nil,p) ? \ #endif #define rdim_noop ((void)0) +//////////////////////////////// +//~ rjf: RDI Subsets + +#define RDIM_Subset_XList \ +X(BinarySections, binary_sections)\ +X(Units, units)\ +X(Procedures, procedures)\ +X(GlobalVariables, global_variables)\ +X(ThreadVariables, thread_variables)\ +X(Scopes, scopes)\ +X(Locals, locals)\ +X(Types, types)\ +X(UDTs, udts)\ +X(LineInfo, line_info)\ +X(InlineLineInfo, inline_line_info)\ +X(GlobalVariableNameMap, global_variable_name_map)\ +X(ThreadVariableNameMap, thread_variable_name_map)\ +X(ProcedureNameMap, procedure_name_map)\ +X(ConstantNameMap, constant_name_map)\ +X(TypeNameMap, type_name_map)\ +X(LinkNameProcedureNameMap, link_name_procedure_name_map)\ +X(NormalSourcePathNameMap, normal_source_path_name_map)\ + +typedef enum RDIM_Subset +{ +#define X(name, name_lower) RDIM_Subset_##name, + RDIM_Subset_XList +#undef X +} +RDIM_Subset; + +typedef U32 RDIM_SubsetFlags; +enum +{ +#define X(name, name_lower) RDIM_SubsetFlag_##name = (1<inputs.first; n != 0; n = n->next) + { + OS_Handle file = os_file_open(OS_AccessFlag_Read, n->string); + RB_FileFormat file_format = RB_FileFormat_Null; + + //- rjf: PDB magic -> PDB input + if(file_format == RB_FileFormat_Null) + { + U8 msf20_magic_maybe[sizeof(msf_msf20_magic)] = {0}; + os_file_read(file, r1u64(0, sizeof(msf20_magic_maybe)), msf20_magic_maybe); + if(MemoryMatch(msf20_magic_maybe, msf_msf20_magic, sizeof(msf20_magic_maybe))) + { + file_format = RB_FileFormat_PDB; + } + } + if(file_format == RB_FileFormat_Null) + { + U8 msf70_magic_maybe[sizeof(msf_msf70_magic)] = {0}; + os_file_read(file, r1u64(0, sizeof(msf70_magic_maybe)), msf70_magic_maybe); + if(MemoryMatch(msf70_magic_maybe, msf_msf70_magic, sizeof(msf70_magic_maybe))) + { + file_format = RB_FileFormat_PDB; + } + } + + //- rjf: PE magic -> PE input + if(file_format == RB_FileFormat_Null) + { + PE_DosHeader dos_header_maybe = {0}; + os_file_read_struct(file, 0, &dos_header_maybe); + if(dos_header_maybe.magic == PE_DOS_MAGIC) + { + U32 pe_magic_maybe = 0; + os_file_read_struct(file, dos_header_maybe.coff_file_offset, &pe_magic_maybe); + if(pe_magic_maybe == PE_MAGIC) + { + file_format = RB_FileFormat_PE; + } + } + } + + //- rjf: COFF archive magic -> COFF archive input + if(file_format == RB_FileFormat_Null) + { + U8 coff_archive_sig_maybe[sizeof(g_coff_archive_sig)] = {0}; + os_file_read(file, r1u64(0, sizeof(coff_archive_sig_maybe)), coff_archive_sig_maybe); + if(MemoryMatch(coff_archive_sig_maybe, g_coff_archive_sig, sizeof(g_coff_archive_sig))) + { + file_format = RB_FileFormat_COFF_Archive; + } + } + if(file_format == RB_FileFormat_Null) + { + U8 coff_thin_archive_sig_maybe[sizeof(g_coff_thin_archive_sig)] = {0}; + os_file_read(file, r1u64(0, sizeof(coff_thin_archive_sig_maybe)), coff_thin_archive_sig_maybe); + if(MemoryMatch(coff_thin_archive_sig_maybe, g_coff_thin_archive_sig, sizeof(g_coff_thin_archive_sig))) + { + file_format = RB_FileFormat_COFF_ThinArchive; + } + } + + //- rjf: COFF obj magic -> COFF obj input + if(file_format == RB_FileFormat_Null) + { + COFF_BigObjHeader header_maybe = {0}; + os_file_read_struct(file, 0, &header_maybe); + if(header_maybe.sig1 == COFF_MachineType_Unknown && + header_maybe.sig2 == max_U16 && + header_maybe.version >= 2 && + MemoryMatch(header_maybe.magic, g_coff_big_header_magic, sizeof(header_maybe.magic))) + { + file_format = RB_FileFormat_COFF_BigOBJ; + } + } + if(file_format == RB_FileFormat_Null) + { + Temp scratch = scratch_begin(&arena, 1); + COFF_FileHeader header_maybe = {0}; + os_file_read_struct(file, 0, &header_maybe); + U64 section_count = header_maybe.section_count; + U64 section_hdr_opl_off = sizeof(header_maybe) + section_count*sizeof(COFF_SectionHeader); + FileProperties props = os_properties_from_file(file); + + // rjf: check if machine type is valid + B32 machine_type_is_valid = 0; + switch(header_maybe.machine) + { + case COFF_MachineType_Unknown: + case COFF_MachineType_X86: case COFF_MachineType_X64: + case COFF_MachineType_Am33: case COFF_MachineType_Arm: + case COFF_MachineType_Arm64: case COFF_MachineType_ArmNt: + case COFF_MachineType_Ebc: case COFF_MachineType_Ia64: + case COFF_MachineType_M32R: case COFF_MachineType_Mips16: + case COFF_MachineType_MipsFpu:case COFF_MachineType_MipsFpu16: + case COFF_MachineType_PowerPc:case COFF_MachineType_PowerPcFp: + case COFF_MachineType_R4000: case COFF_MachineType_RiscV32: + case COFF_MachineType_RiscV64:case COFF_MachineType_RiscV128: + case COFF_MachineType_Sh3: case COFF_MachineType_Sh3Dsp: + case COFF_MachineType_Sh4: case COFF_MachineType_Sh5: + case COFF_MachineType_Thumb: case COFF_MachineType_WceMipsV2: + { + machine_type_is_valid = 1; + }break; + } + + // rjf: check if sections are valid + B32 sections_are_valid = 0; + if(machine_type_is_valid) + { + if(props.size >= section_hdr_opl_off) + { + COFF_SectionHeader *section_hdrs = push_array(scratch.arena, COFF_SectionHeader, section_count); + os_file_read(file, r1u64(sizeof(header_maybe), sizeof(header_maybe) + section_count*sizeof(COFF_SectionHeader)), section_hdrs); + B32 section_ranges_valid = 1; + for EachIndex(section_hdr_idx, section_count) + { + COFF_SectionHeader *hdr = §ion_hdrs[section_hdr_idx]; + if(!(hdr->flags & COFF_SectionFlag_CntUninitializedData)) + { + U64 min = hdr->foff; + U64 max = min + hdr->fsize; + if(hdr->fsize > 0 && !(section_hdr_opl_off <= min && min <= max && max <= props.size)) + { + section_ranges_valid = 0; + break; + } + } + } + sections_are_valid = section_ranges_valid; + } + } + + // rjf: check if symbol table is valid + B32 symbol_table_is_valid = 0; + if(sections_are_valid) + { + U64 symbol_table_off = header_maybe.symbol_table_foff; + U64 symbol_table_size = sizeof(COFF_Symbol16)*header_maybe.symbol_count; + U64 symbol_table_opl_off = symbol_table_off+symbol_table_size; + if(symbol_table_off == 0 && symbol_table_size == 0) + { + symbol_table_off = section_hdr_opl_off; + symbol_table_opl_off = section_hdr_opl_off; + } + symbol_table_is_valid = (section_hdr_opl_off <= symbol_table_off && + symbol_table_off <= symbol_table_opl_off && + symbol_table_opl_off <= props.size); + } + + // rjf: symbol table is valid -> is COFF OBJ + if(symbol_table_is_valid) + { + file_format = RB_FileFormat_COFF_OBJ; + } + + scratch_end(scratch); + } + + //- rjf: ELF magic -> ELF input + if(file_format == RB_FileFormat_Null) + { + U8 identifier_maybe[ELF_Identifier_Max] = {0}; + os_file_read(file, r1u64(0, sizeof(identifier_maybe)), identifier_maybe); + B32 is_elf_magic = (identifier_maybe[ELF_Identifier_Mag0] == 0x7f && + identifier_maybe[ELF_Identifier_Mag1] == 'E' && + identifier_maybe[ELF_Identifier_Mag2] == 'L' && + identifier_maybe[ELF_Identifier_Mag3] == 'F'); + if(is_elf_magic) + { + file_format = RB_FileFormat_ELF; + } + } + + //- rjf: RDI magic -> RDI input + if(file_format == RB_FileFormat_Null) + { + RDI_Header rdi_header_maybe = {0}; + os_file_read_struct(file, 0, &rdi_header_maybe); + if(rdi_header_maybe.magic == RDI_MAGIC_CONSTANT) + { + file_format = RB_FileFormat_RDI; + } + } + + //- rjf: log file recognition + if(file_format != RB_FileFormat_Null) + { + log_infof("%S recognized as %S\n", n->string, rb_file_format_display_name_table[file_format]); + } + else + { + log_infof("%S was not recognized as a supported format.\n", n->string); + } + + //- rjf: push to list + File *file_n = push_array(arena, File, 1); + file_n->path = n->string; + file_n->format = file_format; + SLLQueuePush(first_input_file, last_input_file, file_n); + + os_file_close(file); + } + + ////////////////////////////// + //- rjf: bucket input files by format + // + String8List input_paths_from_format_table[RB_FileFormat_COUNT] = {0}; + for(File *f = first_input_file; f != 0; f = f->next) + { + str8_list_push(arena, &input_paths_from_format_table[f->format], f->path); + } + + ////////////////////////////// + //- rjf: unpack which kind of output we're producing, and to where + // + typedef enum OutputKind + { + OutputKind_Null, + OutputKind_RDI, + OutputKind_Dump, + OutputKind_Breakpad, + OutputKind_COUNT + } + OutputKind; + struct + { + String8 arg; + String8 title; + } + output_kind_info[] = + { + {str8_lit_comp(""), str8_lit_comp("")}, + {str8_lit_comp("rdi"), str8_lit_comp("RAD Debug Info (.rdi) Conversion")}, + {str8_lit_comp("dump"), str8_lit_comp("Textual Dumping")}, + {str8_lit_comp("breakpad"), str8_lit_comp("Breakpad Debug Info Conversion")}, + }; + OutputKind output_kind = OutputKind_Null; + String8 output_path = cmd_line_string(cmdline, str8_lit("out")); + { + //- rjf: user manually specified output kind + if(output_kind == OutputKind_Null) + { + for EachNonZeroEnumVal(OutputKind, k) + { + if(cmd_line_has_flag(cmdline, output_kind_info[k].arg)) + { + log_infof("User specified --%S; performing `%S`\n", output_kind_info[k].arg, output_kind_info[k].title); + output_kind = k; + break; + } + } + } + + //- rjf: we can infer from the user-specified output path + else if(str8_match(str8_skip_last_dot(output_path), str8_lit("rdi"), StringMatchFlag_CaseInsensitive)) + { + output_kind = OutputKind_RDI; + log_infof("Output path has .rdi extension; performing `%S`\n", output_kind_info[output_kind].title); + } + else if(str8_match(str8_skip_last_dot(output_path), str8_lit("dump"), StringMatchFlag_CaseInsensitive) || + str8_match(str8_skip_last_dot(output_path), str8_lit("txt"), StringMatchFlag_CaseInsensitive)) + { + output_kind = OutputKind_Dump; + log_infof("Output path has .dump or .txt extension; performing `%S`\n", output_kind_info[output_kind].title); + } + else if(str8_match(str8_skip_last_dot(output_path), str8_lit("psym"), StringMatchFlag_CaseInsensitive) || + str8_match(str8_skip_last_dot(output_path), str8_lit("psyms"), StringMatchFlag_CaseInsensitive)) + { + output_kind = OutputKind_Breakpad; + log_infof("Output path has .psym or .psyms extension; performing `%S`\n", output_kind_info[output_kind].title); + } + } + + ////////////////////////////// + //- rjf: print help preamble + // + if(output_kind == OutputKind_Null || cmdline->inputs.node_count == 0) + { + fprintf(stderr, "-------------------------------------------------------------------------------\n"); + fprintf(stderr, "%s\n\n", BUILD_TITLE); + if(output_kind != OutputKind_Null) + { + fprintf(stderr, "%.*s Help\n", str8_varg(output_kind_info[output_kind].title)); + fprintf(stderr, "To see top-level options for radbin, run the binary with no arguments.\n\n"); + } + } + + ////////////////////////////// + //- rjf: perform operation based on output kind + // + String8List output_blobs = {0}; + switch(output_kind) + { + //////////////////////////// + //- rjf: no output kind -> nothing to do. output help information to user + // + default: + case OutputKind_Null: + { + fprintf(stderr, "This utility provides a number of operations relating to executable image and\n"); + fprintf(stderr, "debug information formats. It can convert debug information to the RAD Debug\n"); + fprintf(stderr, "Info (.rdi) format. It can also parse and dump textualized contents of several\n"); + fprintf(stderr, "formats, including PDB, PE, ELF, and RDI. It also supports converting debug\n"); + fprintf(stderr, "information to the textual Breakpad format.\n\n"); + + fprintf(stderr, "The following top-level arguments are accepted:\n\n"); + + fprintf(stderr, "--rdi Specifies that the utility should convert debug information\n"); + fprintf(stderr, " data to the RAD Debug Info (.rdi) format.\n\n"); + + fprintf(stderr, "--dump Specifies that the utility should dump textualized contents of\n"); + fprintf(stderr, " all input files.\n\n"); + + fprintf(stderr, "--breakpad Specifies that the utility should convert debug information\n"); + fprintf(stderr, " data to the textual Breakpad format.\n\n"); + + fprintf(stderr, "--out: Specifies the path to which output data should be written. If\n"); + fprintf(stderr, " not specified, the utility will choose a fallback. If dumping\n"); + fprintf(stderr, " textual contents, the utility will write to `stdout`. If\n"); + fprintf(stderr, " converting to another format, the utility will form a path by\n"); + fprintf(stderr, " changing the extension of input files accordingly.\n\n"); + + fprintf(stderr, "--deterministic Turns off all sources of non-determinism in generated output,\n"); + fprintf(stderr, " like build names, versions, and dates.\n\n"); + + fprintf(stderr, "--verbose Outputs all log information collected during execution.\n\n"); + + fprintf(stderr, "There are also operation-specific arguments. To see them, run this binary with\n"); + fprintf(stderr, "the operation selected, with no additional inputs (e.g. `radbin --rdi`).\n\n"); + }break; + + //////////////////////////// + //- rjf: RDI -> conversion based on inputs + // + case OutputKind_RDI: + { + //- rjf: unpack subset flags + RDIM_SubsetFlags subset_flags = 0xffffffff; + { + String8List only_names = cmd_line_strings(cmdline, str8_lit("only")); + if(only_names.node_count != 0) + { + subset_flags = 0; + } + for(String8Node *n = only_names.first; n != 0; n = n->next) + { + if(0){} +#define X(name, name_lower) else if(str8_match(n->string, str8_lit(#name_lower), 0)) { subset_flags |= RDIM_SubsetFlag_##name; } + RDIM_Subset_XList +#undef X + } + String8List omit_names = cmd_line_strings(cmdline, str8_lit("omit")); + for(String8Node *n = omit_names.first; n != 0; n = n->next) + { + if(0){} +#define X(name, name_lower) else if(str8_match(n->string, str8_lit(#name_lower), 0)) { subset_flags &= ~RDIM_SubsetFlag_##name; } + RDIM_Subset_XList +#undef X + } + } + + //- rjf: no inputs => help + if(cmdline->inputs.node_count == 0) + { + fprintf(stderr, "The following arguments are accepted:\n\n"); + + fprintf(stderr, "--compress Compresses the RDI file's contents.\n"); + fprintf(stderr, "\n"); + fprintf(stderr, "--only: Specifies that only the named subsets of debug\n"); + fprintf(stderr, " information should be generated. See below for\n"); + fprintf(stderr, " a list of valid debug info subset names.\n"); + fprintf(stderr, "\n"); + fprintf(stderr, "--omit: Specifies that the named subsets of debug\n"); + fprintf(stderr, " information should not be generated. See below\n"); + fprintf(stderr, " for a list of valid debug info subset names.\n"); + fprintf(stderr, "\n"); + + fprintf(stderr, "RAD Debug Info Subsets:\n"); + fprintf(stderr, " - binary_sections Sections in the executable image\n"); + fprintf(stderr, " - units Compilation unit info\n"); + fprintf(stderr, " - procedures Procedure info\n"); + fprintf(stderr, " - global_variables Global variable info\n"); + fprintf(stderr, " - thread_variables Thread-local variable info\n"); + fprintf(stderr, " - scopes Scope info\n"); + fprintf(stderr, " - locals Local variable info\n"); + fprintf(stderr, " - types Type nodes\n"); + fprintf(stderr, " - udts User-defined-type (UDT) info\n"); + fprintf(stderr, " - line_info Source code line <-> virtual offset mapping\n"); + fprintf(stderr, " - global_variable_name_map The name -> global variable table\n"); + fprintf(stderr, " - thread_variable_name_map The name -> thread variable table\n"); + fprintf(stderr, " - procedure_name_map The name -> procedure table\n"); + fprintf(stderr, " - constant_name_map The name -> constant table\n"); + fprintf(stderr, " - type_name_map The name -> user-defined-type table\n"); + fprintf(stderr, " - link_name_procedure_name_map The link_name -> procedure table\n"); + fprintf(stderr, " - normal_source_path_name_map The path -> source file table\n"); + } + + //- rjf: PDB inputs => PDB -> RDI conversion + else if(input_paths_from_format_table[RB_FileFormat_PDB].node_count != 0) + { + log_infof("PDBs specified; producing RDI by converting PDB data\n"); + + // rjf: convert + P2R_ConvertParams convert_params = {0}; + { + convert_params.input_pdb_name = str8_list_first(&input_paths_from_format_table[RB_FileFormat_PDB]); + convert_params.input_exe_name = str8_list_first(&input_paths_from_format_table[RB_FileFormat_PE]); + convert_params.input_pdb_data = os_data_from_file_path(arena, convert_params.input_pdb_name); + convert_params.input_exe_data = os_data_from_file_path(arena, convert_params.input_exe_name); + convert_params.subset_flags = subset_flags; + convert_params.deterministic = cmd_line_has_flag(cmdline, str8_lit("deterministic")); + } + RDIM_BakeParams bake_params = {0}; + ProfScope("convert") bake_params = p2r_convert(arena, async_root, &convert_params); + + // rjf: bake + RDIM_BakeResults bake_results = {0}; + ProfScope("bake") bake_results = rdim_bake(arena, async_root, &bake_params); + + // rjf: serialize + RDIM_SerializedSectionBundle serialized_section_bundle = {0}; + ProfScope("serialize") serialized_section_bundle = rdim_serialized_section_bundle_from_bake_results(&bake_results); + + // rjf: compress + RDIM_SerializedSectionBundle serialized_section_bundle__compressed = serialized_section_bundle; + if(cmd_line_has_flag(cmdline, str8_lit("compress"))) ProfScope("compress") + { + serialized_section_bundle__compressed = rdim_compress(arena, &serialized_section_bundle); + } + + // rjf: serialize + String8List blobs = rdim_file_blobs_from_section_bundle(arena, &serialized_section_bundle__compressed); + str8_list_concat_in_place(&output_blobs, &blobs); + + // rjf: no output path? -> pick one based on PDB + if(output_path.size == 0) + { + output_path = push_str8f(arena, "%S.rdi", str8_chop_last_dot(convert_params.input_pdb_name)); + } + } + + //- rjf: no viable input paths + else + { + log_user_errorf("Could not create an RDI file from the specified inputs. You must provide either a valid PDB file or an executable image (PE, ELF) file with DWARF information."); + } + }break; + + //////////////////////////// + //- rjf: dump -> textual dump of inputs + // + case OutputKind_Dump: + { + //- rjf: no inputs => help + if(cmdline->inputs.node_count == 0) + { + fprintf(stderr, "All input files specified on the command line will be dumped. The following\n"); + fprintf(stderr, "formats are currently supported: PE, COFF, RDI, and ELF\n\n"); + } + + //- rjf: dump input files in order + String8List dump = {0}; + for(File *f = first_input_file; f != 0; f = f->next) + { + + } + + //- rjf: join with output + str8_list_concat_in_place(&output_blobs, &dump); + }break; + + //////////////////////////// + //- rjf: breakpad -> conversion based on inputs + // + case OutputKind_Breakpad: + { + //- rjf: no inputs => help + if(cmdline->inputs.node_count == 0) + { + fprintf(stderr, "Pass a path to a PDB file, and optionally its associated PE file, for which\n"); + fprintf(stderr, "a Breakpad file should be generated.\n"); + } + + //- rjf: PDB inputs => PDB -> Breakpad conversion + else if(input_paths_from_format_table[RB_FileFormat_PDB].node_count != 0) + { + log_infof("PDBs specified; producing Breakpad by converting PDB data\n"); + + // rjf: convert + P2R_ConvertParams convert_params = {0}; + { + convert_params.input_pdb_name = str8_list_first(&input_paths_from_format_table[RB_FileFormat_PDB]); + convert_params.input_exe_name = str8_list_first(&input_paths_from_format_table[RB_FileFormat_PE]); + convert_params.input_pdb_data = os_data_from_file_path(arena, convert_params.input_pdb_name); + convert_params.input_exe_data = os_data_from_file_path(arena, convert_params.input_exe_name); + convert_params.subset_flags = RDIM_SubsetFlag_All & ~(RDIM_SubsetFlag_Types|RDIM_SubsetFlag_UDTs); + convert_params.deterministic = cmd_line_has_flag(cmdline, str8_lit("deterministic")); + } + RDIM_BakeParams bake_params = {0}; + ProfScope("convert") bake_params = p2r_convert(arena, async_root, &convert_params); + + //- rjf: produce breakpad text + String8List dump = {0}; + ProfScope("dump breakpad text") + { + p2b_async_root = async_root; + RDIM_BakeParams *params = &bake_params; + + //- rjf: kick off unit vmap baking + P2B_BakeUnitVMapIn bake_unit_vmap_in = {¶ms->units}; + ASYNC_Task *bake_unit_vmap_task = async_task_launch(arena, p2b_bake_unit_vmap_work, .input = &bake_unit_vmap_in); + + //- rjf: kick off line-table baking + P2B_BakeLineTablesIn bake_line_tables_in = {¶ms->line_tables}; + ASYNC_Task *bake_line_tables_task = async_task_launch(arena, p2b_bake_line_table_work, .input = &bake_line_tables_in); + + //- rjf: build unit -> line table idx array + U64 unit_count = params->units.total_count; + U32 *unit_line_table_idxs = push_array(arena, U32, unit_count+1); + { + U64 dst_idx = 1; + for(RDIM_UnitChunkNode *n = params->units.first; n != 0; n = n->next) + { + for(U64 n_idx = 0; n_idx < n->count; n_idx += 1, dst_idx += 1) + { + unit_line_table_idxs[dst_idx] = rdim_idx_from_line_table(n->v[n_idx].line_table); + } + } + } + + //- rjf: dump MODULE record + str8_list_pushf(arena, &dump, "MODULE windows x86_64 %I64x %S\n", params->top_level_info.exe_hash, params->top_level_info.exe_name); + + //- rjf: dump FILE records + ProfScope("dump FILE records") + { + for(RDIM_SrcFileChunkNode *n = params->src_files.first; n != 0; n = n->next) + { + for(U64 idx = 0; idx < n->count; idx += 1) + { + U64 file_idx = rdim_idx_from_src_file(&n->v[idx]); + String8 src_path = n->v[idx].normal_full_path; + str8_list_pushf(arena, &dump, "FILE %I64u %S\n", file_idx, src_path); + } + } + } + + //- rjf: join unit vmap + ProfBegin("join unit vmap"); + RDIM_UnitVMapBakeResult *bake_unit_vmap_out = async_task_join_struct(bake_unit_vmap_task, RDIM_UnitVMapBakeResult); + RDI_VMapEntry *unit_vmap = bake_unit_vmap_out->vmap.vmap; + U32 unit_vmap_count = bake_unit_vmap_out->vmap.count; + ProfEnd(); + + //- rjf: join line tables + ProfBegin("join line table"); + RDIM_LineTableBakeResult *bake_line_tables_out = async_task_join_struct(bake_line_tables_task, RDIM_LineTableBakeResult); + ProfEnd(); + + //- rjf: kick off FUNC & line record dump tasks + P2B_DumpProcChunkIn *dump_proc_chunk_in = push_array(arena, P2B_DumpProcChunkIn, params->procedures.chunk_count); + ASYNC_Task **dump_proc_chunk_tasks = push_array(arena, ASYNC_Task *, params->procedures.chunk_count); + ProfScope("kick off FUNC & line record dump tasks") + { + U64 task_idx = 0; + for(RDIM_SymbolChunkNode *n = params->procedures.first; n != 0; n = n->next, task_idx += 1) + { + dump_proc_chunk_in[task_idx].unit_vmap = unit_vmap; + dump_proc_chunk_in[task_idx].unit_vmap_count = unit_vmap_count; + dump_proc_chunk_in[task_idx].unit_line_table_idxs = unit_line_table_idxs; + dump_proc_chunk_in[task_idx].unit_count = unit_count; + dump_proc_chunk_in[task_idx].line_tables_bake = bake_line_tables_out; + dump_proc_chunk_in[task_idx].chunk = n; + dump_proc_chunk_tasks[task_idx] = async_task_launch(arena, p2b_dump_proc_chunk_work, .input = &dump_proc_chunk_in[task_idx]); + } + } + + //- rjf: join FUNC & line record dump tasks + ProfScope("join FUNC & line record dump tasks") + { + for(U64 idx = 0; idx < params->procedures.chunk_count; idx += 1) + { + String8List *out = async_task_join_struct(dump_proc_chunk_tasks[idx], String8List); + str8_list_concat_in_place(&dump, out); + } + } + } + + //- rjf: join with out + str8_list_concat_in_place(&output_blobs, &dump); + + // rjf: no output path? -> pick one based on PDB + if(output_path.size == 0) + { + output_path = push_str8f(arena, "%S.psyms", str8_chop_last_dot(convert_params.input_pdb_name)); + } + } + + //- rjf: no viable input paths + else + { + log_user_errorf("Could not create a Breakpad file from the specified inputs. You must provide either a valid PDB file or an executable image (PE, ELF) file with DWARF information."); + } + }break; + } + + ////////////////////////////// + //- rjf: write info & errors + // + LogScopeResult log_scope = log_scope_end(arena); + if(cmd_line_has_flag(cmdline, str8_lit("verbose")) && log_scope.strings[LogMsgKind_Info].size != 0) + { + String8List lines = wrapped_lines_from_string(arena, log_scope.strings[LogMsgKind_Info], 80, 80, 0); + for(String8Node *n = lines.first; n != 0; n = n->next) + { + fprintf(stderr, "%.*s\n", str8_varg(n->string)); + } + } + if(log_scope.strings[LogMsgKind_UserError].size != 0) + { + String8List lines = wrapped_lines_from_string(arena, log_scope.strings[LogMsgKind_UserError], 80, 80, 0); + for(String8Node *n = lines.first; n != 0; n = n->next) + { + fprintf(stderr, "%.*s\n", str8_varg(n->string)); + } + } + + ////////////////////////////// + //- rjf: write outputs + // + if(output_path.size != 0) ProfScope("write outputs [file]") + { + OS_Handle output_file = os_file_open(OS_AccessFlag_Read|OS_AccessFlag_Write, output_path); + U64 off = 0; + for(String8Node *n = output_blobs.first; n != 0; n = n->next) + { + os_file_write(output_file, r1u64(off, off+n->string.size), n->string.str); + off += n->string.size; + } + os_file_close(output_file); + } + else ProfScope("write outputs [stdout]") + { + for(String8Node *n = output_blobs.first; n != 0; n = n->next) + { + for(U64 off = 0; off < n->string.size;) + { + U64 size_to_write = Min(n->string.size - off, GB(2)); + fwrite(n->string.str + off, size_to_write, 1, stdout); + off += size_to_write; + } + } + } +} diff --git a/src/radbin/radbin.h b/src/radbin/radbin.h new file mode 100644 index 00000000..86c8e064 --- /dev/null +++ b/src/radbin/radbin.h @@ -0,0 +1,17 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +#ifndef RADBIN_H +#define RADBIN_H + +//////////////////////////////// +//~ rjf: Generated Code + +#include "radbin/generated/radbin.meta.h" + +//////////////////////////////// +//~ rjf: Top-Level Entry Point + +internal void rb_entry_point(CmdLine *cmdline); + +#endif //RADBIN_H diff --git a/src/radbin/radbin.mdesk b/src/radbin/radbin.mdesk new file mode 100644 index 00000000..7678420d --- /dev/null +++ b/src/radbin/radbin.mdesk @@ -0,0 +1,28 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +@table(name display_name) +RB_FileFormatTable: +{ + { PDB "PDB" } + { PE "PE" } + { COFF_OBJ "COFF (OBJ)" } + { COFF_BigOBJ "COFF (Big OBJ)" } + { COFF_Archive "COFF (Archive)" } + { COFF_ThinArchive "COFF (Thin Archive)" } + { ELF "ELF" } + { RDI "RDI" } +} + +@enum RB_FileFormat: +{ + Null, + @expand(RB_FileFormatTable a) `$(a.name)`, + COUNT +} + +@data(String8) rb_file_format_display_name_table: +{ + `{0}`, + @expand(RB_FileFormatTable a) `str8_lit_comp("$(a.display_name)")`, +} diff --git a/src/radbin/radbin_main.c b/src/radbin/radbin_main.c new file mode 100644 index 00000000..c5a95fbb --- /dev/null +++ b/src/radbin/radbin_main.c @@ -0,0 +1,70 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +//////////////////////////////// +//~ rjf: Build Options + +#define BUILD_TITLE "Epic Games Tools (R) RAD CLI Binary Utility" +#define BUILD_CONSOLE_INTERFACE 1 + +//////////////////////////////// +//~ rjf: Includes + +//- rjf: [lib] +#include "third_party/rad_lzb_simple/rad_lzb_simple.h" +#include "third_party/rad_lzb_simple/rad_lzb_simple.c" + +//- rjf: [h] +#include "base/base_inc.h" +#include "os/os_inc.h" +#include "path/path.h" +#include "async/async.h" +#include "rdi_format/rdi_format_local.h" +#include "rdi_make/rdi_make_local.h" +#include "coff/coff.h" +#include "coff/coff_parse.h" +#include "pe/pe.h" +#include "elf/elf.h" +#include "elf/elf_parse.h" +#include "codeview/codeview.h" +#include "codeview/codeview_parse.h" +#include "msf/msf.h" +#include "msf/msf_parse.h" +#include "pdb/pdb.h" +#include "pdb/pdb_parse.h" +#include "pdb/pdb_stringize.h" +#include "rdi_from_pdb/rdi_from_pdb.h" +#include "rdi_breakpad_from_pdb/rdi_breakpad_from_pdb.h" +#include "radbin/radbin.h" + +//- rjf: [c] +#include "base/base_inc.c" +#include "os/os_inc.c" +#include "path/path.c" +#include "async/async.c" +#include "rdi_format/rdi_format_local.c" +#include "rdi_make/rdi_make_local.c" +#include "coff/coff.c" +#include "coff/coff_parse.c" +#include "pe/pe.c" +#include "elf/elf.c" +#include "elf/elf_parse.c" +#include "codeview/codeview.c" +#include "codeview/codeview_parse.c" +#include "msf/msf.c" +#include "msf/msf_parse.c" +#include "pdb/pdb.c" +#include "pdb/pdb_parse.c" +#include "pdb/pdb_stringize.c" +#include "rdi_from_pdb/rdi_from_pdb.c" +#include "rdi_breakpad_from_pdb/rdi_breakpad_from_pdb.c" +#include "radbin/radbin.c" + +//////////////////////////////// +//~ rjf: Entry Point + +internal void +entry_point(CmdLine *cmdline) +{ + rb_entry_point(cmdline); +} diff --git a/src/radcon/radcon_pdb.c b/src/radcon/radcon_pdb.c index 44bd224e..752b2286 100644 --- a/src/radcon/radcon_pdb.c +++ b/src/radcon/radcon_pdb.c @@ -2124,7 +2124,7 @@ ASYNC_WORK_DEF(p2r_symbol_stream_convert_work) //////////////////////////////// //~ rjf: Top-Level Conversion Entry Point -internal RDIM_BakeParams * +internal RDIM_BakeParams p2r_convert(Arena *arena, RDIM_LocalState *local_state, RC_Context *in) { Temp scratch = scratch_begin(&arena, 1); diff --git a/src/radcon/radcon_pdb.h b/src/radcon/radcon_pdb.h index 50323699..89c8472c 100644 --- a/src/radcon/radcon_pdb.h +++ b/src/radcon/radcon_pdb.h @@ -226,7 +226,7 @@ ASYNC_WORK_DEF(p2r_symbol_stream_convert_work); //////////////////////////////// //~ rjf: Top-Level Conversion Entry Point -internal RDIM_BakeParams *p2r_convert(Arena *arena, RDIM_LocalState *local_state, RC_Context *in); +internal RDIM_BakeParams p2r_convert(Arena *arena, RDIM_LocalState *local_state, RC_Context *in); //////////////////////////////// diff --git a/src/raddbg/raddbg_main.c b/src/raddbg/raddbg_main.c index 8c624bd7..4261d92a 100644 --- a/src/raddbg/raddbg_main.c +++ b/src/raddbg/raddbg_main.c @@ -193,7 +193,6 @@ #define FNT_INIT_MANUAL 1 #define D_INIT_MANUAL 1 #define RD_INIT_MANUAL 1 -#define P2R_INIT_MANUAL 1 //////////////////////////////// //~ rjf: Includes @@ -217,6 +216,8 @@ #include "coff/coff.h" #include "coff/coff_parse.h" #include "pe/pe.h" +#include "elf/elf.h" +#include "elf/elf_parse.h" #include "codeview/codeview.h" #include "codeview/codeview_parse.h" #include "msf/msf.h" @@ -225,6 +226,8 @@ #include "pdb/pdb_parse.h" #include "pdb/pdb_stringize.h" #include "rdi_from_pdb/rdi_from_pdb.h" +#include "rdi_breakpad_from_pdb/rdi_breakpad_from_pdb.h" +#include "radbin/radbin.h" #include "regs/regs.h" #include "regs/rdi/regs_rdi.h" #include "dbgi/dbgi.h" @@ -259,6 +262,8 @@ #include "coff/coff.c" #include "coff/coff_parse.c" #include "pe/pe.c" +#include "elf/elf.c" +#include "elf/elf_parse.c" #include "codeview/codeview.c" #include "codeview/codeview_parse.c" #include "msf/msf.c" @@ -267,6 +272,8 @@ #include "pdb/pdb_parse.c" #include "pdb/pdb_stringize.c" #include "rdi_from_pdb/rdi_from_pdb.c" +#include "rdi_breakpad_from_pdb/rdi_breakpad_from_pdb.c" +#include "radbin/radbin.c" #include "regs/regs.c" #include "regs/rdi/regs_rdi.c" #include "dbgi/dbgi.c" @@ -788,71 +795,7 @@ entry_point(CmdLine *cmd_line) //- rjf: built-in pdb/dwarf -> rdi converter mode case ExecMode_Converter: { - Temp scratch = scratch_begin(0, 0); - - //- rjf: initializer pdb -> rdi conversion layer - p2r_init(); - - //- rjf: parse arguments - P2R_User2Convert *user2convert = p2r_user2convert_from_cmdln(scratch.arena, cmd_line); - - //- rjf: open output file - String8 output_name = push_str8_copy(scratch.arena, user2convert->output_name); - OS_Handle out_file = os_file_open(OS_AccessFlag_Read|OS_AccessFlag_Write, output_name); - B32 out_file_is_good = !os_handle_match(out_file, os_handle_zero()); - - //- rjf: convert - P2R_Convert2Bake *convert2bake = 0; - if(out_file_is_good) ProfScope("convert") - { - convert2bake = p2r_convert(scratch.arena, user2convert); - } - - //- rjf: bake - P2R_Bake2Serialize *bake2srlz = 0; - if(out_file_is_good) ProfScope("bake") - { - bake2srlz = p2r_bake(scratch.arena, convert2bake); - } - - //- rjf: serialize - P2R_Serialize2File *srlz2file = 0; - if(out_file_is_good) ProfScope("serialize") - { - srlz2file = push_array(scratch.arena, P2R_Serialize2File, 1); - srlz2file->bundle = rdim_serialized_section_bundle_from_bake_results(&bake2srlz->bake_results); - } - - //- rjf: compress - P2R_Serialize2File *srlz2file_compressed = srlz2file; - if(out_file_is_good) if(cmd_line_has_flag(cmd_line, str8_lit("compress"))) ProfScope("compress") - { - srlz2file_compressed = push_array(scratch.arena, P2R_Serialize2File, 1); - srlz2file_compressed = p2r_compress(scratch.arena, srlz2file); - } - - //- rjf: serialize - String8List blobs = {0}; - if(out_file_is_good) - { - blobs = rdim_file_blobs_from_section_bundle(scratch.arena, &srlz2file_compressed->bundle); - } - - //- rjf: write - if(out_file_is_good) - { - U64 off = 0; - for(String8Node *n = blobs.first; n != 0; n = n->next) - { - os_file_write(out_file, r1u64(off, off+n->string.size), n->string.str); - off += n->string.size; - } - } - - //- rjf: close output file - os_file_close(out_file); - - scratch_end(scratch); + rb_entry_point(cmd_line); }break; //- rjf: help message box diff --git a/src/raddump/raddump_main.c b/src/raddump/raddump_main.c index 958d24ab..f3be6eb5 100644 --- a/src/raddump/raddump_main.c +++ b/src/raddump/raddump_main.c @@ -313,4 +313,3 @@ entry_point(CmdLine *cmdline) arena_release(arena); } - diff --git a/src/rdi_breakpad_from_pdb/rdi_breakpad_from_pdb.c b/src/rdi_breakpad_from_pdb/rdi_breakpad_from_pdb.c new file mode 100644 index 00000000..e60dd482 --- /dev/null +++ b/src/rdi_breakpad_from_pdb/rdi_breakpad_from_pdb.c @@ -0,0 +1,106 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +//////////////////////////////// +//~ rjf: Baking Tasks + +//- rjf: unit vmap baking + +ASYNC_WORK_DEF(p2b_bake_unit_vmap_work) +{ + ProfBeginFunction(); + Arena *arena = async_root_thread_arena(p2b_async_root); + P2B_BakeUnitVMapIn *in = (P2B_BakeUnitVMapIn *)input; + RDIM_UnitVMapBakeResult *out = push_array(arena, RDIM_UnitVMapBakeResult, 1); + *out = rdim_bake_unit_vmap(arena, in->units); + ProfEnd(); + return out; +} + +//- rjf: line table baking + +ASYNC_WORK_DEF(p2b_bake_line_table_work) +{ + ProfBeginFunction(); + Arena *arena = async_root_thread_arena(p2b_async_root); + P2B_BakeLineTablesIn *in = (P2B_BakeLineTablesIn *)input; + RDIM_LineTableBakeResult *out = push_array(arena, RDIM_LineTableBakeResult, 1); + *out = rdim_bake_line_tables(arena, in->line_tables); + ProfEnd(); + return out; +} + +//- rjf: per-procedure chunk dumping + +ASYNC_WORK_DEF(p2b_dump_proc_chunk_work) +{ + ProfBeginFunction(); + Arena *arena = async_root_thread_arena(p2b_async_root); + P2B_DumpProcChunkIn *in = (P2B_DumpProcChunkIn *)input; + String8List *out = push_array(arena, String8List, 1); + RDI_LineTable *line_tables = in->line_tables_bake->line_tables; + RDI_U64 line_tables_count = in->line_tables_bake->line_tables_count; + RDI_U64 *line_table_voffs = in->line_tables_bake->line_table_voffs; + RDI_U64 line_table_voffs_count = in->line_tables_bake->line_table_voffs_count; + RDI_Line *line_table_lines = in->line_tables_bake->line_table_lines; + RDI_U64 line_table_lines_count = in->line_tables_bake->line_table_lines_count; + for(U64 idx = 0; idx < in->chunk->count; idx += 1) + { + // NOTE(rjf): breakpad does not support multiple voff ranges per procedure. + RDIM_Symbol *proc = &in->chunk->v[idx]; + RDIM_Scope *root_scope = proc->root_scope; + if(root_scope != 0 && root_scope->voff_ranges.first != 0) + { + // rjf: dump function record + RDIM_Rng1U64 voff_range = root_scope->voff_ranges.first->v; + str8_list_pushf(arena, out, "FUNC %I64x %I64x %I64x %S\n", voff_range.min, voff_range.max-voff_range.min, 0ull, proc->name); + + // rjf: dump function lines + U64 unit_idx = rdi_vmap_idx_from_voff(in->unit_vmap, in->unit_vmap_count, voff_range.min); + if(0 < unit_idx && unit_idx <= in->unit_count) + { + U32 line_table_idx = in->unit_line_table_idxs[unit_idx]; + if(0 < line_table_idx && line_table_idx <= line_tables_count) + { + // rjf: unpack unit line info + RDI_LineTable *line_table = &line_tables[line_table_idx]; + RDI_ParsedLineTable line_info = + { + line_table_voffs + line_table->voffs_base_idx, + line_table_lines + line_table->lines_base_idx, + 0, + line_table->lines_count, + 0 + }; + for(U64 voff = voff_range.min, last_voff = 0; + voff < voff_range.max && voff > last_voff;) + { + RDI_U64 line_info_idx = rdi_line_info_idx_from_voff(&line_info, voff); + if(line_info_idx < line_info.count) + { + RDI_Line *line = &line_info.lines[line_info_idx]; + U64 line_voff_min = line_info.voffs[line_info_idx]; + U64 line_voff_opl = line_info.voffs[line_info_idx+1]; + if(line->file_idx != 0) + { + str8_list_pushf(arena, out, "%I64x %I64x %I64u %I64u\n", + line_voff_min, + line_voff_opl-line_voff_min, + (U64)line->line_num, + (U64)line->file_idx); + } + last_voff = voff; + voff = line_voff_opl; + } + else + { + break; + } + } + } + } + } + } + ProfEnd(); + return out; +} diff --git a/src/rdi_breakpad_from_pdb/rdi_breakpad_from_pdb.h b/src/rdi_breakpad_from_pdb/rdi_breakpad_from_pdb.h new file mode 100644 index 00000000..1cc29d20 --- /dev/null +++ b/src/rdi_breakpad_from_pdb/rdi_breakpad_from_pdb.h @@ -0,0 +1,47 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +#ifndef RDI_BREAKPAD_FROM_PDB_H +#define RDI_BREAKPAD_FROM_PDB_H + +//////////////////////////////// +//~ rjf: Baking Tasks + +//- rjf: unit vmap baking + +typedef struct P2B_BakeUnitVMapIn P2B_BakeUnitVMapIn; +struct P2B_BakeUnitVMapIn +{ + RDIM_UnitChunkList *units; +}; +ASYNC_WORK_DEF(p2b_bake_unit_vmap_work); + +//- rjf: line table baking + +typedef struct P2B_BakeLineTablesIn P2B_BakeLineTablesIn; +struct P2B_BakeLineTablesIn +{ + RDIM_LineTableChunkList *line_tables; +}; +ASYNC_WORK_DEF(p2b_bake_line_table_work); + +//- rjf: per-procedure chunk dumping + +typedef struct P2B_DumpProcChunkIn P2B_DumpProcChunkIn; +struct P2B_DumpProcChunkIn +{ + RDI_VMapEntry *unit_vmap; + U32 unit_vmap_count; + U32 *unit_line_table_idxs; + U64 unit_count; + RDIM_LineTableBakeResult *line_tables_bake; + RDIM_SymbolChunkNode *chunk; +}; +ASYNC_WORK_DEF(p2b_dump_proc_chunk_work); + +//////////////////////////////// +//~ rjf: Globals + +global ASYNC_Root *p2b_async_root = 0; + +#endif // RDI_BREAKPAD_FROM_PDB_H diff --git a/src/rdi_breakpad_from_pdb/rdi_breakpad_from_pdb_main.c b/src/rdi_breakpad_from_pdb/rdi_breakpad_from_pdb_main.c deleted file mode 100644 index f0cf50c8..00000000 --- a/src/rdi_breakpad_from_pdb/rdi_breakpad_from_pdb_main.c +++ /dev/null @@ -1,331 +0,0 @@ -// Copyright (c) Epic Games Tools -// Licensed under the MIT license (https://opensource.org/license/mit/) - -//////////////////////////////// -//~ rjf: Build Options - -#define BUILD_TITLE "rdi_breakpad_from_pdb" -#define BUILD_CONSOLE_INTERFACE 1 - -//////////////////////////////// -//~ rjf: Includes - -//- rjf: [lib] -#include "lib_rdi_format/rdi_format.h" -#include "lib_rdi_format/rdi_format.c" -#include "lib_rdi_format/rdi_format_parse.h" -#include "lib_rdi_format/rdi_format_parse.c" -#include "third_party/rad_lzb_simple/rad_lzb_simple.h" -#include "third_party/rad_lzb_simple/rad_lzb_simple.c" - -//- rjf: [h] -#include "base/base_inc.h" -#include "os/os_inc.h" -#include "path/path.h" -#include "async/async.h" -#include "rdi_make/rdi_make_local.h" -#include "coff/coff.h" -#include "coff/coff_parse.h" -#include "codeview/codeview.h" -#include "codeview/codeview_parse.h" -#include "codeview/codeview_enum.h" -#include "msf/msf.h" -#include "msf/msf_parse.h" -#include "pdb/pdb.h" -#include "pdb/pdb_parse.h" -#include "pdb/pdb_stringize.h" -#include "rdi_from_pdb/rdi_from_pdb.h" - -//- rjf: [c] -#include "base/base_inc.c" -#include "os/os_inc.c" -#include "path/path.c" -#include "async/async.c" -#include "rdi_make/rdi_make_local.c" -#include "coff/coff.c" -#include "coff/coff_parse.c" -#include "codeview/codeview.c" -#include "codeview/codeview_parse.c" -#include "codeview/codeview_enum.c" -#include "msf/msf.c" -#include "msf/msf_parse.c" -#include "pdb/pdb.c" -#include "pdb/pdb_parse.c" -#include "pdb/pdb_stringize.c" -#include "rdi_from_pdb/rdi_from_pdb.c" - -//////////////////////////////// -//~ rjf: Baking Tasks - -//- rjf: unit vmap baking - -typedef struct P2B_BakeUnitVMapIn P2B_BakeUnitVMapIn; -struct P2B_BakeUnitVMapIn -{ - RDIM_UnitChunkList *units; -}; - -ASYNC_WORK_DEF(p2b_bake_unit_vmap_work) -{ - ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; - P2B_BakeUnitVMapIn *in = (P2B_BakeUnitVMapIn *)input; - RDIM_UnitVMapBakeResult *out = push_array(arena, RDIM_UnitVMapBakeResult, 1); - *out = rdim_bake_unit_vmap(arena, in->units); - ProfEnd(); - return out; -} - -//- rjf: line table baking - -typedef struct P2B_BakeLineTablesIn P2B_BakeLineTablesIn; -struct P2B_BakeLineTablesIn -{ - RDIM_LineTableChunkList *line_tables; -}; - -ASYNC_WORK_DEF(p2b_bake_line_table_work) -{ - ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; - P2B_BakeLineTablesIn *in = (P2B_BakeLineTablesIn *)input; - RDIM_LineTableBakeResult *out = push_array(arena, RDIM_LineTableBakeResult, 1); - *out = rdim_bake_line_tables(arena, in->line_tables); - ProfEnd(); - return out; -} - -//- rjf: per-procedure chunk dumping - -typedef struct P2B_DumpProcChunkIn P2B_DumpProcChunkIn; -struct P2B_DumpProcChunkIn -{ - RDI_VMapEntry *unit_vmap; - U32 unit_vmap_count; - U32 *unit_line_table_idxs; - U64 unit_count; - RDIM_LineTableBakeResult *line_tables_bake; - RDIM_SymbolChunkNode *chunk; -}; - -ASYNC_WORK_DEF(p2b_dump_proc_chunk_work) -{ - ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; - P2B_DumpProcChunkIn *in = (P2B_DumpProcChunkIn *)input; - String8List *out = push_array(arena, String8List, 1); - RDI_LineTable *line_tables = in->line_tables_bake->line_tables; - RDI_U64 line_tables_count = in->line_tables_bake->line_tables_count; - RDI_U64 *line_table_voffs = in->line_tables_bake->line_table_voffs; - RDI_U64 line_table_voffs_count = in->line_tables_bake->line_table_voffs_count; - RDI_Line *line_table_lines = in->line_tables_bake->line_table_lines; - RDI_U64 line_table_lines_count = in->line_tables_bake->line_table_lines_count; - for(U64 idx = 0; idx < in->chunk->count; idx += 1) - { - // NOTE(rjf): breakpad does not support multiple voff ranges per procedure. - RDIM_Symbol *proc = &in->chunk->v[idx]; - RDIM_Scope *root_scope = proc->root_scope; - if(root_scope != 0 && root_scope->voff_ranges.first != 0) - { - // rjf: dump function record - RDIM_Rng1U64 voff_range = root_scope->voff_ranges.first->v; - str8_list_pushf(arena, out, "FUNC %I64x %I64x %I64x %S\n", voff_range.min, voff_range.max-voff_range.min, 0ull, proc->name); - - // rjf: dump function lines - U64 unit_idx = rdi_vmap_idx_from_voff(in->unit_vmap, in->unit_vmap_count, voff_range.min); - if(0 < unit_idx && unit_idx <= in->unit_count) - { - U32 line_table_idx = in->unit_line_table_idxs[unit_idx]; - if(0 < line_table_idx && line_table_idx <= line_tables_count) - { - // rjf: unpack unit line info - RDI_LineTable *line_table = &line_tables[line_table_idx]; - RDI_ParsedLineTable line_info = - { - line_table_voffs + line_table->voffs_base_idx, - line_table_lines + line_table->lines_base_idx, - 0, - line_table->lines_count, - 0 - }; - for(U64 voff = voff_range.min, last_voff = 0; - voff < voff_range.max && voff > last_voff;) - { - RDI_U64 line_info_idx = rdi_line_info_idx_from_voff(&line_info, voff); - if(line_info_idx < line_info.count) - { - RDI_Line *line = &line_info.lines[line_info_idx]; - U64 line_voff_min = line_info.voffs[line_info_idx]; - U64 line_voff_opl = line_info.voffs[line_info_idx+1]; - if(line->file_idx != 0) - { - str8_list_pushf(arena, out, "%I64x %I64x %I64u %I64u\n", - line_voff_min, - line_voff_opl-line_voff_min, - (U64)line->line_num, - (U64)line->file_idx); - } - last_voff = voff; - voff = line_voff_opl; - } - else - { - break; - } - } - } - } - } - } - ProfEnd(); - return out; -} - -//////////////////////////////// -//~ rjf: Entry Point - -internal void -entry_point(CmdLine *cmdline) -{ - //- rjf: initialize state, unpack command line - Arena *arena = arena_alloc(); - B32 do_help = (cmd_line_has_flag(cmdline, str8_lit("help")) || - cmd_line_has_flag(cmdline, str8_lit("h")) || - cmd_line_has_flag(cmdline, str8_lit("?"))); - P2R_User2Convert *user2convert = p2r_user2convert_from_cmdln(arena, cmdline); - user2convert->flags &= ~(P2R_ConvertFlag_Types|P2R_ConvertFlag_UDTs); - - //- rjf: display help - if(do_help || user2convert->errors.node_count != 0) - { - fprintf(stderr, "--- rdi_breakpad_from_pdb -----------------------------------------------------\n\n"); - - fprintf(stderr, "This utility converts debug information from PDBs into the textual Breakpad\n"); - fprintf(stderr, "symbol information format, used for various external utilities, using the RAD\n"); - fprintf(stderr, "Debug Info conversion systems. The following arguments are accepted:\n\n"); - - fprintf(stderr, "--exe: [optional] Specifies the path of the executable file for which the\n"); - fprintf(stderr, " debug info was generated.\n"); - fprintf(stderr, "--pdb: Specifies the path of the PDB debug info file to\n"); - fprintf(stderr, " convert.\n"); - fprintf(stderr, "--out: Specifies the path at which the output Breakpad debug\n"); - fprintf(stderr, " info will be written.\n\n"); - - if(!do_help) - { - for(String8Node *n = user2convert->errors.first; n != 0; n = n->next) - { - fprintf(stderr, "error(input): %.*s\n", str8_varg(n->string)); - } - } - os_abort(0); - } - - //- rjf: convert - P2R_Convert2Bake *convert2bake = 0; - ProfScope("convert") - { - convert2bake = p2r_convert(arena, user2convert); - } - - //- rjf: dump breakpad text - String8List dump = {0}; - ProfScope("dump breakpad text") - { - RDIM_BakeParams *params = &convert2bake->bake_params; - - //- rjf: kick off unit vmap baking - P2B_BakeUnitVMapIn bake_unit_vmap_in = {¶ms->units}; - ASYNC_Task *bake_unit_vmap_task = async_task_launch(arena, p2b_bake_unit_vmap_work, .input = &bake_unit_vmap_in); - - //- rjf: kick off line-table baking - P2B_BakeLineTablesIn bake_line_tables_in = {¶ms->line_tables}; - ASYNC_Task *bake_line_tables_task = async_task_launch(arena, p2b_bake_line_table_work, .input = &bake_line_tables_in); - - //- rjf: build unit -> line table idx array - U64 unit_count = params->units.total_count; - U32 *unit_line_table_idxs = push_array(arena, U32, unit_count+1); - { - U64 dst_idx = 1; - for(RDIM_UnitChunkNode *n = params->units.first; n != 0; n = n->next) - { - for(U64 n_idx = 0; n_idx < n->count; n_idx += 1, dst_idx += 1) - { - unit_line_table_idxs[dst_idx] = rdim_idx_from_line_table(n->v[n_idx].line_table); - } - } - } - - //- rjf: dump MODULE record - str8_list_pushf(arena, &dump, "MODULE windows x86_64 %I64x %S\n", params->top_level_info.exe_hash, params->top_level_info.exe_name); - - //- rjf: dump FILE records - ProfScope("dump FILE records") - { - for(RDIM_SrcFileChunkNode *n = params->src_files.first; n != 0; n = n->next) - { - for(U64 idx = 0; idx < n->count; idx += 1) - { - U64 file_idx = rdim_idx_from_src_file(&n->v[idx]); - String8 src_path = n->v[idx].normal_full_path; - str8_list_pushf(arena, &dump, "FILE %I64u %S\n", file_idx, src_path); - } - } - } - - //- rjf: join unit vmap - ProfBegin("join unit vmap"); - RDIM_UnitVMapBakeResult *bake_unit_vmap_out = async_task_join_struct(bake_unit_vmap_task, RDIM_UnitVMapBakeResult); - RDI_VMapEntry *unit_vmap = bake_unit_vmap_out->vmap.vmap; - U32 unit_vmap_count = bake_unit_vmap_out->vmap.count; - ProfEnd(); - - //- rjf: join line tables - ProfBegin("join line table"); - RDIM_LineTableBakeResult *bake_line_tables_out = async_task_join_struct(bake_line_tables_task, RDIM_LineTableBakeResult); - ProfEnd(); - - //- rjf: kick off FUNC & line record dump tasks - P2B_DumpProcChunkIn *dump_proc_chunk_in = push_array(arena, P2B_DumpProcChunkIn, params->procedures.chunk_count); - ASYNC_Task **dump_proc_chunk_tasks = push_array(arena, ASYNC_Task *, params->procedures.chunk_count); - ProfScope("kick off FUNC & line record dump tasks") - { - U64 task_idx = 0; - for(RDIM_SymbolChunkNode *n = params->procedures.first; n != 0; n = n->next, task_idx += 1) - { - dump_proc_chunk_in[task_idx].unit_vmap = unit_vmap; - dump_proc_chunk_in[task_idx].unit_vmap_count = unit_vmap_count; - dump_proc_chunk_in[task_idx].unit_line_table_idxs = unit_line_table_idxs; - dump_proc_chunk_in[task_idx].unit_count = unit_count; - dump_proc_chunk_in[task_idx].line_tables_bake = bake_line_tables_out; - dump_proc_chunk_in[task_idx].chunk = n; - dump_proc_chunk_tasks[task_idx] = async_task_launch(arena, p2b_dump_proc_chunk_work, .input = &dump_proc_chunk_in[task_idx]); - } - } - - //- rjf: join FUNC & line record dump tasks - ProfScope("join FUNC & line record dump tasks") - { - for(U64 idx = 0; idx < params->procedures.chunk_count; idx += 1) - { - String8List *out = async_task_join_struct(dump_proc_chunk_tasks[idx], String8List); - str8_list_concat_in_place(&dump, out); - } - } - } - - //- rjf: bake - String8 baked = {0}; - ProfScope("bake") - { - baked = str8_list_join(arena, &dump, 0); - } - - //- rjf: write - ProfScope("write") - { - OS_Handle output_file = os_file_open(OS_AccessFlag_Read|OS_AccessFlag_Write, user2convert->output_name); - os_file_write(output_file, r1u64(0, baked.size), baked.str); - os_file_close(output_file); - } -} diff --git a/src/rdi_format/rdi_format.mdesk b/src/rdi_format/rdi_format.mdesk index 3310b57f..a43bae13 100644 --- a/src/rdi_format/rdi_format.mdesk +++ b/src/rdi_format/rdi_format.mdesk @@ -173,10 +173,10 @@ RDI_SectionEncodingTable: RDI_SectionMemberTable: { {encoding RDI_SectionEncoding ""} - {pad RDI_U32 ""} - {off RDI_U64 ""} - {encoded_size RDI_U64 ""} - {unpacked_size RDI_U64 ""} + {pad RDI_U32 ""} + {off RDI_U64 ""} + {encoded_size RDI_U64 ""} + {unpacked_size RDI_U64 ""} } @enum(RDI_U32) RDI_SectionKind: diff --git a/src/rdi_from_dwarf/rdi_from_dwarf_main.c b/src/rdi_from_dwarf/rdi_from_dwarf_main.c deleted file mode 100644 index 152d943a..00000000 --- a/src/rdi_from_dwarf/rdi_from_dwarf_main.c +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright (c) Epic Games Tools -// Licensed under the MIT license (https://opensource.org/license/mit/) - -#define BUILD_TITLE "Epic Games Tools (R) DWARF Converter" -#define BUILD_CONSOLE_INTERFACE 1 - -//////////////////////////////// - -#include "third_party/rad_lzb_simple/rad_lzb_simple.h" -#include "third_party/rad_lzb_simple/rad_lzb_simple.c" -#include "third_party/xxHash/xxhash.c" -#include "third_party/xxHash/xxhash.h" -#include "third_party/radsort/radsort.h" - -//////////////////////////////// - -#include "lib_rdi_format/rdi_format.h" -#include "lib_rdi_format/rdi_format.c" -#include "lib_rdi_format/rdi_format_parse.h" -#include "lib_rdi_format/rdi_format_parse.c" - -//////////////////////////////// - -#include "base/base_inc.h" -#include "os/os_inc.h" -#include "async/async.h" -#include "rdi_make/rdi_make_local.h" -#include "linker/path_ext/path.h" -#include "linker/hash_table.h" -#include "coff/coff.h" -#include "coff/coff_parse.h" -#include "dwarf/dwarf.h" -#include "dwarf/dwarf_parse.h" -#include "dwarf/dwarf_coff.h" -#include "pe/pe.h" -#include "linker/rdi/rdi_coff.h" -#include "rdi_from_dwarf/rdi_from_dwarf.h" - -#include "base/base_inc.c" -#include "os/os_inc.c" -#include "async/async.c" -#include "coff/coff.c" -#include "coff/coff_parse.c" -#include "pe/pe.c" -#include "rdi_make/rdi_make_local.c" -#include "linker/rdi/rdi_coff.c" -#include "linker/path_ext/path.c" -#include "linker/hash_table.c" -#include "dwarf/dwarf.c" -#include "dwarf/dwarf_parse.c" -#include "dwarf/dwarf_coff.c" -#include "rdi_from_dwarf/rdi_from_dwarf.c" - -//////////////////////////////// -// Entry Point - -internal void -entry_point(CmdLine *cmdline) -{ - // initialize state and unpack command line - Arena *arena = arena_alloc(); - B32 do_help = (cmd_line_has_flag(cmdline, str8_lit("help")) || - cmd_line_has_flag(cmdline, str8_lit("h")) || - cmd_line_has_flag(cmdline, str8_lit("?"))); - - D2R_User2Convert *user2convert = d2r_user2convert_from_cmdln(arena, cmdline); - - // display help - if (do_help) { - fprintf(stderr, "--- rdi_from_dwarf ------------------------------------------------------------\n\n"); - - fprintf(stderr, "This utility converts debug information from DWARF into the RAD Debug Info\n"); - fprintf(stderr, "format. The following arguments are accepted:\n\n"); - - fprintf(stderr, "--exe: [optional] Specifies the path of the executable filefor which the\n"); - fprintf(stderr, " debug info was generated.\n"); - fprintf(stderr, "--debug: Specifies the path of the .DEBUG debug info file to\n"); - fprintf(stderr, " convert.\n"); - fprintf(stderr, "--out: Specifies the path at which the output will be written.\n\n"); - - if (!do_help) { - for (String8Node *n = user2convert->errors.first; n != 0; n = n->next) { - fprintf(stderr, "error(input): %.*s\n", str8_varg(n->string)); - } - } - - os_abort(0); - } - - RDIM_LocalState *rdim_local_state = rdim_local_init(); - - ProfBegin("convert"); - RDIM_BakeParams *convert2bake = d2r_convert(arena, user2convert); - ProfEnd(); - - ProfBegin("bake"); - RDIM_BakeResults bake2srlz = d2r_bake(rdim_local_state, convert2bake); - ProfEnd(); - - ProfBegin("serialize bake"); - RDIM_SerializedSectionBundle srlz2file = rdim_serialized_section_bundle_from_bake_results(&bake2srlz); - ProfEnd(); - - RDIM_SerializedSectionBundle srlz2file_compressed = srlz2file; - if (cmd_line_has_flag(cmdline, str8_lit("compress"))) { - ProfBegin("compress"); - srlz2file_compressed = d2r_compress(arena, srlz2file); - ProfEnd(); - } - - ProfBegin("serialize blobs"); - String8List blobs = rdim_file_blobs_from_section_bundle(arena, &srlz2file_compressed); - ProfEnd(); - - ProfBegin("write"); - if (!os_write_data_list_to_file_path(user2convert->output_name, blobs)) { - fprintf(stderr, "error(ouptut): unable to write to %.*s\n", str8_varg(user2convert->output_name)); - } - ProfEnd(); -} - diff --git a/src/rdi_from_pdb/rdi_from_pdb.c b/src/rdi_from_pdb/rdi_from_pdb.c index 2b60a0fd..66b15cba 100644 --- a/src/rdi_from_pdb/rdi_from_pdb.c +++ b/src/rdi_from_pdb/rdi_from_pdb.c @@ -40,11 +40,12 @@ p2r_hash_from_voff(U64 voff) //////////////////////////////// //~ rjf: Command Line -> Conversion Inputs - -internal P2R_User2Convert * + +#if 0 +internal P2R_ConvertParams * p2r_user2convert_from_cmdln(Arena *arena, CmdLine *cmdline) { - P2R_User2Convert *result = push_array(arena, P2R_User2Convert, 1); + P2R_ConvertParams *result = push_array(arena, P2R_ConvertParams, 1); //- rjf: get input pdb { @@ -152,7 +153,8 @@ Case("source_path_name_map",NormalSourcePathNameMap)\ #undef FlagNameMapXList return result; -} +} +#endif //////////////////////////////// //~ rjf: COFF <-> RDI Canonical Conversions @@ -503,7 +505,7 @@ p2r_location_over_lvar_addr_range(Arena *arena, RDIM_ScopeChunkList *scopes, RDI ASYNC_WORK_DEF(p2r_exe_hash_work) { ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(p2r_async_root); P2R_EXEHashIn *in = (P2R_EXEHashIn *)input; U64 *out = push_array(arena, U64, 1); ProfScope("hash exe") *out = rdi_hash(in->exe_data.str, in->exe_data.size); @@ -514,7 +516,7 @@ ASYNC_WORK_DEF(p2r_exe_hash_work) ASYNC_WORK_DEF(p2r_tpi_hash_parse_work) { ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(p2r_async_root); P2R_TPIHashParseIn *in = (P2R_TPIHashParseIn *)input; void *out = 0; ProfScope("parse tpi hash") out = pdb_tpi_hash_from_data(arena, in->strtbl, in->tpi, in->hash_data, in->aux_data); @@ -525,7 +527,7 @@ ASYNC_WORK_DEF(p2r_tpi_hash_parse_work) ASYNC_WORK_DEF(p2r_tpi_leaf_parse_work) { ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(p2r_async_root); P2R_TPILeafParseIn *in = (P2R_TPILeafParseIn *)input; void *out = 0; ProfScope("parse tpi leaf") out = cv_leaf_from_data(arena, in->leaf_data, in->itype_first); @@ -536,7 +538,7 @@ ASYNC_WORK_DEF(p2r_tpi_leaf_parse_work) ASYNC_WORK_DEF(p2r_symbol_stream_parse_work) { ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(p2r_async_root); P2R_SymbolStreamParseIn *in = (P2R_SymbolStreamParseIn *)input; void *out = 0; ProfScope("parse symbol stream") out = cv_sym_from_data(arena, in->data, 4); @@ -547,7 +549,7 @@ ASYNC_WORK_DEF(p2r_symbol_stream_parse_work) ASYNC_WORK_DEF(p2r_c13_stream_parse_work) { ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(p2r_async_root); P2R_C13StreamParseIn *in = (P2R_C13StreamParseIn *)input; void *out = 0; ProfScope("parse c13 stream") out = cv_c13_parsed_from_data(arena, in->data, in->strtbl, in->coff_sections); @@ -558,7 +560,7 @@ ASYNC_WORK_DEF(p2r_c13_stream_parse_work) ASYNC_WORK_DEF(p2r_comp_unit_parse_work) { ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(p2r_async_root); P2R_CompUnitParseIn *in = (P2R_CompUnitParseIn *)input; void *out = 0; ProfScope("parse comp units") out = pdb_comp_unit_array_from_data(arena, in->data); @@ -569,7 +571,7 @@ ASYNC_WORK_DEF(p2r_comp_unit_parse_work) ASYNC_WORK_DEF(p2r_comp_unit_contributions_parse_work) { ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(p2r_async_root); P2R_CompUnitContributionsParseIn *in = (P2R_CompUnitContributionsParseIn *)input; void *out = 0; ProfScope("parse comp unit contributions") out = pdb_comp_unit_contribution_array_from_data(arena, in->data, in->coff_sections); @@ -580,7 +582,7 @@ ASYNC_WORK_DEF(p2r_comp_unit_contributions_parse_work) ASYNC_WORK_DEF(p2r_comp_unit_contributions_bucket_work) { ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(p2r_async_root); P2R_CompUnitContributionsBucketIn *in = (P2R_CompUnitContributionsBucketIn *)input; P2R_CompUnitContributionsBucketOut *out = push_array(arena, P2R_CompUnitContributionsBucketOut, 1); { @@ -605,7 +607,7 @@ ASYNC_WORK_DEF(p2r_comp_unit_contributions_bucket_work) ASYNC_WORK_DEF(p2r_gather_unit_src_file_work) { ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(p2r_async_root); Temp scratch = scratch_begin(&arena, 1); P2R_GatherUnitSrcFilesIn *in = (P2R_GatherUnitSrcFilesIn *)input; P2R_GatherUnitSrcFilesOut *out = push_array(arena, P2R_GatherUnitSrcFilesOut, 1); @@ -856,7 +858,7 @@ ASYNC_WORK_DEF(p2r_gather_unit_src_file_work) ASYNC_WORK_DEF(p2r_unit_convert_work) { ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(p2r_async_root); Temp scratch = scratch_begin(&arena, 1); P2R_UnitConvertIn *in = (P2R_UnitConvertIn *)input; P2R_UnitConvertOut *out = push_array(arena, P2R_UnitConvertOut, 1); @@ -1208,7 +1210,7 @@ ASYNC_WORK_DEF(p2r_unit_convert_work) ASYNC_WORK_DEF(p2r_src_file_seq_equip_work) { ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(p2r_async_root); P2R_SrcFileSeqEquipIn *in = (P2R_SrcFileSeqEquipIn *)input; for(RDIM_LineTableChunkNode *line_table_chunk_n = in->line_tables.first; line_table_chunk_n != 0; line_table_chunk_n = line_table_chunk_n->next) { @@ -1231,7 +1233,7 @@ ASYNC_WORK_DEF(p2r_src_file_seq_equip_work) ASYNC_WORK_DEF(p2r_link_name_map_build_work) { ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(p2r_async_root); P2R_LinkNameMapBuildIn *in = (P2R_LinkNameMapBuildIn *)input; CV_RecRange *rec_ranges_first = in->sym->sym_ranges.ranges; CV_RecRange *rec_ranges_opl = rec_ranges_first + in->sym->sym_ranges.count; @@ -1289,7 +1291,7 @@ ASYNC_WORK_DEF(p2r_link_name_map_build_work) ASYNC_WORK_DEF(p2r_itype_fwd_map_fill_work) { ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(p2r_async_root); P2R_ITypeFwdMapFillIn *in = (P2R_ITypeFwdMapFillIn *)input; ProfScope("fill itype fwd map") for(CV_TypeId itype = in->itype_first; itype < in->itype_opl; itype += 1) { @@ -1419,7 +1421,7 @@ ASYNC_WORK_DEF(p2r_itype_fwd_map_fill_work) ASYNC_WORK_DEF(p2r_itype_chain_build_work) { ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(p2r_async_root); Temp scratch = scratch_begin(&arena, 1); P2R_ITypeChainBuildIn *in = (P2R_ITypeChainBuildIn *)input; ProfScope("dependency itype chain build") @@ -1722,7 +1724,7 @@ ASYNC_WORK_DEF(p2r_itype_chain_build_work) ASYNC_WORK_DEF(p2r_udt_convert_work) { ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(p2r_async_root); P2R_UDTConvertIn *in = (P2R_UDTConvertIn *)input; #define p2r_type_ptr_from_itype(itype) ((in->itype_type_ptrs && (itype) < in->tpi_leaf->itype_opl) ? (in->itype_type_ptrs[(in->itype_fwd_map[(itype)] ? in->itype_fwd_map[(itype)] : (itype))]) : 0) RDIM_UDTChunkList *udts = push_array(arena, RDIM_UDTChunkList, 1); @@ -2359,7 +2361,7 @@ ASYNC_WORK_DEF(p2r_udt_convert_work) ASYNC_WORK_DEF(p2r_symbol_stream_convert_work) { ProfBeginFunction(); - Arena *arena = p2r_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(p2r_async_root); Temp scratch = scratch_begin(&arena, 1); P2R_SymbolStreamConvertIn *in = (P2R_SymbolStreamConvertIn *)input; #define p2r_type_ptr_from_itype(itype) ((in->itype_type_ptrs && (itype) < in->tpi_leaf->itype_opl) ? (in->itype_type_ptrs[(in->itype_fwd_map[(itype)] ? in->itype_fwd_map[(itype)] : (itype))]) : 0) @@ -3226,7 +3228,11 @@ ASYNC_WORK_DEF(p2r_symbol_stream_convert_work) U8 *name_ptr = val_ptr + val.encoded_size; String8 name = str8_cstring_capped(name_ptr, sym_data_opl); String8 val_data = str8_struct(&val64); - U64 container_name_opl = p2r_end_of_cplusplus_container_name(type->name); + U64 container_name_opl = 0; + if(type != 0) + { + container_name_opl = p2r_end_of_cplusplus_container_name(type->name); + } String8 name_qualified = name; if(container_name_opl != 0) { @@ -3269,11 +3275,12 @@ ASYNC_WORK_DEF(p2r_symbol_stream_convert_work) //////////////////////////////// //~ rjf: Top-Level Conversion Entry Point -internal P2R_Convert2Bake * -p2r_convert(Arena *arena, P2R_User2Convert *in) +internal RDIM_BakeParams +p2r_convert(Arena *arena, ASYNC_Root *async_root, P2R_ConvertParams *in) { Temp scratch = scratch_begin(&arena, 1); - + p2r_async_root = async_root; + ////////////////////////////////////////////////////////////// //- rjf: parse MSF structure // @@ -3569,7 +3576,7 @@ p2r_convert(Arena *arena, P2R_User2Convert *in) top_level_info.exe_name = str8_skip_last_slash(in->input_exe_name); top_level_info.exe_hash = exe_hash; top_level_info.voff_max = exe_voff_max; - if(!(in->flags & P2R_ConvertFlag_Deterministic)) + if(in->deterministic) { top_level_info.producer_name = str8_lit(BUILD_TITLE_STRING_LITERAL); } @@ -3742,7 +3749,7 @@ p2r_convert(Arena *arena, P2R_User2Convert *in) CV_TypeId *itype_fwd_map = 0; CV_TypeId itype_first = 0; CV_TypeId itype_opl = 0; - if(tpi_leaf != 0 && in->flags & P2R_ConvertFlag_Types) ProfScope("types pass 1: produce type forward resolution map") + if(tpi_leaf != 0 && in->subset_flags & RDIM_SubsetFlag_Types) ProfScope("types pass 1: produce type forward resolution map") { //- rjf: allocate forward resolution map itype_first = tpi_leaf->itype_first; @@ -3783,7 +3790,7 @@ p2r_convert(Arena *arena, P2R_User2Convert *in) // as such, always show up *earlier* in the actually built types. // P2R_TypeIdChain **itype_chains = 0; - if(tpi_leaf != 0 && in->flags & P2R_ConvertFlag_Types) ProfScope("types pass 2: produce per-itype itype chain (for producing dependent types first)") + if(tpi_leaf != 0 && in->subset_flags & RDIM_SubsetFlag_Types) ProfScope("types pass 2: produce per-itype itype chain (for producing dependent types first)") { //- rjf: allocate itype chain table itype_chains = push_array(arena, P2R_TypeIdChain *, (U64)itype_opl); @@ -3821,7 +3828,7 @@ p2r_convert(Arena *arena, P2R_User2Convert *in) RDIM_Type **itype_type_ptrs = 0; RDIM_TypeChunkList all_types = rdim_init_type_chunk_list(arena, top_level_info.arch); #define p2r_type_ptr_from_itype(itype) ((itype_type_ptrs && (itype) < itype_opl) ? (itype_type_ptrs[(itype_fwd_map[(itype)] ? itype_fwd_map[(itype)] : (itype))]) : 0) - if(in->flags & P2R_ConvertFlag_Types) ProfScope("types pass 3: construct all root/stub types from TPI") + if(in->subset_flags & RDIM_SubsetFlag_Types) ProfScope("types pass 3: construct all root/stub types from TPI") { itype_type_ptrs = push_array(arena, RDIM_Type *, (U64)(itype_opl)); @@ -4341,7 +4348,7 @@ p2r_convert(Arena *arena, P2R_User2Convert *in) U64 udt_tasks_count = ((U64)itype_opl+(udt_task_size_itypes-1))/udt_task_size_itypes; P2R_UDTConvertIn *udt_tasks_inputs = push_array(scratch.arena, P2R_UDTConvertIn, udt_tasks_count); ASYNC_Task **udt_tasks = push_array(scratch.arena, ASYNC_Task *, udt_tasks_count); - if(in->flags & P2R_ConvertFlag_UDTs) ProfScope("types pass 4: kick off UDT build") + if(in->subset_flags & RDIM_SubsetFlag_UDTs) ProfScope("types pass 4: kick off UDT build") { for(U64 idx = 0; idx < udt_tasks_count; idx += 1) { @@ -4475,71 +4482,27 @@ p2r_convert(Arena *arena, P2R_User2Convert *in) ////////////////////////////////////////////////////////////// //- rjf: fill output // - P2R_Convert2Bake *out = push_array(arena, P2R_Convert2Bake, 1); + RDIM_BakeParams out = {0}; { - out->bake_params.top_level_info = top_level_info; - out->bake_params.binary_sections = binary_sections; - out->bake_params.units = all_units; - out->bake_params.types = all_types; - out->bake_params.udts = all_udts; - out->bake_params.src_files = all_src_files; - out->bake_params.line_tables = all_line_tables; - out->bake_params.global_variables = all_global_variables; - out->bake_params.thread_variables = all_thread_variables; - out->bake_params.constants = all_constants; - out->bake_params.procedures = all_procedures; - out->bake_params.scopes = all_scopes; - out->bake_params.inline_sites = all_inline_sites; + out.top_level_info = top_level_info; + out.binary_sections = binary_sections; + out.units = all_units; + out.types = all_types; + out.udts = all_udts; + out.src_files = all_src_files; + out.line_tables = all_line_tables; + out.global_variables = all_global_variables; + out.thread_variables = all_thread_variables; + out.constants = all_constants; + out.procedures = all_procedures; + out.scopes = all_scopes; + out.inline_sites = all_inline_sites; } scratch_end(scratch); return out; } -//////////////////////////////// -//~ rjf: Top-Level Initialization - -internal void -p2r_init(void) -{ - Arena *arena = arena_alloc(); - p2r_state = push_array(arena, P2R_State, 1); - p2r_state->arena = arena; - p2r_state->work_thread_arenas_count = async_thread_count(); - p2r_state->work_thread_arenas = push_array(arena, Arena *, p2r_state->work_thread_arenas_count); - for EachIndex(idx, p2r_state->work_thread_arenas_count) - { - p2r_state->work_thread_arenas[idx] = arena_alloc(); - } -} - -//////////////////////////////// -//~ rjf: Top-Level Baking Entry Point - -internal P2R_Bake2Serialize * -p2r_bake(Arena *arena, P2R_Convert2Bake *in) -{ - RDIM_LocalState local_state = {0}; - local_state.arena = p2r_state->arena; - local_state.work_thread_arenas_count = p2r_state->work_thread_arenas_count; - local_state.work_thread_arenas = p2r_state->work_thread_arenas; - - P2R_Bake2Serialize *result = push_array(arena, P2R_Bake2Serialize, 1); - result->bake_results = rdim_bake(&local_state, &in->bake_params); - return result; -} - -//////////////////////////////// -//~ rjf: Top-Level Compression Entry Point - -internal P2R_Serialize2File * -p2r_compress(Arena *arena, P2R_Serialize2File *in) -{ - P2R_Serialize2File *out = push_array(arena, P2R_Serialize2File, 1); - out->bundle = rdim_compress(arena, &in->bundle); - return out; -} - //////////////////////////////// internal B32 diff --git a/src/rdi_from_pdb/rdi_from_pdb.h b/src/rdi_from_pdb/rdi_from_pdb.h index a61f79a6..c9b2dcd5 100644 --- a/src/rdi_from_pdb/rdi_from_pdb.h +++ b/src/rdi_from_pdb/rdi_from_pdb.h @@ -4,65 +4,18 @@ #ifndef RDI_FROM_PDB_H #define RDI_FROM_PDB_H -//////////////////////////////// -//~ rjf: Export Artifact Flags - -typedef U32 P2R_ConvertFlags; -enum -{ - P2R_ConvertFlag_Strings = (1<<0), - P2R_ConvertFlag_IndexRuns = (1<<1), - P2R_ConvertFlag_BinarySections = (1<<2), - P2R_ConvertFlag_Units = (1<<3), - P2R_ConvertFlag_Procedures = (1<<4), - P2R_ConvertFlag_GlobalVariables = (1<<5), - P2R_ConvertFlag_ThreadVariables = (1<<6), - P2R_ConvertFlag_Scopes = (1<<7), - P2R_ConvertFlag_Locals = (1<<8), - P2R_ConvertFlag_Types = (1<<9), - P2R_ConvertFlag_UDTs = (1<<10), - P2R_ConvertFlag_LineInfo = (1<<11), - P2R_ConvertFlag_GlobalVariableNameMap = (1<<12), - P2R_ConvertFlag_ThreadVariableNameMap = (1<<13), - P2R_ConvertFlag_ProcedureNameMap = (1<<14), - P2R_ConvertFlag_TypeNameMap = (1<<15), - P2R_ConvertFlag_LinkNameProcedureNameMap= (1<<16), - P2R_ConvertFlag_NormalSourcePathNameMap = (1<<17), - P2R_ConvertFlag_Deterministic = (1<<18), - P2R_ConvertFlag_All = 0xffffffff, -}; - //////////////////////////////// //~ rjf: Conversion Stage Inputs/Outputs -typedef struct P2R_User2Convert P2R_User2Convert; -struct P2R_User2Convert +typedef struct P2R_ConvertParams P2R_ConvertParams; +struct P2R_ConvertParams { String8 input_pdb_name; String8 input_pdb_data; String8 input_exe_name; String8 input_exe_data; - String8 output_name; - P2R_ConvertFlags flags; - String8List errors; -}; - -typedef struct P2R_Convert2Bake P2R_Convert2Bake; -struct P2R_Convert2Bake -{ - RDIM_BakeParams bake_params; -}; - -typedef struct P2R_Bake2Serialize P2R_Bake2Serialize; -struct P2R_Bake2Serialize -{ - RDIM_BakeResults bake_results; -}; - -typedef struct P2R_Serialize2File P2R_Serialize2File; -struct P2R_Serialize2File -{ - RDIM_SerializedSectionBundle bundle; + RDIM_SubsetFlags subset_flags; + B32 deterministic; }; //////////////////////////////// @@ -319,21 +272,10 @@ struct P2R_SymbolStreamConvertOut RDIM_TypeChunkList typedefs; }; -//////////////////////////////// -//~ rjf: Top-Level State - -typedef struct P2R_State P2R_State; -struct P2R_State -{ - Arena *arena; - U64 work_thread_arenas_count; - Arena **work_thread_arenas; -}; - //////////////////////////////// //~ rjf: Globals -global P2R_State *p2r_state = 0; +global ASYNC_Root *p2r_async_root = 0; //////////////////////////////// //~ rjf: Basic Helpers @@ -344,7 +286,9 @@ internal U64 p2r_hash_from_voff(U64 voff); //////////////////////////////// //~ rjf: Command Line -> Conversion Inputs -internal P2R_User2Convert *p2r_user2convert_from_cmdln(Arena *arena, CmdLine *cmdline); +#if 0 +internal P2R_ConvertParams *p2r_user2convert_from_cmdln(Arena *arena, CmdLine *cmdline); +#endif //////////////////////////////// //~ rjf: COFF => RDI Canonical Conversions @@ -417,22 +361,7 @@ ASYNC_WORK_DEF(p2r_symbol_stream_convert_work); //////////////////////////////// //~ rjf: Top-Level Conversion Entry Point -internal P2R_Convert2Bake *p2r_convert(Arena *arena, P2R_User2Convert *in); - -//////////////////////////////// -//~ rjf: Top-Level Initialization - -internal void p2r_init(void); - -//////////////////////////////// -//~ rjf: Top-Level Baking Entry Point - -internal P2R_Bake2Serialize *p2r_bake(Arena *arena, P2R_Convert2Bake *in); - -//////////////////////////////// -//~ rjf: Top-Level Compression Entry Point - -internal P2R_Serialize2File *p2r_compress(Arena *arena, P2R_Serialize2File *in); +internal RDIM_BakeParams p2r_convert(Arena *arena, ASYNC_Root *async_root, P2R_ConvertParams *in); //////////////////////////////// diff --git a/src/rdi_from_pdb/rdi_from_pdb_main.c b/src/rdi_from_pdb/rdi_from_pdb_main.c deleted file mode 100644 index 95470474..00000000 --- a/src/rdi_from_pdb/rdi_from_pdb_main.c +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright (c) Epic Games Tools -// Licensed under the MIT license (https://opensource.org/license/mit/) - -//////////////////////////////// -//~ rjf: Build Options - -#define BUILD_TITLE "rdi_from_pdb" -#define BUILD_CONSOLE_INTERFACE 1 - -//////////////////////////////// -//~ rjf: Includes - -//- rjf: [lib] -#include "lib_rdi_format/rdi_format.h" -#include "lib_rdi_format/rdi_format.c" -#include "third_party/rad_lzb_simple/rad_lzb_simple.h" -#include "third_party/rad_lzb_simple/rad_lzb_simple.c" - -//- rjf: [h] -#include "base/base_inc.h" -#include "os/os_inc.h" -#include "path/path.h" -#include "async/async.h" -#include "rdi_make/rdi_make_local.h" -#include "coff/coff.h" -#include "coff/coff_parse.h" -#include "codeview/codeview.h" -#include "codeview/codeview_parse.h" -#include "msf/msf.h" -#include "msf/msf_parse.h" -#include "pdb/pdb.h" -#include "pdb/pdb_parse.h" -#include "pdb/pdb_stringize.h" -#include "rdi_from_pdb.h" - -//- rjf: [c] -#include "base/base_inc.c" -#include "os/os_inc.c" -#include "path/path.c" -#include "async/async.c" -#include "rdi_make/rdi_make_local.c" -#include "coff/coff.c" -#include "coff/coff_parse.c" -#include "codeview/codeview.c" -#include "codeview/codeview_parse.c" -#include "msf/msf.c" -#include "msf/msf_parse.c" -#include "pdb/pdb.c" -#include "pdb/pdb_parse.c" -#include "pdb/pdb_stringize.c" -#include "rdi_from_pdb.c" - -//////////////////////////////// -//~ rjf: Entry Point - -internal void -entry_point(CmdLine *cmdline) -{ - //- rjf: initialize state, unpack command line - Arena *arena = arena_alloc(); - B32 do_help = (cmd_line_has_flag(cmdline, str8_lit("help")) || - cmd_line_has_flag(cmdline, str8_lit("h")) || - cmd_line_has_flag(cmdline, str8_lit("?"))); - P2R_User2Convert *user2convert = p2r_user2convert_from_cmdln(arena, cmdline); - - //- rjf: display help - if(do_help || user2convert->errors.node_count != 0) - { - fprintf(stderr, "--- rdi_from_pdb --------------------------------------------------------------\n\n"); - - fprintf(stderr, "This utility converts debug information from PDBs into the RAD Debug Info\n"); - fprintf(stderr, "format. The following arguments are accepted:\n\n"); - - fprintf(stderr, "--exe: [optional] Specifies the path of the executable file for which the\n"); - fprintf(stderr, " debug info was generated.\n"); - fprintf(stderr, "--pdb: Specifies the path of the PDB debug info file to\n"); - fprintf(stderr, " convert.\n"); - fprintf(stderr, "--out: Specifies the path at which the output Breakpad debug\n"); - fprintf(stderr, " info will be written.\n\n"); - - if(!do_help) - { - for(String8Node *n = user2convert->errors.first; n != 0; n = n->next) - { - fprintf(stderr, "error(input): %.*s\n", str8_varg(n->string)); - } - } - os_abort(0); - } - - //- rjf: convert - P2R_Convert2Bake *convert2bake = 0; - ProfScope("convert") - { - convert2bake = p2r_convert(arena, user2convert); - } - - //- rjf: bake - P2R_Bake2Serialize *bake2srlz = 0; - ProfScope("bake") - { - bake2srlz = p2r_bake(arena, convert2bake); - } - - //- rjf: serialize - P2R_Serialize2File *srlz2file = 0; - ProfScope("serialize") - { - srlz2file = push_array(arena, P2R_Serialize2File, 1); - srlz2file->bundle = rdim_serialized_section_bundle_from_bake_results(&bake2srlz->bake_results); - } - - //- rjf: compress - P2R_Serialize2File *srlz2file_compressed = srlz2file; - if(cmd_line_has_flag(cmdline, str8_lit("compress"))) ProfScope("compress") - { - srlz2file_compressed = push_array(arena, P2R_Serialize2File, 1); - srlz2file_compressed = p2r_compress(arena, srlz2file); - } - - //- rjf: serialize - String8List blobs = rdim_file_blobs_from_section_bundle(arena, &srlz2file_compressed->bundle); - - //- rjf: write - ProfScope("write") - { - OS_Handle output_file = os_file_open(OS_AccessFlag_Read|OS_AccessFlag_Write, user2convert->output_name); - U64 off = 0; - for(String8Node *n = blobs.first; n != 0; n = n->next) - { - os_file_write(output_file, r1u64(off, off+n->string.size), n->string.str); - off += n->string.size; - } - os_file_close(output_file); - } -} diff --git a/src/rdi_make/rdi_make_help.c b/src/rdi_make/rdi_make_help.c deleted file mode 100644 index a0ed9407..00000000 --- a/src/rdi_make/rdi_make_help.c +++ /dev/null @@ -1,927 +0,0 @@ -// Copyright (c) Epic Games Tools -// Licensed under the MIT license (https://opensource.org/license/mit/) - -//////////////////////////////// -//~ rjf: Baking Stage Tasks - -//- rjf: bake string map building - -#define rdim_make_string_map_if_needed() do {if(in->maps[thread_idx] == 0) ProfScope("make map") {in->maps[thread_idx] = rdim_bake_string_map_loose_make(arena, in->top);}} while(0) - -ASYNC_WORK_DEF(rdim_bake_src_files_strings_work) -{ - ProfBeginFunction(); - Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; - RDIM_BakeSrcFilesStringsIn *in = (RDIM_BakeSrcFilesStringsIn *)input; - rdim_make_string_map_if_needed(); - ProfScope("bake src file strings") rdim_bake_string_map_loose_push_src_files(arena, in->top, in->maps[thread_idx], in->list); - ProfEnd(); - return 0; -} - -ASYNC_WORK_DEF(rdim_bake_units_strings_work) -{ - ProfBeginFunction(); - Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; - RDIM_BakeUnitsStringsIn *in = (RDIM_BakeUnitsStringsIn *)input; - rdim_make_string_map_if_needed(); - ProfScope("bake unit strings") rdim_bake_string_map_loose_push_units(arena, in->top, in->maps[thread_idx], in->list); - ProfEnd(); - return 0; -} - -ASYNC_WORK_DEF(rdim_bake_types_strings_work) -{ - ProfBeginFunction(); - Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; - RDIM_BakeTypesStringsIn *in = (RDIM_BakeTypesStringsIn *)input; - rdim_make_string_map_if_needed(); - ProfScope("bake type strings") - { - for(RDIM_BakeTypesStringsInNode *n = in->first; n != 0; n = n->next) - { - rdim_bake_string_map_loose_push_type_slice(arena, in->top, in->maps[thread_idx], n->v, n->count); - } - } - ProfEnd(); - return 0; -} - -ASYNC_WORK_DEF(rdim_bake_udts_strings_work) -{ - ProfBeginFunction(); - Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; - RDIM_BakeUDTsStringsIn *in = (RDIM_BakeUDTsStringsIn *)input; - rdim_make_string_map_if_needed(); - ProfScope("bake udt strings") - { - for(RDIM_BakeUDTsStringsInNode *n = in->first; n != 0; n = n->next) - { - rdim_bake_string_map_loose_push_udt_slice(arena, in->top, in->maps[thread_idx], n->v, n->count); - } - } - ProfEnd(); - return 0; -} - -ASYNC_WORK_DEF(rdim_bake_symbols_strings_work) -{ - ProfBeginFunction(); - Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; - RDIM_BakeSymbolsStringsIn *in = (RDIM_BakeSymbolsStringsIn *)input; - rdim_make_string_map_if_needed(); - ProfScope("bake symbol strings") - { - for(RDIM_BakeSymbolsStringsInNode *n = in->first; n != 0; n = n->next) - { - rdim_bake_string_map_loose_push_symbol_slice(arena, in->top, in->maps[thread_idx], n->v, n->count); - } - } - ProfEnd(); - return 0; -} - -ASYNC_WORK_DEF(rdim_bake_inline_site_strings_work) -{ - ProfBeginFunction(); - Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; - RDIM_BakeInlineSiteStringsIn *in = input; - rdim_make_string_map_if_needed(); - ProfScope("bake inline site strings") - { - for(RDIM_BakeInlineSiteStringsInNode *n = in->first; n != 0; n = n->next) - { - rdim_bake_string_map_loose_push_inline_site_slice(arena, in->top, in->maps[thread_idx], n->v, n->count); - } - } - ProfEnd(); - return 0; -} - -ASYNC_WORK_DEF(rdim_bake_scopes_strings_work) -{ - ProfBeginFunction(); - Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; - RDIM_BakeScopesStringsIn *in = (RDIM_BakeScopesStringsIn *)input; - rdim_make_string_map_if_needed(); - ProfScope("bake scope strings") - { - for(RDIM_BakeScopesStringsInNode *n = in->first; n != 0; n = n->next) - { - rdim_bake_string_map_loose_push_scope_slice(arena, in->top, in->maps[thread_idx], n->v, n->count); - } - } - ProfEnd(); - return 0; -} - -ASYNC_WORK_DEF(rdim_bake_line_tables_work) -{ - ProfBeginFunction(); - Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; - RDIM_BakeLineTablesIn *in = (RDIM_BakeLineTablesIn *)input; - RDIM_LineTableBakeResult *out = push_array(arena, RDIM_LineTableBakeResult, 1); - ProfScope("bake line tables") *out = rdim_bake_line_tables(arena, in->line_tables); - ProfEnd(); - return out; -} - -#undef rdim_make_string_map_if_needed - -//- rjf: bake string map joining - -ASYNC_WORK_DEF(rdim_bake_string_map_join_work) -{ - ProfBeginFunction(); - Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; - RDIM_JoinBakeStringMapSlotsIn *in = (RDIM_JoinBakeStringMapSlotsIn *)input; - ProfScope("join bake string maps") - { - for(U64 src_map_idx = 0; src_map_idx < in->src_maps_count; src_map_idx += 1) - { - for(U64 slot_idx = in->slot_idx_range.min; slot_idx < in->slot_idx_range.max; slot_idx += 1) - { - B32 src_slots_good = (in->src_maps[src_map_idx] != 0 && in->src_maps[src_map_idx]->slots != 0); - B32 dst_slot_is_zero = (in->dst_map->slots[slot_idx] == 0); - if(src_slots_good && dst_slot_is_zero) - { - in->dst_map->slots[slot_idx] = in->src_maps[src_map_idx]->slots[slot_idx]; - } - else if(src_slots_good && in->src_maps[src_map_idx]->slots[slot_idx] != 0) - { - rdim_bake_string_chunk_list_concat_in_place(in->dst_map->slots[slot_idx], in->src_maps[src_map_idx]->slots[slot_idx]); - } - } - } - } - ProfEnd(); - return 0; -} - -//- rjf: bake string map sorting - -ASYNC_WORK_DEF(rdim_bake_string_map_sort_work) -{ - ProfBeginFunction(); - Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; - RDIM_SortBakeStringMapSlotsIn *in = (RDIM_SortBakeStringMapSlotsIn *)input; - ProfScope("sort bake string chunk list map range") - { - for(U64 slot_idx = in->slot_idx; - slot_idx < in->slot_idx+in->slot_count; - slot_idx += 1) - { - if(in->src_map->slots[slot_idx] != 0) - { - if(in->src_map->slots[slot_idx]->total_count > 1) - { - in->dst_map->slots[slot_idx] = push_array(arena, RDIM_BakeStringChunkList, 1); - *in->dst_map->slots[slot_idx] = rdim_bake_string_chunk_list_sorted_from_unsorted(arena, in->src_map->slots[slot_idx]); - } - else - { - in->dst_map->slots[slot_idx] = in->src_map->slots[slot_idx]; - } - } - } - } - ProfEnd(); - return 0; -} - -//- rjf: pass 1: interner/deduper map builds - -ASYNC_WORK_DEF(rdim_build_bake_name_map_work) -{ - ProfBeginFunction(); - Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; - RDIM_BuildBakeNameMapIn *in = (RDIM_BuildBakeNameMapIn *)input; - RDIM_BakeNameMap *name_map = 0; - ProfScope("build name map %i", in->k) name_map = rdim_bake_name_map_from_kind_params(arena, in->k, in->type_indices, in->params); - ProfEnd(); - return name_map; -} - -//- rjf: pass 2: string-map-dependent debug info stream builds - -ASYNC_WORK_DEF(rdim_bake_units_work) -{ - ProfBeginFunction(); - Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; - RDIM_BakeUnitsIn *in = (RDIM_BakeUnitsIn *)input; - RDIM_UnitBakeResult *out = push_array(arena, RDIM_UnitBakeResult, 1); - ProfScope("bake units") *out = rdim_bake_units(arena, in->strings, in->path_tree, in->units); - ProfEnd(); - return out; -} - -ASYNC_WORK_DEF(rdim_bake_unit_vmap_work) -{ - ProfBeginFunction(); - Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; - RDIM_BakeUnitVMapIn *in = (RDIM_BakeUnitVMapIn *)input; - RDIM_UnitVMapBakeResult *out = push_array(arena, RDIM_UnitVMapBakeResult, 1); - ProfScope("bake unit vmap") *out = rdim_bake_unit_vmap(arena, in->units); - ProfEnd(); - return out; -} - -ASYNC_WORK_DEF(rdim_bake_src_files_work) -{ - ProfBeginFunction(); - Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; - RDIM_BakeSrcFilesIn *in = (RDIM_BakeSrcFilesIn *)input; - RDIM_SrcFileBakeResult *out = push_array(arena, RDIM_SrcFileBakeResult, 1); - ProfScope("bake src files") *out = rdim_bake_src_files(arena, in->strings, in->path_tree, in->src_files); - ProfEnd(); - return out; -} - -ASYNC_WORK_DEF(rdim_bake_udts_work) -{ - ProfBeginFunction(); - Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; - RDIM_BakeUDTsIn *in = (RDIM_BakeUDTsIn *)input; - RDIM_UDTBakeResult *out = push_array(arena, RDIM_UDTBakeResult, 1); - ProfScope("bake udts") *out = rdim_bake_udts(arena, in->strings, in->type_indices, in->udts); - ProfEnd(); - return out; -} - -ASYNC_WORK_DEF(rdim_bake_global_variables_work) -{ - ProfBeginFunction(); - Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; - RDIM_BakeGlobalVariablesIn *in = (RDIM_BakeGlobalVariablesIn *)input; - RDIM_GlobalVariableBakeResult *out = push_array(arena, RDIM_GlobalVariableBakeResult, 1); - ProfScope("bake global variables") *out = rdim_bake_global_variables(arena, in->strings, in->type_indices, in->global_variables); - ProfEnd(); - return out; -} - -ASYNC_WORK_DEF(rdim_bake_global_vmap_work) -{ - ProfBeginFunction(); - Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; - RDIM_BakeGlobalVMapIn *in = (RDIM_BakeGlobalVMapIn *)input; - RDIM_GlobalVMapBakeResult *out = push_array(arena, RDIM_GlobalVMapBakeResult, 1); - ProfScope("bake global vmap") *out = rdim_bake_global_vmap(arena, in->global_variables); - ProfEnd(); - return out; -} - -ASYNC_WORK_DEF(rdim_bake_thread_variables_work) -{ - ProfBeginFunction(); - Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; - RDIM_BakeThreadVariablesIn *in = (RDIM_BakeThreadVariablesIn *)input; - RDIM_ThreadVariableBakeResult *out = push_array(arena, RDIM_ThreadVariableBakeResult, 1); - ProfScope("bake thread variables") *out = rdim_bake_thread_variables(arena, in->strings, in->type_indices, in->thread_variables); - ProfEnd(); - return out; -} - -ASYNC_WORK_DEF(rdim_bake_procedures_work) -{ - ProfBeginFunction(); - Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; - RDIM_BakeProceduresIn *in = (RDIM_BakeProceduresIn *)input; - RDIM_ProcedureBakeResult *out = push_array(arena, RDIM_ProcedureBakeResult, 1); - ProfScope("bake procedures") *out = rdim_bake_procedures(arena, in->strings, in->type_indices, in->location_blocks, in->location_data_blobs, in->procedures); - ProfEnd(); - return out; -} - -ASYNC_WORK_DEF(rdim_bake_scopes_work) -{ - ProfBeginFunction(); - Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; - RDIM_BakeScopesIn *in = (RDIM_BakeScopesIn *)input; - RDIM_ScopeBakeResult *out = push_array(arena, RDIM_ScopeBakeResult, 1); - ProfScope("bake scopes") *out = rdim_bake_scopes(arena, in->strings, in->type_indices, in->location_blocks, in->location_data_blobs, in->scopes); - ProfEnd(); - return out; -} - -ASYNC_WORK_DEF(rdim_bake_scope_vmap_work) -{ - ProfBeginFunction(); - Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; - RDIM_BakeScopeVMapIn *in = (RDIM_BakeScopeVMapIn *)input; - RDIM_ScopeVMapBakeResult *out = push_array(arena, RDIM_ScopeVMapBakeResult, 1); - ProfScope("bake scope vmap") *out = rdim_bake_scope_vmap(arena, in->scopes); - ProfEnd(); - return out; -} - -ASYNC_WORK_DEF(rdim_bake_inline_sites_work) -{ - ProfBeginFunction(); - Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; - RDIM_BakeInlineSitesIn *in = (RDIM_BakeInlineSitesIn *)input; - RDIM_InlineSiteBakeResult *out = push_array(arena, RDIM_InlineSiteBakeResult, 1); - ProfScope("bake inline sites") *out = rdim_bake_inline_sites(arena, in->strings, in->type_indices, in->inline_sites); - ProfEnd(); - return out; -} - -ASYNC_WORK_DEF(rdim_bake_file_paths_work) -{ - ProfBeginFunction(); - Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; - RDIM_BakeFilePathsIn *in = (RDIM_BakeFilePathsIn *)input; - RDIM_FilePathBakeResult *out = push_array(arena, RDIM_FilePathBakeResult, 1); - ProfScope("bake file paths") *out = rdim_bake_file_paths(arena, in->strings, in->path_tree); - ProfEnd(); - return out; -} - -ASYNC_WORK_DEF(rdim_bake_strings_work) -{ - ProfBeginFunction(); - Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; - RDIM_BakeStringsIn *in = (RDIM_BakeStringsIn *)input; - RDIM_StringBakeResult *out = push_array(arena, RDIM_StringBakeResult, 1); - ProfScope("bake strings") *out = rdim_bake_strings(arena, in->strings); - ProfEnd(); - return out; -} - -//- rjf: pass 3: idx-run-map-dependent debug info stream builds - -ASYNC_WORK_DEF(rdim_bake_type_nodes_work) -{ - ProfBeginFunction(); - Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; - RDIM_BakeTypeNodesIn *in = (RDIM_BakeTypeNodesIn *)input; - RDIM_TypeNodeBakeResult *out = push_array(arena, RDIM_TypeNodeBakeResult, 1); - ProfScope("bake type nodes") *out = rdim_bake_types(arena, in->strings, in->idx_runs, in->type_indices, in->types); - ProfEnd(); - return out; -} - -ASYNC_WORK_DEF(rdim_bake_name_map_work) -{ - ProfBeginFunction(); - Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; - RDIM_BakeNameMapIn *in = (RDIM_BakeNameMapIn *)input; - RDIM_NameMapBakeResult *out = push_array(arena, RDIM_NameMapBakeResult, 1); - ProfScope("bake name map %i", in->kind) *out = rdim_bake_name_map(arena, in->strings, in->idx_runs, in->map); - ProfEnd(); - return out; -} - -ASYNC_WORK_DEF(rdim_bake_idx_runs_work) -{ - ProfBeginFunction(); - Arena *arena = rdim_help_state->work_thread_arenas[thread_idx]; - RDIM_BakeIdxRunsIn *in = (RDIM_BakeIdxRunsIn *)input; - RDIM_IndexRunBakeResult *out = push_array(arena, RDIM_IndexRunBakeResult, 1); - ProfScope("bake idx runs") *out = rdim_bake_index_runs(arena, in->idx_runs); - ProfEnd(); - return out; -} - -internal RDIM_HelpState * -rdim_help_init(void) -{ - Arena *arena = arena_alloc(); - RDIM_HelpState *state = push_array(arena, RDIM_HelpState, 1); - state->arena = arena; - state->work_thread_arenas_count = async_thread_count(); - state->work_thread_arenas = push_array(arena, Arena *, state->work_thread_arenas_count); - for EachIndex(idx, state->work_thread_arenas_count) - { - state->work_thread_arenas[idx] = arena_alloc(); - } - return state; -} - -internal RDIM_BakeResults -rdim_bake(RDIM_HelpState *state, RDIM_BakeParams *in_params) -{ - Temp scratch = scratch_begin(0,0); - RDIM_BakeResults out = {0}; - - rdim_help_state = state; - - //////////////////////////////// - // compute type indices - - RDI_U64 *type_indices = rdim_make_type_indices(scratch.arena, &in_params->types); - - ////////////////////////////// - //- rjf: kick off line tables baking - // - ASYNC_Task *bake_line_tables_task = 0; - { - RDIM_BakeLineTablesIn *in = push_array(scratch.arena, RDIM_BakeLineTablesIn, 1); - in->line_tables = &in_params->line_tables; - bake_line_tables_task = async_task_launch(scratch.arena, rdim_bake_line_tables_work, .input = in); - } - - ////////////////////////////// - //- rjf: build interned path tree - // - RDIM_BakePathTree *path_tree = 0; - ProfScope("build interned path tree") - { - path_tree = rdim_bake_path_tree_from_params(state->arena, in_params); - } - - ////////////////////////////// - //- rjf: kick off string map building tasks - // - RDIM_BakeStringMapTopology bake_string_map_topology = {(64 + - in_params->procedures.total_count*1 + - in_params->global_variables.total_count*1 + - in_params->thread_variables.total_count*1 + - in_params->constants.total_count*1 + - in_params->types.total_count/2)}; - RDIM_BakeStringMapLoose **bake_string_maps__in_progress = push_array(scratch.arena, RDIM_BakeStringMapLoose *, async_thread_count()); - ASYNC_TaskList bake_string_map_build_tasks = {0}; - { - // rjf: src files - ProfScope("kick off src files string map build task") - { - RDIM_BakeSrcFilesStringsIn *in = push_array(scratch.arena, RDIM_BakeSrcFilesStringsIn, 1); - in->top = &bake_string_map_topology; - in->maps = bake_string_maps__in_progress; - in->list = &in_params->src_files; - async_task_list_push(scratch.arena, &bake_string_map_build_tasks, async_task_launch(scratch.arena, rdim_bake_src_files_strings_work, .input = in)); - } - - // rjf: units - ProfScope("kick off units string map build task") - { - RDIM_BakeUnitsStringsIn *in = push_array(scratch.arena, RDIM_BakeUnitsStringsIn, 1); - in->top = &bake_string_map_topology; - in->maps = bake_string_maps__in_progress; - in->list = &in_params->units; - async_task_list_push(scratch.arena, &bake_string_map_build_tasks, async_task_launch(scratch.arena, rdim_bake_units_strings_work, .input = in)); - } - - // rjf: types - ProfScope("kick off types string map build tasks") - { - U64 items_per_task = 4096; - U64 num_tasks = (in_params->types.total_count+items_per_task-1)/items_per_task; - RDIM_TypeChunkNode *chunk = in_params->types.first; - U64 chunk_off = 0; - for(U64 task_idx = 0; task_idx < num_tasks; task_idx += 1) - { - RDIM_BakeTypesStringsIn *in = push_array(scratch.arena, RDIM_BakeTypesStringsIn, 1); - in->top = &bake_string_map_topology; - in->maps = bake_string_maps__in_progress; - U64 items_left = items_per_task; - for(;chunk != 0 && items_left > 0;) - { - U64 items_in_this_chunk = Min(items_per_task, chunk->count-chunk_off); - RDIM_BakeTypesStringsInNode *n = push_array(scratch.arena, RDIM_BakeTypesStringsInNode, 1); - SLLQueuePush(in->first, in->last, n); - n->v = chunk->v + chunk_off; - n->count = items_in_this_chunk; - chunk_off += items_in_this_chunk; - items_left -= items_in_this_chunk; - if(chunk_off >= chunk->count) - { - chunk = chunk->next; - chunk_off = 0; - } - } - async_task_list_push(scratch.arena, &bake_string_map_build_tasks, async_task_launch(scratch.arena, rdim_bake_types_strings_work, .input = in)); - } - } - - // rjf: UDTs - ProfScope("kick off udts string map build tasks") - { - U64 items_per_task = 4096; - U64 num_tasks = (in_params->udts.total_count+items_per_task-1)/items_per_task; - RDIM_UDTChunkNode *chunk = in_params->udts.first; - U64 chunk_off = 0; - for(U64 task_idx = 0; task_idx < num_tasks; task_idx += 1) - { - RDIM_BakeUDTsStringsIn *in = push_array(scratch.arena, RDIM_BakeUDTsStringsIn, 1); - in->top = &bake_string_map_topology; - in->maps = bake_string_maps__in_progress; - U64 items_left = items_per_task; - for(;chunk != 0 && items_left > 0;) - { - U64 items_in_this_chunk = Min(items_per_task, chunk->count-chunk_off); - RDIM_BakeUDTsStringsInNode *n = push_array(scratch.arena, RDIM_BakeUDTsStringsInNode, 1); - SLLQueuePush(in->first, in->last, n); - n->v = chunk->v + chunk_off; - n->count = items_in_this_chunk; - chunk_off += items_in_this_chunk; - items_left -= items_in_this_chunk; - if(chunk_off >= chunk->count) - { - chunk = chunk->next; - chunk_off = 0; - } - } - async_task_list_push(scratch.arena, &bake_string_map_build_tasks, async_task_launch(scratch.arena, rdim_bake_udts_strings_work, .input = in)); - } - } - - // rjf: symbols - ProfScope("kick off symbols string map build tasks") - { - RDIM_SymbolChunkList *symbol_lists[] = - { - &in_params->global_variables, - &in_params->thread_variables, - &in_params->constants, - &in_params->procedures, - }; - for(U64 list_idx = 0; list_idx < ArrayCount(symbol_lists); list_idx += 1) - { - U64 items_per_task = 4096; - U64 num_tasks = (symbol_lists[list_idx]->total_count+items_per_task-1)/items_per_task; - RDIM_SymbolChunkNode *chunk = symbol_lists[list_idx]->first; - U64 chunk_off = 0; - for(U64 task_idx = 0; task_idx < num_tasks; task_idx += 1) - { - RDIM_BakeSymbolsStringsIn *in = push_array(scratch.arena, RDIM_BakeSymbolsStringsIn, 1); - in->top = &bake_string_map_topology; - in->maps = bake_string_maps__in_progress; - U64 items_left = items_per_task; - for(;chunk != 0 && items_left > 0;) - { - U64 items_in_this_chunk = Min(items_per_task, chunk->count-chunk_off); - RDIM_BakeSymbolsStringsInNode *n = push_array(scratch.arena, RDIM_BakeSymbolsStringsInNode, 1); - SLLQueuePush(in->first, in->last, n); - n->v = chunk->v + chunk_off; - n->count = items_in_this_chunk; - chunk_off += items_in_this_chunk; - items_left -= items_in_this_chunk; - if(chunk_off >= chunk->count) - { - chunk = chunk->next; - chunk_off = 0; - } - } - async_task_list_push(scratch.arena, &bake_string_map_build_tasks, async_task_launch(scratch.arena, rdim_bake_symbols_strings_work, .input = in)); - } - } - } - - ProfScope("kick off inline site string map build task") - { - U64 items_per_task = 4096; - U64 num_tasks = CeilIntegerDiv(in_params->inline_sites.total_count, items_per_task); - RDIM_InlineSiteChunkNode *chunk = in_params->inline_sites.first; - U64 chunk_off = 0; - for(U64 task_idx = 0; task_idx < num_tasks; task_idx += 1) - { - RDIM_BakeInlineSiteStringsIn *in = push_array(scratch.arena, RDIM_BakeInlineSiteStringsIn, 1); - in->top = &bake_string_map_topology; - in->maps = bake_string_maps__in_progress; - U64 items_left = items_per_task; - for(;chunk != 0 && items_left > 0;) - { - U64 items_in_this_chunk = Min(items_per_task, chunk->count-chunk_off); - RDIM_BakeInlineSiteStringsInNode *n = push_array(scratch.arena, RDIM_BakeInlineSiteStringsInNode, 1); - SLLQueuePush(in->first, in->last, n); - n->v = chunk->v + chunk_off; - n->count = items_in_this_chunk; - chunk_off += items_in_this_chunk; - items_left -= items_in_this_chunk; - if(chunk_off >= chunk->count) - { - chunk = chunk->next; - chunk_off = 0; - } - } - async_task_list_push(scratch.arena, &bake_string_map_build_tasks, async_task_launch(scratch.arena, rdim_bake_inline_site_strings_work, .input = in)); - } - } - - // rjf: scope chunks - ProfScope("kick off scope chunks string map build tasks") - { - U64 items_per_task = 4096; - U64 num_tasks = (in_params->scopes.total_count+items_per_task-1)/items_per_task; - RDIM_ScopeChunkNode *chunk = in_params->scopes.first; - U64 chunk_off = 0; - for(U64 task_idx = 0; task_idx < num_tasks; task_idx += 1) - { - RDIM_BakeScopesStringsIn *in = push_array(scratch.arena, RDIM_BakeScopesStringsIn, 1); - in->top = &bake_string_map_topology; - in->maps = bake_string_maps__in_progress; - U64 items_left = items_per_task; - for(;chunk != 0 && items_left > 0;) - { - U64 items_in_this_chunk = Min(items_per_task, chunk->count-chunk_off); - RDIM_BakeScopesStringsInNode *n = push_array(scratch.arena, RDIM_BakeScopesStringsInNode, 1); - SLLQueuePush(in->first, in->last, n); - n->v = chunk->v + chunk_off; - n->count = items_in_this_chunk; - chunk_off += items_in_this_chunk; - items_left -= items_in_this_chunk; - if(chunk_off >= chunk->count) - { - chunk = chunk->next; - chunk_off = 0; - } - } - async_task_list_push(scratch.arena, &bake_string_map_build_tasks, async_task_launch(scratch.arena, rdim_bake_scopes_strings_work, .input = in)); - } - } - } - - ////////////////////////////// - //- rjf: kick off name map building tasks - // - RDIM_BuildBakeNameMapIn build_bake_name_map_in[RDI_NameMapKind_COUNT] = {0}; - ASYNC_Task *build_bake_name_map_task[RDI_NameMapKind_COUNT] = {0}; - for(RDI_NameMapKind k = (RDI_NameMapKind)(RDI_NameMapKind_NULL+1); - k < RDI_NameMapKind_COUNT; - k = (RDI_NameMapKind)(k+1)) - { - build_bake_name_map_in[k].k = k; - build_bake_name_map_in[k].type_indices = type_indices; - build_bake_name_map_in[k].params = in_params; - build_bake_name_map_task[k] = async_task_launch(scratch.arena, rdim_build_bake_name_map_work, .input = &build_bake_name_map_in[k]); - } - - ////////////////////////////// - //- rjf: join string map building tasks - // - ProfScope("join string map building tasks") - { - for(ASYNC_TaskNode *n = bake_string_map_build_tasks.first; n != 0; n = n->next) - { - async_task_join(n->v); - } - } - - ////////////////////////////// - //- rjf: produce joined string map - // - RDIM_BakeStringMapLoose *unsorted_bake_string_map = rdim_bake_string_map_loose_make(state->arena, &bake_string_map_topology); - ProfScope("produce joined string map") - { - U64 slots_per_task = 16384; - U64 num_tasks = (bake_string_map_topology.slots_count+slots_per_task-1)/slots_per_task; - ASYNC_Task **tasks = push_array(scratch.arena, ASYNC_Task *, num_tasks); - - // rjf: kickoff tasks - for(U64 task_idx = 0; task_idx < num_tasks; task_idx += 1) - { - RDIM_JoinBakeStringMapSlotsIn *in = push_array(scratch.arena, RDIM_JoinBakeStringMapSlotsIn, 1); - in->top = &bake_string_map_topology; - in->src_maps = bake_string_maps__in_progress; - in->src_maps_count = async_thread_count(); - in->dst_map = unsorted_bake_string_map; - in->slot_idx_range = r1u64(task_idx*slots_per_task, task_idx*slots_per_task + slots_per_task); - in->slot_idx_range.max = Min(in->slot_idx_range.max, in->top->slots_count); - tasks[task_idx] = async_task_launch(scratch.arena, rdim_bake_string_map_join_work, .input = in); - } - - // rjf: join tasks - for(U64 task_idx = 0; task_idx < num_tasks; task_idx += 1) - { - async_task_join(tasks[task_idx]); - } - - // rjf: insert small top-level stuff - rdim_bake_string_map_loose_push_top_level_info(state->arena, &bake_string_map_topology, unsorted_bake_string_map, &in_params->top_level_info); - rdim_bake_string_map_loose_push_binary_sections(state->arena, &bake_string_map_topology, unsorted_bake_string_map, &in_params->binary_sections); - rdim_bake_string_map_loose_push_path_tree(state->arena, &bake_string_map_topology, unsorted_bake_string_map, path_tree); - } - - ////////////////////////////// - //- rjf: kick off string map sorting tasks - // - ASYNC_TaskList sort_bake_string_map_tasks = {0}; - RDIM_BakeStringMapLoose *sorted_bake_string_map__in_progress = rdim_bake_string_map_loose_make(state->arena, &bake_string_map_topology); - { - U64 slots_per_task = 4096; - U64 num_tasks = (bake_string_map_topology.slots_count+slots_per_task-1)/slots_per_task; - for(U64 task_idx = 0; task_idx < num_tasks; task_idx += 1) - { - RDIM_SortBakeStringMapSlotsIn *in = push_array(scratch.arena, RDIM_SortBakeStringMapSlotsIn, 1); - { - in->top = &bake_string_map_topology; - in->src_map = unsorted_bake_string_map; - in->dst_map = sorted_bake_string_map__in_progress; - in->slot_idx = task_idx*slots_per_task; - in->slot_count = slots_per_task; - if(in->slot_idx+in->slot_count > bake_string_map_topology.slots_count) - { - in->slot_count = bake_string_map_topology.slots_count - in->slot_idx; - } - } - async_task_list_push(scratch.arena, &sort_bake_string_map_tasks, async_task_launch(scratch.arena, rdim_bake_string_map_sort_work, .input = in)); - } - } - - ////////////////////////////// - //- rjf: join string map sorting tasks - // - ProfScope("join string map sorting tasks") - { - for(ASYNC_TaskNode *n = sort_bake_string_map_tasks.first; n != 0; n = n->next) - { - async_task_join(n->v); - } - } - RDIM_BakeStringMapLoose *sorted_bake_string_map = sorted_bake_string_map__in_progress; - - ////////////////////////////// - //- rjf: build finalized string map - // - ProfBegin("build finalized string map base indices"); - RDIM_BakeStringMapBaseIndices bake_string_map_base_idxes = rdim_bake_string_map_base_indices_from_map_loose(state->arena, &bake_string_map_topology, sorted_bake_string_map); - ProfEnd(); - ProfBegin("build finalized string map"); - RDIM_BakeStringMapTight bake_strings = rdim_bake_string_map_tight_from_loose(state->arena, &bake_string_map_topology, &bake_string_map_base_idxes, sorted_bake_string_map); - ProfEnd(); - - ////////////////////////////// - //- rjf: kick off pass 2 tasks - // - RDIM_BakeUnitsIn bake_units_top_level_in = {&bake_strings, path_tree, &in_params->units}; - ASYNC_Task *bake_units_task = async_task_launch(scratch.arena, rdim_bake_units_work, .input = &bake_units_top_level_in); - RDIM_BakeUnitVMapIn bake_unit_vmap_in = {&in_params->units}; - ASYNC_Task *bake_unit_vmap_task = async_task_launch(scratch.arena, rdim_bake_unit_vmap_work, .input = &bake_unit_vmap_in); - RDIM_BakeSrcFilesIn bake_src_files_in = {&bake_strings, path_tree, &in_params->src_files}; - ASYNC_Task *bake_src_files_task = async_task_launch(scratch.arena, rdim_bake_src_files_work, .input = &bake_src_files_in); - RDIM_BakeUDTsIn bake_udts_in = {&bake_strings, &in_params->udts, type_indices}; - ASYNC_Task *bake_udts_task = async_task_launch(scratch.arena, rdim_bake_udts_work, .input = &bake_udts_in); - RDIM_BakeGlobalVMapIn bake_global_vmap_in = {&in_params->global_variables}; - ASYNC_Task *bake_global_vmap_task = async_task_launch(scratch.arena, rdim_bake_global_vmap_work, .input = &bake_global_vmap_in); - RDIM_BakeScopeVMapIn bake_scope_vmap_in = {&in_params->scopes}; - ASYNC_Task *bake_scope_vmap_task = async_task_launch(scratch.arena, rdim_bake_scope_vmap_work, .input = &bake_scope_vmap_in); - RDIM_BakeInlineSitesIn bake_inline_sites_in = {&bake_strings, &in_params->inline_sites, type_indices}; - ASYNC_Task *bake_inline_sites_task = async_task_launch(scratch.arena, rdim_bake_inline_sites_work, .input = &bake_inline_sites_in); - RDIM_BakeFilePathsIn bake_file_paths_in = {&bake_strings, path_tree}; - ASYNC_Task *bake_file_paths_task = async_task_launch(scratch.arena, rdim_bake_file_paths_work, .input = &bake_file_paths_in); - RDIM_BakeStringsIn bake_strings_in = {&bake_strings}; - ASYNC_Task *bake_strings_task = async_task_launch(scratch.arena, rdim_bake_strings_work, .input = &bake_strings_in); - - RDIM_String8List location_blocks = {0}; - RDIM_String8List location_data_blobs = {0}; - - // reserve null location block for opl - rdim_location_block_chunk_list_push_array(state->arena, &location_blocks, 1); - - // TODO: export location instead of VOFF - RDIM_BakeThreadVariablesIn bake_thread_variables_in = {&bake_strings, &in_params->thread_variables, type_indices}; - ASYNC_Task *bake_thread_variables_task = async_task_launch(scratch.arena, rdim_bake_thread_variables_work, .input = &bake_thread_variables_in); - ProfScope("thread variables") out.thread_variables = *async_task_join_struct(bake_thread_variables_task, RDIM_ThreadVariableBakeResult); - - // TODO: export location instead of VOFF - RDIM_BakeGlobalVariablesIn bake_global_variables_in = {&bake_strings, &in_params->global_variables, type_indices}; - ASYNC_Task *bake_global_variables_task = async_task_launch(scratch.arena, rdim_bake_global_variables_work, .input = &bake_global_variables_in); - ProfScope("global variables") out.global_variables = *async_task_join_struct(bake_global_variables_task, RDIM_GlobalVariableBakeResult); - - RDIM_BakeScopesIn bake_scopes_in = {&bake_strings, &in_params->scopes, type_indices, &location_blocks, &location_data_blobs}; - ASYNC_Task *bake_scopes_task = async_task_launch(scratch.arena, rdim_bake_scopes_work, .input = &bake_scopes_in); - ProfScope("scopes") out.scopes = *async_task_join_struct(bake_scopes_task, RDIM_ScopeBakeResult); - - RDIM_BakeProceduresIn bake_procedures_in = {&bake_strings, &in_params->procedures, type_indices, &location_blocks, &location_data_blobs}; - ASYNC_Task *bake_procedures_task = async_task_launch(scratch.arena, rdim_bake_procedures_work, .input = &bake_procedures_in); - ProfScope("procedures") out.procedures = *async_task_join_struct(bake_procedures_task, RDIM_ProcedureBakeResult); - - ////////////////////////////// - //- rjf: join name map building tasks - // - RDIM_BakeNameMap *name_maps[RDI_NameMapKind_COUNT] = {0}; - ProfScope("join name map building tasks") - { - for(RDI_NameMapKind k = (RDI_NameMapKind)(RDI_NameMapKind_NULL+1); - k < RDI_NameMapKind_COUNT; - k = (RDI_NameMapKind)(k+1)) - { - name_maps[k] = async_task_join_struct(build_bake_name_map_task[k], RDIM_BakeNameMap); - } - } - - ////////////////////////////// - //- rjf: build interned idx run map - // - RDIM_BakeIdxRunMap *idx_runs = 0; - ProfScope("build interned idx run map") - { - idx_runs = rdim_bake_idx_run_map_from_params(state->arena, name_maps, type_indices, in_params); - } - - ////////////////////////////// - //- rjf: do small top-level bakes - // - ProfScope("top level info") out.top_level_info = rdim_bake_top_level_info(state->arena, &bake_strings, &in_params->top_level_info); - ProfScope("binary sections") out.binary_sections = rdim_bake_binary_sections(state->arena, &bake_strings, &in_params->binary_sections); - ProfScope("top level name maps section") out.top_level_name_maps = rdim_bake_name_maps_top_level(state->arena, &bake_strings, idx_runs, name_maps); - - ////////////////////////////// - //- rjf: kick off pass 3 tasks - // - RDIM_BakeTypeNodesIn bake_type_nodes_in = {&bake_strings, idx_runs, &in_params->types, type_indices}; - ASYNC_Task *bake_type_nodes_task = async_task_launch(scratch.arena, rdim_bake_type_nodes_work, .input = &bake_type_nodes_in); - ASYNC_Task *bake_name_maps_tasks[RDI_NameMapKind_COUNT] = {0}; - { - for EachNonZeroEnumVal(RDI_NameMapKind, k) - { - if(name_maps[k] == 0 || name_maps[k]->name_count == 0) - { - continue; - } - RDIM_BakeNameMapIn *in = push_array(scratch.arena, RDIM_BakeNameMapIn, 1); - in->strings = &bake_strings; - in->idx_runs = idx_runs; - in->map = name_maps[k]; - in->kind = k; - bake_name_maps_tasks[k] = async_task_launch(scratch.arena, rdim_bake_name_map_work, .input = in); - } - } - RDIM_BakeIdxRunsIn bake_idx_runs_in = {idx_runs}; - ASYNC_Task *bake_idx_runs_task = async_task_launch(scratch.arena, rdim_bake_idx_runs_work, .input = &bake_idx_runs_in); - - ////////////////////////////// - //- rjf: join remaining completed bakes - // - ProfScope("top-level units info") out.units = *async_task_join_struct(bake_units_task, RDIM_UnitBakeResult); - ProfScope("unit vmap") out.unit_vmap = *async_task_join_struct(bake_unit_vmap_task, RDIM_UnitVMapBakeResult); - ProfScope("source files") out.src_files = *async_task_join_struct(bake_src_files_task, RDIM_SrcFileBakeResult); - ProfScope("UDTs") out.udts = *async_task_join_struct(bake_udts_task, RDIM_UDTBakeResult); - ProfScope("global vmap") out.global_vmap = *async_task_join_struct(bake_global_vmap_task, RDIM_GlobalVMapBakeResult); - ProfScope("scope vmap") out.scope_vmap = *async_task_join_struct(bake_scope_vmap_task, RDIM_ScopeVMapBakeResult); - ProfScope("inline sites") out.inline_sites = *async_task_join_struct(bake_inline_sites_task, RDIM_InlineSiteBakeResult); - ProfScope("file paths") out.file_paths = *async_task_join_struct(bake_file_paths_task, RDIM_FilePathBakeResult); - ProfScope("strings") out.strings = *async_task_join_struct(bake_strings_task, RDIM_StringBakeResult); - ProfScope("type nodes") out.type_nodes = *async_task_join_struct(bake_type_nodes_task, RDIM_TypeNodeBakeResult); - ProfScope("idx runs") out.idx_runs = *async_task_join_struct(bake_idx_runs_task, RDIM_IndexRunBakeResult); - ProfScope("line tables") out.line_tables = *async_task_join_struct(bake_line_tables_task, RDIM_LineTableBakeResult); - - ////////////////////////////// - //- rjf: join individual name map bakes - // - RDIM_NameMapBakeResult name_map_bakes[RDI_NameMapKind_COUNT] = {0}; - ProfScope("name maps") - { - for EachNonZeroEnumVal(RDI_NameMapKind, k) - { - RDIM_NameMapBakeResult *bake = async_task_join_struct(bake_name_maps_tasks[k], RDIM_NameMapBakeResult); - if(bake != 0) - { - name_map_bakes[k] = *bake; - } - } - } - - ////////////////////////////// - //- rjf: join all individual name map bakes - // - ProfScope("join all name map bakes into final name map bake") - { - out.name_maps = rdim_name_map_bake_results_combine(state->arena, name_map_bakes, ArrayCount(name_map_bakes)); - } - - - //////////////////////////////// - - out.location_blocks = rdim_str8_list_join(state->arena, &location_blocks, rdim_str8(0,0)); - out.location_data = rdim_str8_list_join(state->arena, &location_data_blobs, rdim_str8(0,0)); - - rdim_help_state = 0; - - scratch_end(scratch); - return out; -} - -internal RDIM_SerializedSectionBundle -rdim_compress(Arena *arena, RDIM_SerializedSectionBundle *in) -{ - RDIM_SerializedSectionBundle out = {0}; - - //- rjf: set up compression context - rr_lzb_simple_context ctx = {0}; - ctx.m_tableSizeBits = 14; - ctx.m_hashTable = push_array(arena, U16, 1<sections[k]; - RDIM_SerializedSection *dst = &out.sections[k]; - MemoryCopyStruct(dst, src); - - // rjf: determine if this section should be compressed - B32 should_compress = 1; - - // rjf: compress if needed - if(should_compress) - { - MemoryZero(ctx.m_hashTable, sizeof(U16)*(1<data = push_array_no_zero(arena, U8, src->encoded_size); - dst->encoded_size = rr_lzb_simple_encode_veryfast(&ctx, src->data, src->encoded_size, dst->data); - dst->unpacked_size = src->encoded_size; - dst->encoding = RDI_SectionEncoding_LZB; - } - } - - return out; -} diff --git a/src/rdi_make/rdi_make_help.h b/src/rdi_make/rdi_make_help.h deleted file mode 100644 index 59a4320b..00000000 --- a/src/rdi_make/rdi_make_help.h +++ /dev/null @@ -1,342 +0,0 @@ -// Copyright (c) Epic Games Tools -// Licensed under the MIT license (https://opensource.org/license/mit/) - -#ifndef RDIM_MAKE_HELP -#define RDIM_MAKE_HELP - -//- rjf: line table baking task types - -typedef struct RDIM_BakeLineTablesIn RDIM_BakeLineTablesIn; -struct RDIM_BakeLineTablesIn -{ - RDIM_LineTableChunkList *line_tables; -}; - -//- rjf: string map baking task types - -typedef struct RDIM_BakeSrcFilesStringsIn RDIM_BakeSrcFilesStringsIn; -struct RDIM_BakeSrcFilesStringsIn -{ - RDIM_BakeStringMapTopology *top; - RDIM_BakeStringMapLoose **maps; - RDIM_SrcFileChunkList *list; -}; - -typedef struct RDIM_BakeUnitsStringsIn RDIM_BakeUnitsStringsIn; -struct RDIM_BakeUnitsStringsIn -{ - RDIM_BakeStringMapTopology *top; - RDIM_BakeStringMapLoose **maps; - RDIM_UnitChunkList *list; -}; - -typedef struct RDIM_BakeUDTsStringsInNode RDIM_BakeUDTsStringsInNode; -struct RDIM_BakeUDTsStringsInNode -{ - RDIM_BakeUDTsStringsInNode *next; - RDIM_UDT *v; - RDI_U64 count; -}; - -typedef struct RDIM_BakeTypesStringsInNode RDIM_BakeTypesStringsInNode; -struct RDIM_BakeTypesStringsInNode -{ - RDIM_BakeTypesStringsInNode *next; - RDIM_Type *v; - RDI_U64 count; -}; - -typedef struct RDIM_BakeTypesStringsIn RDIM_BakeTypesStringsIn; -struct RDIM_BakeTypesStringsIn -{ - RDIM_BakeStringMapTopology *top; - RDIM_BakeStringMapLoose **maps; - RDIM_BakeTypesStringsInNode *first; - RDIM_BakeTypesStringsInNode *last; -}; - -typedef struct RDIM_BakeUDTsStringsIn RDIM_BakeUDTsStringsIn; -struct RDIM_BakeUDTsStringsIn -{ - RDIM_BakeStringMapTopology *top; - RDIM_BakeStringMapLoose **maps; - RDIM_BakeUDTsStringsInNode *first; - RDIM_BakeUDTsStringsInNode *last; -}; - -typedef struct RDIM_BakeSymbolsStringsInNode RDIM_BakeSymbolsStringsInNode; -struct RDIM_BakeSymbolsStringsInNode -{ - RDIM_BakeSymbolsStringsInNode *next; - RDIM_Symbol *v; - RDI_U64 count; -}; - -typedef struct RDIM_BakeSymbolsStringsIn RDIM_BakeSymbolsStringsIn; -struct RDIM_BakeSymbolsStringsIn -{ - RDIM_BakeStringMapTopology *top; - RDIM_BakeStringMapLoose **maps; - RDIM_BakeSymbolsStringsInNode *first; - RDIM_BakeSymbolsStringsInNode *last; -}; - -typedef struct RDIM_BakeInlineSiteStringsInNode RDIM_BakeInlineSiteStringsInNode; -struct RDIM_BakeInlineSiteStringsInNode -{ - RDIM_BakeInlineSiteStringsInNode *next; - RDIM_InlineSite *v; - RDI_U64 count; -}; - -typedef struct RDIM_BakeInlineSiteStringsIn RDIM_BakeInlineSiteStringsIn; -struct RDIM_BakeInlineSiteStringsIn -{ - RDIM_BakeStringMapTopology *top; - RDIM_BakeStringMapLoose **maps; - RDIM_BakeInlineSiteStringsInNode *first; - RDIM_BakeInlineSiteStringsInNode *last; -}; - -typedef struct RDIM_BakeScopesStringsInNode RDIM_BakeScopesStringsInNode; -struct RDIM_BakeScopesStringsInNode -{ - RDIM_BakeScopesStringsInNode *next; - RDIM_Scope *v; - RDI_U64 count; -}; - -typedef struct RDIM_BakeScopesStringsIn RDIM_BakeScopesStringsIn; -struct RDIM_BakeScopesStringsIn -{ - RDIM_BakeStringMapTopology *top; - RDIM_BakeStringMapLoose **maps; - RDIM_BakeScopesStringsInNode *first; - RDIM_BakeScopesStringsInNode *last; -}; - -//- rjf: OLD string map baking types - -typedef struct RDIM_BuildBakeStringMapIn RDIM_BuildBakeStringMapIn; -struct RDIM_BuildBakeStringMapIn -{ - RDIM_BakePathTree *path_tree; - RDIM_BakeParams *params; -}; - -typedef struct RDIM_BuildBakeNameMapIn RDIM_BuildBakeNameMapIn; -struct RDIM_BuildBakeNameMapIn -{ - RDI_NameMapKind k; - RDI_U64 *type_indices; - RDIM_BakeParams *params; -}; - -//- rjf: string map joining task types - -typedef struct RDIM_JoinBakeStringMapSlotsIn RDIM_JoinBakeStringMapSlotsIn; -struct RDIM_JoinBakeStringMapSlotsIn -{ - RDIM_BakeStringMapTopology *top; - RDIM_BakeStringMapLoose **src_maps; - U64 src_maps_count; - RDIM_BakeStringMapLoose *dst_map; - Rng1U64 slot_idx_range; -}; - -//- rjf: string map sorting task types - -typedef struct RDIM_SortBakeStringMapSlotsIn RDIM_SortBakeStringMapSlotsIn; -struct RDIM_SortBakeStringMapSlotsIn -{ - RDIM_BakeStringMapTopology *top; - RDIM_BakeStringMapLoose *src_map; - RDIM_BakeStringMapLoose *dst_map; - U64 slot_idx; - U64 slot_count; -}; - -//- rjf: debug info baking task types - -typedef struct RDIM_BakeUnitsIn RDIM_BakeUnitsIn; -struct RDIM_BakeUnitsIn -{ - RDIM_BakeStringMapTight *strings; - RDIM_BakePathTree *path_tree; - RDIM_UnitChunkList *units; -}; - -typedef struct RDIM_BakeUnitVMapIn RDIM_BakeUnitVMapIn; -struct RDIM_BakeUnitVMapIn -{ - RDIM_UnitChunkList *units; -}; - -typedef struct RDIM_BakeSrcFilesIn RDIM_BakeSrcFilesIn; -struct RDIM_BakeSrcFilesIn -{ - RDIM_BakeStringMapTight *strings; - RDIM_BakePathTree *path_tree; - RDIM_SrcFileChunkList *src_files; -}; - -typedef struct RDIM_BakeUDTsIn RDIM_BakeUDTsIn; -struct RDIM_BakeUDTsIn -{ - RDIM_BakeStringMapTight *strings; - RDIM_UDTChunkList *udts; - RDI_U64 *type_indices; -}; - -typedef struct RDIM_BakeGlobalVariablesIn RDIM_BakeGlobalVariablesIn; -struct RDIM_BakeGlobalVariablesIn -{ - RDIM_BakeStringMapTight *strings; - RDIM_SymbolChunkList *global_variables; - RDI_U64 *type_indices; -}; - -typedef struct RDIM_BakeGlobalVMapIn RDIM_BakeGlobalVMapIn; -struct RDIM_BakeGlobalVMapIn -{ - RDIM_SymbolChunkList *global_variables; -}; - -typedef struct RDIM_BakeThreadVariablesIn RDIM_BakeThreadVariablesIn; -struct RDIM_BakeThreadVariablesIn -{ - RDIM_BakeStringMapTight *strings; - RDIM_SymbolChunkList *thread_variables; - RDI_U64 *type_indices; -}; - -typedef struct RDIM_BakeProceduresIn RDIM_BakeProceduresIn; -struct RDIM_BakeProceduresIn -{ - RDIM_BakeStringMapTight *strings; - RDIM_SymbolChunkList *procedures; - RDI_U64 *type_indices; - RDIM_String8List *location_blocks; - RDIM_String8List *location_data_blobs; -}; - -typedef struct RDIM_BakeScopesIn RDIM_BakeScopesIn; -struct RDIM_BakeScopesIn -{ - RDIM_BakeStringMapTight *strings; - RDIM_ScopeChunkList *scopes; - RDI_U64 *type_indices; - RDIM_String8List *location_blocks; - RDIM_String8List *location_data_blobs; -}; - -typedef struct RDIM_BakeScopeVMapIn RDIM_BakeScopeVMapIn; -struct RDIM_BakeScopeVMapIn -{ - RDIM_ScopeChunkList *scopes; -}; - -typedef struct RDIM_BakeInlineSitesIn RDIM_BakeInlineSitesIn; -struct RDIM_BakeInlineSitesIn -{ - RDIM_BakeStringMapTight *strings; - RDIM_InlineSiteChunkList *inline_sites; - RDI_U64 *type_indices; -}; - -typedef struct RDIM_BakeFilePathsIn RDIM_BakeFilePathsIn; -struct RDIM_BakeFilePathsIn -{ - RDIM_BakeStringMapTight *strings; - RDIM_BakePathTree *path_tree; -}; - -typedef struct RDIM_BakeStringsIn RDIM_BakeStringsIn; -struct RDIM_BakeStringsIn -{ - RDIM_BakeStringMapTight *strings; -}; - -typedef struct RDIM_BakeTypeNodesIn RDIM_BakeTypeNodesIn; -struct RDIM_BakeTypeNodesIn -{ - RDIM_BakeStringMapTight *strings; - RDIM_BakeIdxRunMap *idx_runs; - RDIM_TypeChunkList *types; - RDI_U64 *type_indices; -}; - -typedef struct RDIM_BakeNameMapIn RDIM_BakeNameMapIn; -struct RDIM_BakeNameMapIn -{ - RDIM_BakeStringMapTight *strings; - RDIM_BakeIdxRunMap *idx_runs; - RDIM_BakeNameMap *map; - RDI_NameMapKind kind; -}; - -typedef struct RDIM_BakeIdxRunsIn RDIM_BakeIdxRunsIn; -struct RDIM_BakeIdxRunsIn -{ - RDIM_BakeIdxRunMap *idx_runs; -}; - -//////////////////////////////// -//~ rjf: Baking Stage Tasks - -//- rjf: unsorted bake string map building -ASYNC_WORK_DEF(p2r_bake_src_files_strings_work); -ASYNC_WORK_DEF(p2r_bake_units_strings_work); -ASYNC_WORK_DEF(p2r_bake_types_strings_work); -ASYNC_WORK_DEF(p2r_bake_udts_strings_work); -ASYNC_WORK_DEF(p2r_bake_symbols_strings_work); -ASYNC_WORK_DEF(p2r_bake_scopes_strings_work); -ASYNC_WORK_DEF(p2r_bake_line_tables_work); - -//- rjf: bake string map joining -ASYNC_WORK_DEF(p2r_bake_string_map_join_work); - -//- rjf: bake string map sorting -ASYNC_WORK_DEF(p2r_bake_string_map_sort_work); - -//- rjf: pass 1: interner/deduper map builds -ASYNC_WORK_DEF(p2r_build_bake_name_map_work); - -//- rjf: pass 2: string-map-dependent debug info stream builds -ASYNC_WORK_DEF(p2r_bake_units_work); -ASYNC_WORK_DEF(p2r_bake_unit_vmap_work); -ASYNC_WORK_DEF(p2r_bake_src_files_work); -ASYNC_WORK_DEF(p2r_bake_udts_work); -ASYNC_WORK_DEF(p2r_bake_global_variables_work); -ASYNC_WORK_DEF(p2r_bake_global_vmap_work); -ASYNC_WORK_DEF(p2r_bake_thread_variables_work); -ASYNC_WORK_DEF(p2r_bake_procedures_work); -ASYNC_WORK_DEF(p2r_bake_scopes_work); -ASYNC_WORK_DEF(p2r_bake_scope_vmap_work); -ASYNC_WORK_DEF(p2r_bake_file_paths_work); -ASYNC_WORK_DEF(p2r_bake_strings_work); - -//- rjf: pass 3: idx-run-map-dependent debug info stream builds -ASYNC_WORK_DEF(p2r_bake_type_nodes_work); -ASYNC_WORK_DEF(p2r_bake_name_map_work); -ASYNC_WORK_DEF(p2r_bake_idx_runs_work); - -typedef struct RDIM_HelpState RDIM_HelpState; -struct RDIM_HelpState -{ - Arena *arena; - U64 work_thread_arenas_count; - Arena **work_thread_arenas; -}; - -//////////////////////////////// - -global RDIM_HelpState *rdim_help_state = 0; - -//////////////////////////////// - -internal RDIM_HelpState * rdim_help_init(void); -internal RDIM_BakeResults rdim_bake(RDIM_HelpState *state, RDIM_BakeParams *in); -internal RDIM_SerializedSectionBundle rdim_compress(Arena *arena, RDIM_SerializedSectionBundle *in); - -#endif // RDIM_MAKE_HELP diff --git a/src/rdi_make/rdi_make_local.c b/src/rdi_make/rdi_make_local.c index 10635558..dd75baf1 100644 --- a/src/rdi_make/rdi_make_local.c +++ b/src/rdi_make/rdi_make_local.c @@ -62,7 +62,7 @@ rdim_make_top_level_info(String8 image_name, Arch arch, U64 exe_hash, RDIM_Binar ASYNC_WORK_DEF(rdim_bake_src_files_strings_work) { ProfBeginFunction(); - Arena *arena = rdim_local_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(rdim_local_async_root); RDIM_BakeSrcFilesStringsIn *in = (RDIM_BakeSrcFilesStringsIn *)input; rdim_make_string_map_if_needed(); ProfScope("bake src file strings") rdim_bake_string_map_loose_push_src_files(arena, in->top, in->maps[thread_idx], in->list); @@ -73,7 +73,7 @@ ASYNC_WORK_DEF(rdim_bake_src_files_strings_work) ASYNC_WORK_DEF(rdim_bake_units_strings_work) { ProfBeginFunction(); - Arena *arena = rdim_local_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(rdim_local_async_root); RDIM_BakeUnitsStringsIn *in = (RDIM_BakeUnitsStringsIn *)input; rdim_make_string_map_if_needed(); ProfScope("bake unit strings") rdim_bake_string_map_loose_push_units(arena, in->top, in->maps[thread_idx], in->list); @@ -84,7 +84,7 @@ ASYNC_WORK_DEF(rdim_bake_units_strings_work) ASYNC_WORK_DEF(rdim_bake_types_strings_work) { ProfBeginFunction(); - Arena *arena = rdim_local_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(rdim_local_async_root); RDIM_BakeTypesStringsIn *in = (RDIM_BakeTypesStringsIn *)input; rdim_make_string_map_if_needed(); ProfScope("bake type strings") @@ -101,7 +101,7 @@ ASYNC_WORK_DEF(rdim_bake_types_strings_work) ASYNC_WORK_DEF(rdim_bake_udts_strings_work) { ProfBeginFunction(); - Arena *arena = rdim_local_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(rdim_local_async_root); RDIM_BakeUDTsStringsIn *in = (RDIM_BakeUDTsStringsIn *)input; rdim_make_string_map_if_needed(); ProfScope("bake udt strings") @@ -118,7 +118,7 @@ ASYNC_WORK_DEF(rdim_bake_udts_strings_work) ASYNC_WORK_DEF(rdim_bake_symbols_strings_work) { ProfBeginFunction(); - Arena *arena = rdim_local_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(rdim_local_async_root); RDIM_BakeSymbolsStringsIn *in = (RDIM_BakeSymbolsStringsIn *)input; rdim_make_string_map_if_needed(); ProfScope("bake symbol strings") @@ -135,7 +135,7 @@ ASYNC_WORK_DEF(rdim_bake_symbols_strings_work) ASYNC_WORK_DEF(rdim_bake_inline_site_strings_work) { ProfBeginFunction(); - Arena *arena = rdim_local_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(rdim_local_async_root); RDIM_BakeInlineSiteStringsIn *in = input; rdim_make_string_map_if_needed(); ProfScope("bake inline site strings") @@ -152,7 +152,7 @@ ASYNC_WORK_DEF(rdim_bake_inline_site_strings_work) ASYNC_WORK_DEF(rdim_bake_scopes_strings_work) { ProfBeginFunction(); - Arena *arena = rdim_local_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(rdim_local_async_root); RDIM_BakeScopesStringsIn *in = (RDIM_BakeScopesStringsIn *)input; rdim_make_string_map_if_needed(); ProfScope("bake scope strings") @@ -169,7 +169,7 @@ ASYNC_WORK_DEF(rdim_bake_scopes_strings_work) ASYNC_WORK_DEF(rdim_bake_line_tables_work) { ProfBeginFunction(); - Arena *arena = rdim_local_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(rdim_local_async_root); RDIM_BakeLineTablesIn *in = (RDIM_BakeLineTablesIn *)input; RDIM_LineTableBakeResult *out = push_array(arena, RDIM_LineTableBakeResult, 1); ProfScope("bake line tables") *out = rdim_bake_line_tables(arena, in->line_tables); @@ -184,7 +184,7 @@ ASYNC_WORK_DEF(rdim_bake_line_tables_work) ASYNC_WORK_DEF(rdim_bake_string_map_join_work) { ProfBeginFunction(); - Arena *arena = rdim_local_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(rdim_local_async_root); RDIM_JoinBakeStringMapSlotsIn *in = (RDIM_JoinBakeStringMapSlotsIn *)input; ProfScope("join bake string maps") { @@ -214,7 +214,7 @@ ASYNC_WORK_DEF(rdim_bake_string_map_join_work) ASYNC_WORK_DEF(rdim_bake_string_map_sort_work) { ProfBeginFunction(); - Arena *arena = rdim_local_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(rdim_local_async_root); RDIM_SortBakeStringMapSlotsIn *in = (RDIM_SortBakeStringMapSlotsIn *)input; ProfScope("sort bake string chunk list map range") { @@ -245,7 +245,7 @@ ASYNC_WORK_DEF(rdim_bake_string_map_sort_work) ASYNC_WORK_DEF(rdim_build_bake_name_map_work) { ProfBeginFunction(); - Arena *arena = rdim_local_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(rdim_local_async_root); RDIM_BuildBakeNameMapIn *in = (RDIM_BuildBakeNameMapIn *)input; RDIM_BakeNameMap *name_map = 0; ProfScope("build name map %i", in->k) name_map = rdim_bake_name_map_from_kind_params(arena, in->k, in->params); @@ -258,7 +258,7 @@ ASYNC_WORK_DEF(rdim_build_bake_name_map_work) ASYNC_WORK_DEF(rdim_bake_units_work) { ProfBeginFunction(); - Arena *arena = rdim_local_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(rdim_local_async_root); RDIM_BakeUnitsIn *in = (RDIM_BakeUnitsIn *)input; RDIM_UnitBakeResult *out = push_array(arena, RDIM_UnitBakeResult, 1); ProfScope("bake units") *out = rdim_bake_units(arena, in->strings, in->path_tree, in->units); @@ -269,7 +269,7 @@ ASYNC_WORK_DEF(rdim_bake_units_work) ASYNC_WORK_DEF(rdim_bake_unit_vmap_work) { ProfBeginFunction(); - Arena *arena = rdim_local_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(rdim_local_async_root); RDIM_BakeUnitVMapIn *in = (RDIM_BakeUnitVMapIn *)input; RDIM_UnitVMapBakeResult *out = push_array(arena, RDIM_UnitVMapBakeResult, 1); ProfScope("bake unit vmap") *out = rdim_bake_unit_vmap(arena, in->units); @@ -280,7 +280,7 @@ ASYNC_WORK_DEF(rdim_bake_unit_vmap_work) ASYNC_WORK_DEF(rdim_bake_src_files_work) { ProfBeginFunction(); - Arena *arena = rdim_local_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(rdim_local_async_root); RDIM_BakeSrcFilesIn *in = (RDIM_BakeSrcFilesIn *)input; RDIM_SrcFileBakeResult *out = push_array(arena, RDIM_SrcFileBakeResult, 1); ProfScope("bake src files") *out = rdim_bake_src_files(arena, in->strings, in->path_tree, in->src_files); @@ -291,7 +291,7 @@ ASYNC_WORK_DEF(rdim_bake_src_files_work) ASYNC_WORK_DEF(rdim_bake_udts_work) { ProfBeginFunction(); - Arena *arena = rdim_local_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(rdim_local_async_root); RDIM_BakeUDTsIn *in = (RDIM_BakeUDTsIn *)input; RDIM_UDTBakeResult *out = push_array(arena, RDIM_UDTBakeResult, 1); ProfScope("bake udts") *out = rdim_bake_udts(arena, in->strings, in->udts); @@ -302,7 +302,7 @@ ASYNC_WORK_DEF(rdim_bake_udts_work) ASYNC_WORK_DEF(rdim_bake_global_variables_work) { ProfBeginFunction(); - Arena *arena = rdim_local_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(rdim_local_async_root); RDIM_BakeGlobalVariablesIn *in = (RDIM_BakeGlobalVariablesIn *)input; RDIM_GlobalVariableBakeResult *out = push_array(arena, RDIM_GlobalVariableBakeResult, 1); ProfScope("bake global variables") *out = rdim_bake_global_variables(arena, in->strings, in->global_variables); @@ -313,7 +313,7 @@ ASYNC_WORK_DEF(rdim_bake_global_variables_work) ASYNC_WORK_DEF(rdim_bake_global_vmap_work) { ProfBeginFunction(); - Arena *arena = rdim_local_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(rdim_local_async_root); RDIM_BakeGlobalVMapIn *in = (RDIM_BakeGlobalVMapIn *)input; RDIM_GlobalVMapBakeResult *out = push_array(arena, RDIM_GlobalVMapBakeResult, 1); ProfScope("bake global vmap") *out = rdim_bake_global_vmap(arena, in->global_variables); @@ -324,7 +324,7 @@ ASYNC_WORK_DEF(rdim_bake_global_vmap_work) ASYNC_WORK_DEF(rdim_bake_thread_variables_work) { ProfBeginFunction(); - Arena *arena = rdim_local_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(rdim_local_async_root); RDIM_BakeThreadVariablesIn *in = (RDIM_BakeThreadVariablesIn *)input; RDIM_ThreadVariableBakeResult *out = push_array(arena, RDIM_ThreadVariableBakeResult, 1); ProfScope("bake thread variables") *out = rdim_bake_thread_variables(arena, in->strings, in->thread_variables); @@ -335,7 +335,7 @@ ASYNC_WORK_DEF(rdim_bake_thread_variables_work) ASYNC_WORK_DEF(rdim_bake_constants_work) { ProfBeginFunction(); - Arena *arena = rdim_local_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(rdim_local_async_root); RDIM_BakeConstantsIn *in = (RDIM_BakeConstantsIn *)input; RDIM_ConstantsBakeResult *out = push_array(arena, RDIM_ConstantsBakeResult, 1); ProfScope("bake constants") *out = rdim_bake_constants(arena, in->strings, in->constants); @@ -346,7 +346,7 @@ ASYNC_WORK_DEF(rdim_bake_constants_work) ASYNC_WORK_DEF(rdim_bake_procedures_work) { ProfBeginFunction(); - Arena *arena = rdim_local_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(rdim_local_async_root); RDIM_BakeProceduresIn *in = (RDIM_BakeProceduresIn *)input; RDIM_ProcedureBakeResult *out = push_array(arena, RDIM_ProcedureBakeResult, 1); ProfScope("bake procedures") *out = rdim_bake_procedures(arena, in->strings, in->location_blocks, in->location_data_blobs, in->procedures); @@ -357,7 +357,7 @@ ASYNC_WORK_DEF(rdim_bake_procedures_work) ASYNC_WORK_DEF(rdim_bake_scopes_work) { ProfBeginFunction(); - Arena *arena = rdim_local_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(rdim_local_async_root); RDIM_BakeScopesIn *in = (RDIM_BakeScopesIn *)input; RDIM_ScopeBakeResult *out = push_array(arena, RDIM_ScopeBakeResult, 1); ProfScope("bake scopes") *out = rdim_bake_scopes(arena, in->strings, in->location_blocks, in->location_data_blobs, in->scopes); @@ -368,7 +368,7 @@ ASYNC_WORK_DEF(rdim_bake_scopes_work) ASYNC_WORK_DEF(rdim_bake_scope_vmap_work) { ProfBeginFunction(); - Arena *arena = rdim_local_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(rdim_local_async_root); RDIM_BakeScopeVMapIn *in = (RDIM_BakeScopeVMapIn *)input; RDIM_ScopeVMapBakeResult *out = push_array(arena, RDIM_ScopeVMapBakeResult, 1); ProfScope("bake scope vmap") *out = rdim_bake_scope_vmap(arena, in->scopes); @@ -379,7 +379,7 @@ ASYNC_WORK_DEF(rdim_bake_scope_vmap_work) ASYNC_WORK_DEF(rdim_bake_inline_sites_work) { ProfBeginFunction(); - Arena *arena = rdim_local_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(rdim_local_async_root); RDIM_BakeInlineSitesIn *in = (RDIM_BakeInlineSitesIn *)input; RDIM_InlineSiteBakeResult *out = push_array(arena, RDIM_InlineSiteBakeResult, 1); ProfScope("bake inline sites") *out = rdim_bake_inline_sites(arena, in->strings, in->inline_sites); @@ -390,7 +390,7 @@ ASYNC_WORK_DEF(rdim_bake_inline_sites_work) ASYNC_WORK_DEF(rdim_bake_file_paths_work) { ProfBeginFunction(); - Arena *arena = rdim_local_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(rdim_local_async_root); RDIM_BakeFilePathsIn *in = (RDIM_BakeFilePathsIn *)input; RDIM_FilePathBakeResult *out = push_array(arena, RDIM_FilePathBakeResult, 1); ProfScope("bake file paths") *out = rdim_bake_file_paths(arena, in->strings, in->path_tree); @@ -401,7 +401,7 @@ ASYNC_WORK_DEF(rdim_bake_file_paths_work) ASYNC_WORK_DEF(rdim_bake_strings_work) { ProfBeginFunction(); - Arena *arena = rdim_local_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(rdim_local_async_root); RDIM_BakeStringsIn *in = (RDIM_BakeStringsIn *)input; RDIM_StringBakeResult *out = push_array(arena, RDIM_StringBakeResult, 1); ProfScope("bake strings") *out = rdim_bake_strings(arena, in->strings); @@ -414,7 +414,7 @@ ASYNC_WORK_DEF(rdim_bake_strings_work) ASYNC_WORK_DEF(rdim_bake_type_nodes_work) { ProfBeginFunction(); - Arena *arena = rdim_local_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(rdim_local_async_root); RDIM_BakeTypeNodesIn *in = (RDIM_BakeTypeNodesIn *)input; RDIM_TypeNodeBakeResult *out = push_array(arena, RDIM_TypeNodeBakeResult, 1); ProfScope("bake type nodes") *out = rdim_bake_types(arena, in->strings, in->idx_runs, in->types); @@ -425,7 +425,7 @@ ASYNC_WORK_DEF(rdim_bake_type_nodes_work) ASYNC_WORK_DEF(rdim_bake_name_map_work) { ProfBeginFunction(); - Arena *arena = rdim_local_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(rdim_local_async_root); RDIM_BakeNameMapIn *in = (RDIM_BakeNameMapIn *)input; RDIM_NameMapBakeResult *out = push_array(arena, RDIM_NameMapBakeResult, 1); ProfScope("bake name map %i", in->kind) *out = rdim_bake_name_map(arena, in->strings, in->idx_runs, in->map); @@ -436,7 +436,7 @@ ASYNC_WORK_DEF(rdim_bake_name_map_work) ASYNC_WORK_DEF(rdim_bake_idx_runs_work) { ProfBeginFunction(); - Arena *arena = rdim_local_state->work_thread_arenas[thread_idx]; + Arena *arena = async_root_thread_arena(rdim_local_async_root); RDIM_BakeIdxRunsIn *in = (RDIM_BakeIdxRunsIn *)input; RDIM_IndexRunBakeResult *out = push_array(arena, RDIM_IndexRunBakeResult, 1); ProfScope("bake idx runs") *out = rdim_bake_index_runs(arena, in->idx_runs); @@ -623,28 +623,12 @@ rdim_local_resolve_incomplete_types(RDIM_TypeChunkList *types, RDIM_UDTChunkList ProfEnd(); } -internal RDIM_LocalState * -rdim_local_init(void) -{ - Arena *arena = arena_alloc(); - RDIM_LocalState *state = push_array(arena, RDIM_LocalState, 1); - state->arena = arena; - state->work_thread_arenas_count = async_thread_count(); - state->work_thread_arenas = push_array(arena, Arena *, state->work_thread_arenas_count); - for EachIndex(idx, state->work_thread_arenas_count) - { - state->work_thread_arenas[idx] = arena_alloc(); - } - return state; -} - internal RDIM_BakeResults -rdim_bake(RDIM_LocalState *state, RDIM_BakeParams *in_params) +rdim_bake(Arena *arena, ASYNC_Root *async_root, RDIM_BakeParams *in_params) { Temp scratch = scratch_begin(0,0); RDIM_BakeResults out = {0}; - - rdim_local_state = state; + rdim_local_async_root = async_root; ////////////////////////////// //- rjf: kick off line tables baking @@ -662,7 +646,7 @@ rdim_bake(RDIM_LocalState *state, RDIM_BakeParams *in_params) RDIM_BakePathTree *path_tree = 0; ProfScope("build interned path tree") { - path_tree = rdim_bake_path_tree_from_params(state->arena, in_params); + path_tree = rdim_bake_path_tree_from_params(arena, in_params); } ////////////////////////////// @@ -802,6 +786,7 @@ rdim_bake(RDIM_LocalState *state, RDIM_BakeParams *in_params) } } + // rjf: inline sites ProfScope("kick off inline site string map build task") { U64 items_per_task = 4096; @@ -894,7 +879,7 @@ rdim_bake(RDIM_LocalState *state, RDIM_BakeParams *in_params) ////////////////////////////// //- rjf: produce joined string map // - RDIM_BakeStringMapLoose *unsorted_bake_string_map = rdim_bake_string_map_loose_make(state->arena, &bake_string_map_topology); + RDIM_BakeStringMapLoose *unsorted_bake_string_map = rdim_bake_string_map_loose_make(arena, &bake_string_map_topology); ProfScope("produce joined string map") { U64 slots_per_task = 16384; @@ -921,16 +906,16 @@ rdim_bake(RDIM_LocalState *state, RDIM_BakeParams *in_params) } // rjf: insert small top-level stuff - rdim_bake_string_map_loose_push_top_level_info(state->arena, &bake_string_map_topology, unsorted_bake_string_map, &in_params->top_level_info); - rdim_bake_string_map_loose_push_binary_sections(state->arena, &bake_string_map_topology, unsorted_bake_string_map, &in_params->binary_sections); - rdim_bake_string_map_loose_push_path_tree(state->arena, &bake_string_map_topology, unsorted_bake_string_map, path_tree); + rdim_bake_string_map_loose_push_top_level_info(arena, &bake_string_map_topology, unsorted_bake_string_map, &in_params->top_level_info); + rdim_bake_string_map_loose_push_binary_sections(arena, &bake_string_map_topology, unsorted_bake_string_map, &in_params->binary_sections); + rdim_bake_string_map_loose_push_path_tree(arena, &bake_string_map_topology, unsorted_bake_string_map, path_tree); } ////////////////////////////// //- rjf: kick off string map sorting tasks // ASYNC_TaskList sort_bake_string_map_tasks = {0}; - RDIM_BakeStringMapLoose *sorted_bake_string_map__in_progress = rdim_bake_string_map_loose_make(state->arena, &bake_string_map_topology); + RDIM_BakeStringMapLoose *sorted_bake_string_map__in_progress = rdim_bake_string_map_loose_make(arena, &bake_string_map_topology); { U64 slots_per_task = 4096; U64 num_tasks = (bake_string_map_topology.slots_count+slots_per_task-1)/slots_per_task; @@ -968,10 +953,10 @@ rdim_bake(RDIM_LocalState *state, RDIM_BakeParams *in_params) //- rjf: build finalized string map // ProfBegin("build finalized string map base indices"); - RDIM_BakeStringMapBaseIndices bake_string_map_base_idxes = rdim_bake_string_map_base_indices_from_map_loose(state->arena, &bake_string_map_topology, sorted_bake_string_map); + RDIM_BakeStringMapBaseIndices bake_string_map_base_idxes = rdim_bake_string_map_base_indices_from_map_loose(arena, &bake_string_map_topology, sorted_bake_string_map); ProfEnd(); ProfBegin("build finalized string map"); - RDIM_BakeStringMapTight bake_strings = rdim_bake_string_map_tight_from_loose(state->arena, &bake_string_map_topology, &bake_string_map_base_idxes, sorted_bake_string_map); + RDIM_BakeStringMapTight bake_strings = rdim_bake_string_map_tight_from_loose(arena, &bake_string_map_topology, &bake_string_map_base_idxes, sorted_bake_string_map); ProfEnd(); ////////////////////////////// @@ -1019,7 +1004,7 @@ rdim_bake(RDIM_LocalState *state, RDIM_BakeParams *in_params) RDIM_String8List location_data_blobs = {0}; { // reserve null location block for opl - rdim_location_block_chunk_list_push_array(state->arena, &location_blocks, 1); + rdim_location_block_chunk_list_push_array(arena, &location_blocks, 1); // TODO: export location instead of VOFF RDIM_BakeGlobalVariablesIn bake_global_variables_in = {&bake_strings, &in_params->global_variables}; @@ -1063,15 +1048,15 @@ rdim_bake(RDIM_LocalState *state, RDIM_BakeParams *in_params) RDIM_BakeIdxRunMap *idx_runs = 0; ProfScope("build interned idx run map") { - idx_runs = rdim_bake_idx_run_map_from_params(state->arena, name_maps, in_params); + idx_runs = rdim_bake_idx_run_map_from_params(arena, name_maps, in_params); } ////////////////////////////// //- rjf: do small top-level bakes // - ProfScope("top level info") out.top_level_info = rdim_bake_top_level_info(state->arena, &bake_strings, &in_params->top_level_info); - ProfScope("binary sections") out.binary_sections = rdim_bake_binary_sections(state->arena, &bake_strings, &in_params->binary_sections); - ProfScope("top level name maps section") out.top_level_name_maps = rdim_bake_name_maps_top_level(state->arena, &bake_strings, idx_runs, name_maps); + ProfScope("top level info") out.top_level_info = rdim_bake_top_level_info(arena, &bake_strings, &in_params->top_level_info); + ProfScope("binary sections") out.binary_sections = rdim_bake_binary_sections(arena, &bake_strings, &in_params->binary_sections); + ProfScope("top level name maps section") out.top_level_name_maps = rdim_bake_name_maps_top_level(arena, &bake_strings, idx_runs, name_maps); ////////////////////////////// //- rjf: kick off pass 3 tasks @@ -1135,17 +1120,15 @@ rdim_bake(RDIM_LocalState *state, RDIM_BakeParams *in_params) // ProfScope("join all name map bakes into final name map bake") { - out.name_maps = rdim_name_map_bake_results_combine(state->arena, name_map_bakes, ArrayCount(name_map_bakes)); + out.name_maps = rdim_name_map_bake_results_combine(arena, name_map_bakes, ArrayCount(name_map_bakes)); } - //////////////////////////////// - out.location_blocks = rdim_str8_list_join(state->arena, &location_blocks, rdim_str8(0,0)); - out.location_data = rdim_str8_list_join(state->arena, &location_data_blobs, rdim_str8(0,0)); - - rdim_local_state = 0; + out.location_blocks = rdim_str8_list_join(arena, &location_blocks, rdim_str8(0,0)); + out.location_data = rdim_str8_list_join(arena, &location_data_blobs, rdim_str8(0,0)); + rdim_local_async_root = 0; scratch_end(scratch); return out; } diff --git a/src/rdi_make/rdi_make_local.h b/src/rdi_make/rdi_make_local.h index b41ed8fe..0a920532 100644 --- a/src/rdi_make/rdi_make_local.h +++ b/src/rdi_make/rdi_make_local.h @@ -4,13 +4,13 @@ #ifndef RDI_MAKE_LOCAL_H #define RDI_MAKE_LOCAL_H -// rjf: base layer memory ops +//- rjf: base layer memory ops #define RDIM_MEMSET_OVERRIDE #define RDIM_MEMCPY_OVERRIDE #define rdim_memset MemorySet #define rdim_memcpy MemoryCopy -// rjf: base layer string overrides +//- rjf: base layer string overrides #define RDI_STRING8_OVERRIDE #define RDIM_String8 String8 #define RDIM_String8_BaseMember str @@ -25,7 +25,7 @@ #define RDIM_String8List_NodeCountMember node_count #define RDIM_String8List_TotalSizeMember total_size -// rjf: base layer arena overrides +//- rjf: base layer arena overrides #define RDIM_ARENA_OVERRIDE #define RDIM_Arena Arena #define rdim_arena_alloc arena_alloc @@ -34,21 +34,20 @@ #define rdim_arena_push arena_push #define rdim_arena_pop_to arena_pop_to -// rjf: base layer scratch arena overrides +//- rjf: base layer scratch arena overrides #define RDIM_SCRATCH_OVERRIDE #define RDIM_Temp Temp #define rdim_temp_arena(t) ((t).arena) #define rdim_scratch_begin scratch_begin #define rdim_scratch_end scratch_end -// rjf: base layer profiling markup overrides +//- rjf: base layer profiling markup overrides #define RDIM_ProfBegin(...) ProfBeginDynamic(__VA_ARGS__) #define RDIM_ProfEnd(...) ProfEnd() +//- rjf: main library #include "lib_rdi_make/rdi_make.h" -//////////////////////////////// - //- rjf: line table baking task types typedef struct RDIM_BakeLineTablesIn RDIM_BakeLineTablesIn; @@ -370,17 +369,10 @@ ASYNC_WORK_DEF(rdim_bake_type_nodes_work); ASYNC_WORK_DEF(rdim_bake_name_map_work); ASYNC_WORK_DEF(rdim_bake_idx_runs_work); -typedef struct RDIM_LocalState RDIM_LocalState; -struct RDIM_LocalState -{ - Arena *arena; - U64 work_thread_arenas_count; - Arena **work_thread_arenas; -}; - //////////////////////////////// +//~ rjf: Globals -global RDIM_LocalState *rdim_local_state = 0; +global ASYNC_Root *rdim_local_async_root = 0; //////////////////////////////// @@ -389,8 +381,7 @@ internal RDIM_TopLevelInfo rdim_make_top_level_info(String8 image_name, Arch arc //////////////////////////////// -internal RDIM_LocalState * rdim_local_init(void); -internal RDIM_BakeResults rdim_bake(RDIM_LocalState *state, RDIM_BakeParams *in); +internal RDIM_BakeResults rdim_bake(Arena *arena, ASYNC_Root *async_root, RDIM_BakeParams *in); internal RDIM_SerializedSectionBundle rdim_compress(Arena *arena, RDIM_SerializedSectionBundle *in); #endif // RDI_MAKE_LOCAL_H diff --git a/src/tester/tester_main.c b/src/tester/tester_main.c index c722b3f0..ae080379 100644 --- a/src/tester/tester_main.c +++ b/src/tester/tester_main.c @@ -40,10 +40,10 @@ internal void entry_point(CmdLine *cmdline) { Arena *arena = arena_alloc(); - E_TypeCtx *type_ctx = push_array(arena, E_TypeCtx, 1); - e_select_type_ctx(type_ctx); - E_ParseCtx *parse_ctx = push_array(arena, E_ParseCtx, 1); - e_select_parse_ctx(parse_ctx); + E_Cache *eval_cache = e_cache_alloc(); + e_select_cache(eval_cache); + E_BaseCtx *base_ctx = push_array(arena, E_BaseCtx, 1); + e_select_base_ctx(base_ctx); E_IRCtx *ir_ctx = push_array(arena, E_IRCtx, 1); e_select_ir_ctx(ir_ctx); E_InterpretCtx *interpret_ctx = push_array(arena, E_InterpretCtx, 1);