From 58e643095558cc55145f3e6b0e6c1c19498470ef Mon Sep 17 00:00:00 2001 From: Ryan Fleury Date: Wed, 20 May 2026 14:02:08 -0700 Subject: [PATCH] p2r: more carefully choose symbol chunk sizes - we were spending dramatically more memory than needed in cases where there were many translation units, with very few symbols, because of our overly-aggressive chunk sizes (e.g. chrome.dll.pdb). also, since we just parse the msf by forming contiguous streams, let's just always map input files in radbin by default, so that we don't pay twice for the memory. --- src/codeview/codeview_parse.h | 12 +++--- src/radbin/radbin.c | 35 ++++++----------- src/radbin/radbin.h | 2 + src/rdi_from_pdb/rdi_from_pdb.c | 69 +++++++++++++++++++++++++++++---- 4 files changed, 82 insertions(+), 36 deletions(-) diff --git a/src/codeview/codeview_parse.h b/src/codeview/codeview_parse.h index e6a5c6c1..c3337798 100644 --- a/src/codeview/codeview_parse.h +++ b/src/codeview/codeview_parse.h @@ -24,7 +24,7 @@ struct CV_NumericParsed typedef struct CV_RecRange CV_RecRange; struct CV_RecRange { - U32 off; + U32 off; CV_RecHeader hdr; }; @@ -34,7 +34,7 @@ typedef struct CV_RecRangeChunk CV_RecRangeChunk; struct CV_RecRangeChunk { struct CV_RecRangeChunk *next; - CV_RecRange ranges[CV_REC_RANGE_CHUNK_SIZE]; + CV_RecRange ranges[CV_REC_RANGE_CHUNK_SIZE]; }; typedef struct CV_RecRangeStream CV_RecRangeStream; @@ -42,14 +42,14 @@ struct CV_RecRangeStream { CV_RecRangeChunk *first_chunk; CV_RecRangeChunk *last_chunk; - U64 total_count; + U64 total_count; }; typedef struct CV_RecRangeArray CV_RecRangeArray; struct CV_RecRangeArray { CV_RecRange *ranges; - U64 count; + U64 count; }; //////////////////////////////// @@ -58,9 +58,9 @@ struct CV_RecRangeArray typedef struct CV_SymTopLevelInfo CV_SymTopLevelInfo; struct CV_SymTopLevelInfo { - CV_Arch arch; + CV_Arch arch; CV_Language language; - String8 compiler_name; + String8 compiler_name; }; typedef struct CV_SymParsed CV_SymParsed; diff --git a/src/radbin/radbin.c b/src/radbin/radbin.c index 4d553846..380f46a7 100644 --- a/src/radbin/radbin.c +++ b/src/radbin/radbin.c @@ -56,28 +56,6 @@ rb_thread_entry_point(void *p) log_select(log); log_scope_begin(); -#if 0 - ProfScope("work") - { - for(int i = 0; i < 5; i += 1) - { - int sum = 0; - ProfScope("do work") - { - for(int x = 0; x < 10000; x += 1) - { - for(int y = 0; y < 10000; y += 1) - { - sum += x*y + x-y; - } - } - } - lane_sync(); - } - } - abort_self(0); -#endif - ////////////////////////////// //- rjf: set up shared state // @@ -314,10 +292,19 @@ rb_thread_entry_point(void *p) ////////////////////////// //- rjf: load recognized files // + File file = {0}; + FileMap file_map = {0}; String8 file_data = {0}; if(file_format != RB_FileFormat_Null) ProfScope("load recognized file") { - file_data = data_from_file_path(arena, input_file_path); + file = file_open(AccessFlag_ShareRead|AccessFlag_Read, input_file_path); + file_map = file_map_open(AccessFlag_Read, file); + FileProperties props = properties_from_file(file); + void *base = file_map_view_open(file_map, AccessFlag_Read, r1u64(0, props.size)); + if(base != 0) + { + file_data = str8((U8 *)base, props.size); + } } ////////////////////////// @@ -441,6 +428,8 @@ rb_thread_entry_point(void *p) f->format = file_format; f->format_flags = file_format_flags; f->path = input_file_path; + f->file = file; + f->file_map = file_map; f->data = file_data; RB_FileNode *file_n = push_array(arena, RB_FileNode, 1); file_n->v = f; diff --git a/src/radbin/radbin.h b/src/radbin/radbin.h index 3323f6c1..b18e45a5 100644 --- a/src/radbin/radbin.h +++ b/src/radbin/radbin.h @@ -35,6 +35,8 @@ struct RB_File RB_FileFormat format; RB_FileFormatFlags format_flags; String8 path; + File file; + FileMap file_map; String8 data; }; diff --git a/src/rdi_from_pdb/rdi_from_pdb.c b/src/rdi_from_pdb/rdi_from_pdb.c index d4c9bc8f..e6b37e15 100644 --- a/src/rdi_from_pdb/rdi_from_pdb.c +++ b/src/rdi_from_pdb/rdi_from_pdb.c @@ -401,6 +401,7 @@ p2r_convert(Arena *arena, P2R_ConvertParams *params) //- rjf: do base MSF parse // MSF_Parsed *msf = 0; + ProfScope("do base MSF parse") { Temp scratch2 = scratch_begin(&scratch.arena, 1); @@ -3135,13 +3136,6 @@ p2r_convert(Arena *arena, P2R_ConvertParams *params) //- rjf: unpack sym Temp scratch = scratch_begin(&arena, 1); CV_SymParsed *sym = all_syms[sym_idx]; - U64 sym_locations_chunk_cap = 4096; - U64 sym_procedures_chunk_cap = 2048; - U64 sym_global_variables_chunk_cap = 2048; - U64 sym_thread_variables_chunk_cap = 2048; - U64 sym_constants_chunk_cap = 2048; - U64 sym_scopes_chunk_cap = 4096; - U64 sym_inline_sites_chunk_cap = 2048; RDIM_Unit *sym_unit = &all_units_ptr->first->v[sym_idx]; RDIM_SymbolChunkList *sym_procedures = &sym_unit->procedures; RDIM_SymbolChunkList *sym_global_variables = &sym_unit->global_variables; @@ -3151,6 +3145,67 @@ p2r_convert(Arena *arena, P2R_ConvertParams *params) RDIM_InlineSiteChunkList *sym_inline_sites = &sym_unit->inline_sites; RDIM_TypeChunkList *typedefs = &syms_typedefs[sym_idx]; + ////////////////////////// + //- rjf: symbols pass 0: predict symbol chunk counts by record kinds + // + U64 sym_procedures_chunk_cap = sym->sym_ranges.count/4 + 1; + U64 sym_global_variables_chunk_cap = sym->sym_ranges.count/12 + 1; + U64 sym_thread_variables_chunk_cap = sym->sym_ranges.count/12 + 1; + U64 sym_constants_chunk_cap = sym->sym_ranges.count/6 + 1; + U64 sym_scopes_chunk_cap = sym->sym_ranges.count/4 + 1; + U64 sym_inline_sites_chunk_cap = sym->sym_ranges.count/6 + 1; + ProfScope("symbols pass 0: predict symbol chunk counts by record kinds") + { + U64 procedure_record_count = 0; + U64 global_variable_record_count = 0; + U64 thread_variable_record_count = 0; + U64 constant_record_count = 0; + U64 scope_record_count = 0; + U64 inline_site_record_count = 0; + for(CV_RecIter iter = {0}; cv_rec_next(sym->data, &sym->sym_ranges, 0, &iter);) + { + switch(iter.kind) + { + default:{}break; + case CV_SymKind_LPROC32: + case CV_SymKind_GPROC32: + { + procedure_record_count += 1; + scope_record_count += 1; + }break; + case CV_SymKind_BLOCK32: + { + scope_record_count += 1; + }break; + case CV_SymKind_INLINESITE: + { + scope_record_count += 1; + inline_site_record_count += 1; + }break; + case CV_SymKind_LDATA32: + case CV_SymKind_GDATA32: + { + global_variable_record_count += 1; + }break; + case CV_SymKind_CONSTANT: + { + constant_record_count += 1; + }break; + case CV_SymKind_LTHREAD32: + case CV_SymKind_GTHREAD32: + { + thread_variable_record_count += 1; + }break; + } + } + sym_procedures_chunk_cap = Max(1, procedure_record_count); + sym_global_variables_chunk_cap = Max(1, global_variable_record_count); + sym_thread_variables_chunk_cap = Max(1, thread_variable_record_count); + sym_constants_chunk_cap = Max(1, constant_record_count); + sym_scopes_chunk_cap = Max(1, scope_record_count); + sym_inline_sites_chunk_cap = Max(1, inline_site_record_count); + } + ////////////////////////// //- rjf: symbols pass 1: produce procedure frame info map (procedure -> frame info) //