p2r: more carefully choose symbol chunk sizes - we were spending dramatically more memory than needed in cases where there were many translation units, with very few symbols, because of our overly-aggressive chunk sizes (e.g. chrome.dll.pdb). also, since we just parse the msf by forming contiguous streams, let's just always map input files in radbin by default, so that we don't pay twice for the memory.

src/codeview/codeview_parse.h

@@ -24,7 +24,7 @@ struct CV_NumericParsed
 typedef struct CV_RecRange CV_RecRange;
 struct CV_RecRange
 {
-  U32          off;
+  U32 off;
   CV_RecHeader hdr;
 };
 
@@ -34,7 +34,7 @@ typedef struct CV_RecRangeChunk CV_RecRangeChunk;
 struct CV_RecRangeChunk
 {
   struct CV_RecRangeChunk *next;
-  CV_RecRange              ranges[CV_REC_RANGE_CHUNK_SIZE];
+  CV_RecRange ranges[CV_REC_RANGE_CHUNK_SIZE];
 };
 
 typedef struct CV_RecRangeStream CV_RecRangeStream;
@@ -42,14 +42,14 @@ struct CV_RecRangeStream
 {
   CV_RecRangeChunk *first_chunk;
   CV_RecRangeChunk *last_chunk;
-  U64               total_count;
+  U64 total_count;
 };
 
 typedef struct CV_RecRangeArray CV_RecRangeArray;
 struct CV_RecRangeArray
 {
   CV_RecRange *ranges;
-  U64          count;
+  U64 count;
 };
 
 ////////////////////////////////
@@ -58,9 +58,9 @@ struct CV_RecRangeArray
 typedef struct CV_SymTopLevelInfo CV_SymTopLevelInfo;
 struct CV_SymTopLevelInfo
 {
-  CV_Arch     arch;
+  CV_Arch arch;
   CV_Language language;
-  String8     compiler_name;
+  String8 compiler_name;
 };
 
 typedef struct CV_SymParsed CV_SymParsed;

src/radbin/radbin.c

@@ -56,28 +56,6 @@ rb_thread_entry_point(void *p)
   log_select(log);
   log_scope_begin();
   
-#if 0
-  ProfScope("work")
-  {
-    for(int i = 0; i < 5; i += 1)
-    {
-      int sum = 0;
-      ProfScope("do work")
-      {
-        for(int x = 0; x < 10000; x += 1)
-        {
-          for(int y = 0; y < 10000; y += 1)
-          {
-            sum += x*y + x-y;
-          }
-        }
-      }
-      lane_sync();
-    }
-  }
-  abort_self(0);
-#endif
-  
   //////////////////////////////
   //- rjf: set up shared state
   //
@@ -314,10 +292,19 @@ rb_thread_entry_point(void *p)
       //////////////////////////
       //- rjf: load recognized files
       //
+      File file = {0};
+      FileMap file_map = {0};
       String8 file_data = {0};
       if(file_format != RB_FileFormat_Null) ProfScope("load recognized file")
       {
-        file_data = data_from_file_path(arena, input_file_path);
+        file = file_open(AccessFlag_ShareRead|AccessFlag_Read, input_file_path);
+        file_map = file_map_open(AccessFlag_Read, file);
+        FileProperties props = properties_from_file(file);
+        void *base = file_map_view_open(file_map, AccessFlag_Read, r1u64(0, props.size));
+        if(base != 0)
+        {
+          file_data = str8((U8 *)base, props.size);
+        }
       }
       
       //////////////////////////
@@ -441,6 +428,8 @@ rb_thread_entry_point(void *p)
         f->format       = file_format;
         f->format_flags = file_format_flags;
         f->path         = input_file_path;
+        f->file         = file;
+        f->file_map     = file_map;
         f->data         = file_data;
         RB_FileNode *file_n = push_array(arena, RB_FileNode, 1);
         file_n->v = f;

src/radbin/radbin.h

@@ -35,6 +35,8 @@ struct RB_File
   RB_FileFormat format;
   RB_FileFormatFlags format_flags;
   String8 path;
+  File file;
+  FileMap file_map;
   String8 data;
 };
 

src/rdi_from_pdb/rdi_from_pdb.c

@@ -401,6 +401,7 @@ p2r_convert(Arena *arena, P2R_ConvertParams *params)
   //- rjf: do base MSF parse
   //
   MSF_Parsed *msf = 0;
+  ProfScope("do base MSF parse")
   {
     Temp scratch2 = scratch_begin(&scratch.arena, 1);
     
@@ -3135,13 +3136,6 @@ p2r_convert(Arena *arena, P2R_ConvertParams *params)
         //- rjf: unpack sym
         Temp scratch = scratch_begin(&arena, 1);
         CV_SymParsed *sym = all_syms[sym_idx];
-        U64 sym_locations_chunk_cap = 4096;
-        U64 sym_procedures_chunk_cap = 2048;
-        U64 sym_global_variables_chunk_cap = 2048;
-        U64 sym_thread_variables_chunk_cap = 2048;
-        U64 sym_constants_chunk_cap = 2048;
-        U64 sym_scopes_chunk_cap = 4096;
-        U64 sym_inline_sites_chunk_cap = 2048;
         RDIM_Unit *sym_unit = &all_units_ptr->first->v[sym_idx];
         RDIM_SymbolChunkList *sym_procedures = &sym_unit->procedures;
         RDIM_SymbolChunkList *sym_global_variables = &sym_unit->global_variables;
@@ -3151,6 +3145,67 @@ p2r_convert(Arena *arena, P2R_ConvertParams *params)
         RDIM_InlineSiteChunkList *sym_inline_sites = &sym_unit->inline_sites;
         RDIM_TypeChunkList *typedefs = &syms_typedefs[sym_idx];
         
+        //////////////////////////
+        //- rjf: symbols pass 0: predict symbol chunk counts by record kinds
+        //
+        U64 sym_procedures_chunk_cap       = sym->sym_ranges.count/4 + 1;
+        U64 sym_global_variables_chunk_cap = sym->sym_ranges.count/12 + 1;
+        U64 sym_thread_variables_chunk_cap = sym->sym_ranges.count/12 + 1;
+        U64 sym_constants_chunk_cap        = sym->sym_ranges.count/6 + 1;
+        U64 sym_scopes_chunk_cap           = sym->sym_ranges.count/4 + 1;
+        U64 sym_inline_sites_chunk_cap     = sym->sym_ranges.count/6 + 1;
+        ProfScope("symbols pass 0: predict symbol chunk counts by record kinds")
+        {
+          U64 procedure_record_count = 0;
+          U64 global_variable_record_count = 0;
+          U64 thread_variable_record_count = 0;
+          U64 constant_record_count = 0;
+          U64 scope_record_count = 0;
+          U64 inline_site_record_count = 0;
+          for(CV_RecIter iter = {0}; cv_rec_next(sym->data, &sym->sym_ranges, 0, &iter);)
+          {
+            switch(iter.kind)
+            {
+              default:{}break;
+              case CV_SymKind_LPROC32:
+              case CV_SymKind_GPROC32:
+              {
+                procedure_record_count += 1;
+                scope_record_count += 1;
+              }break;
+              case CV_SymKind_BLOCK32:
+              {
+                scope_record_count += 1;
+              }break;
+              case CV_SymKind_INLINESITE:
+              {
+                scope_record_count += 1;
+                inline_site_record_count += 1;
+              }break;
+              case CV_SymKind_LDATA32:
+              case CV_SymKind_GDATA32:
+              {
+                global_variable_record_count += 1;
+              }break;
+              case CV_SymKind_CONSTANT:
+              {
+                constant_record_count += 1;
+              }break;
+              case CV_SymKind_LTHREAD32:
+              case CV_SymKind_GTHREAD32:
+              {
+                thread_variable_record_count += 1;
+              }break;
+            }
+          }
+          sym_procedures_chunk_cap = Max(1, procedure_record_count);
+          sym_global_variables_chunk_cap = Max(1, global_variable_record_count);
+          sym_thread_variables_chunk_cap = Max(1, thread_variable_record_count);
+          sym_constants_chunk_cap = Max(1, constant_record_count);
+          sym_scopes_chunk_cap = Max(1, scope_record_count);
+          sym_inline_sites_chunk_cap = Max(1, inline_site_record_count);
+        }
+        
         //////////////////////////
         //- rjf: symbols pass 1: produce procedure frame info map (procedure -> frame info)
         //