p2r: more carefully choose symbol chunk sizes - we were spending dramatically more memory than needed in cases where there were many translation units, with very few symbols, because of our overly-aggressive chunk sizes (e.g. chrome.dll.pdb). also, since we just parse the msf by forming contiguous streams, let's just always map input files in radbin by default, so that we don't pay twice for the memory.

src/radbin/radbin.c

@@ -56,28 +56,6 @@ rb_thread_entry_point(void *p)
   log_select(log);
   log_scope_begin();
   
-#if 0
-  ProfScope("work")
-  {
-    for(int i = 0; i < 5; i += 1)
-    {
-      int sum = 0;
-      ProfScope("do work")
-      {
-        for(int x = 0; x < 10000; x += 1)
-        {
-          for(int y = 0; y < 10000; y += 1)
-          {
-            sum += x*y + x-y;
-          }
-        }
-      }
-      lane_sync();
-    }
-  }
-  abort_self(0);
-#endif
-  
   //////////////////////////////
   //- rjf: set up shared state
   //
@@ -314,10 +292,19 @@ rb_thread_entry_point(void *p)
       //////////////////////////
       //- rjf: load recognized files
       //
+      File file = {0};
+      FileMap file_map = {0};
       String8 file_data = {0};
       if(file_format != RB_FileFormat_Null) ProfScope("load recognized file")
       {
-        file_data = data_from_file_path(arena, input_file_path);
+        file = file_open(AccessFlag_ShareRead|AccessFlag_Read, input_file_path);
+        file_map = file_map_open(AccessFlag_Read, file);
+        FileProperties props = properties_from_file(file);
+        void *base = file_map_view_open(file_map, AccessFlag_Read, r1u64(0, props.size));
+        if(base != 0)
+        {
+          file_data = str8((U8 *)base, props.size);
+        }
       }
       
       //////////////////////////
@@ -441,6 +428,8 @@ rb_thread_entry_point(void *p)
         f->format       = file_format;
         f->format_flags = file_format_flags;
         f->path         = input_file_path;
+        f->file         = file;
+        f->file_map     = file_map;
         f->data         = file_data;
         RB_FileNode *file_n = push_array(arena, RB_FileNode, 1);
         file_n->v = f;

src/radbin/radbin.h

@@ -35,6 +35,8 @@ struct RB_File
   RB_FileFormat format;
   RB_FileFormatFlags format_flags;
   String8 path;
+  File file;
+  FileMap file_map;
   String8 data;
 };
 

src/rdi_from_pdb/rdi_from_pdb.c

@@ -401,6 +401,7 @@ p2r_convert(Arena *arena, P2R_ConvertParams *params)
   //- rjf: do base MSF parse
   //
   MSF_Parsed *msf = 0;
+  ProfScope("do base MSF parse")
   {
     Temp scratch2 = scratch_begin(&scratch.arena, 1);
     
@@ -3135,13 +3136,6 @@ p2r_convert(Arena *arena, P2R_ConvertParams *params)
         //- rjf: unpack sym
         Temp scratch = scratch_begin(&arena, 1);
         CV_SymParsed *sym = all_syms[sym_idx];
-        U64 sym_locations_chunk_cap = 4096;
-        U64 sym_procedures_chunk_cap = 2048;
-        U64 sym_global_variables_chunk_cap = 2048;
-        U64 sym_thread_variables_chunk_cap = 2048;
-        U64 sym_constants_chunk_cap = 2048;
-        U64 sym_scopes_chunk_cap = 4096;
-        U64 sym_inline_sites_chunk_cap = 2048;
         RDIM_Unit *sym_unit = &all_units_ptr->first->v[sym_idx];
         RDIM_SymbolChunkList *sym_procedures = &sym_unit->procedures;
         RDIM_SymbolChunkList *sym_global_variables = &sym_unit->global_variables;
@@ -3151,6 +3145,67 @@ p2r_convert(Arena *arena, P2R_ConvertParams *params)
         RDIM_InlineSiteChunkList *sym_inline_sites = &sym_unit->inline_sites;
         RDIM_TypeChunkList *typedefs = &syms_typedefs[sym_idx];
         
+        //////////////////////////
+        //- rjf: symbols pass 0: predict symbol chunk counts by record kinds
+        //
+        U64 sym_procedures_chunk_cap       = sym->sym_ranges.count/4 + 1;
+        U64 sym_global_variables_chunk_cap = sym->sym_ranges.count/12 + 1;
+        U64 sym_thread_variables_chunk_cap = sym->sym_ranges.count/12 + 1;
+        U64 sym_constants_chunk_cap        = sym->sym_ranges.count/6 + 1;
+        U64 sym_scopes_chunk_cap           = sym->sym_ranges.count/4 + 1;
+        U64 sym_inline_sites_chunk_cap     = sym->sym_ranges.count/6 + 1;
+        ProfScope("symbols pass 0: predict symbol chunk counts by record kinds")
+        {
+          U64 procedure_record_count = 0;
+          U64 global_variable_record_count = 0;
+          U64 thread_variable_record_count = 0;
+          U64 constant_record_count = 0;
+          U64 scope_record_count = 0;
+          U64 inline_site_record_count = 0;
+          for(CV_RecIter iter = {0}; cv_rec_next(sym->data, &sym->sym_ranges, 0, &iter);)
+          {
+            switch(iter.kind)
+            {
+              default:{}break;
+              case CV_SymKind_LPROC32:
+              case CV_SymKind_GPROC32:
+              {
+                procedure_record_count += 1;
+                scope_record_count += 1;
+              }break;
+              case CV_SymKind_BLOCK32:
+              {
+                scope_record_count += 1;
+              }break;
+              case CV_SymKind_INLINESITE:
+              {
+                scope_record_count += 1;
+                inline_site_record_count += 1;
+              }break;
+              case CV_SymKind_LDATA32:
+              case CV_SymKind_GDATA32:
+              {
+                global_variable_record_count += 1;
+              }break;
+              case CV_SymKind_CONSTANT:
+              {
+                constant_record_count += 1;
+              }break;
+              case CV_SymKind_LTHREAD32:
+              case CV_SymKind_GTHREAD32:
+              {
+                thread_variable_record_count += 1;
+              }break;
+            }
+          }
+          sym_procedures_chunk_cap = Max(1, procedure_record_count);
+          sym_global_variables_chunk_cap = Max(1, global_variable_record_count);
+          sym_thread_variables_chunk_cap = Max(1, thread_variable_record_count);
+          sym_constants_chunk_cap = Max(1, constant_record_count);
+          sym_scopes_chunk_cap = Max(1, scope_record_count);
+          sym_inline_sites_chunk_cap = Max(1, inline_site_record_count);
+        }
+        
         //////////////////////////
         //- rjf: symbols pass 1: produce procedure frame info map (procedure -> frame info)
         //