raddbgi_from_pdb: multithread initial independent parsing/preparation phases of converter

src/base/base_thread_context.c

@@ -19,6 +19,15 @@ tctx_init_and_equip(TCTX *tctx){
   tctx_thread_local = tctx;
 }
 
+internal void
+tctx_release(void)
+{
+  for(U64 i = 0; i < ArrayCount(tctx_thread_local->arenas); i += 1)
+  {
+    arena_release(tctx_thread_local->arenas[i]);
+  }
+}
+
 internal TCTX*
 tctx_get_equipped(void){
   return(tctx_thread_local);

src/base/base_thread_context.h

@@ -23,6 +23,7 @@ struct TCTX
 // NOTE(allen): Thread Context Functions
 
 internal void      tctx_init_and_equip(TCTX *tctx);
+internal void      tctx_release(void);
 internal TCTX*     tctx_get_equipped(void);
 
 internal Arena*    tctx_get_scratch(Arena **conflicts, U64 count);

src/ctrl/ctrl_core.c

@@ -1518,8 +1518,6 @@ ctrl_u2ms_dequeue_req(CTRL_MachineID *out_machine_id, CTRL_Handle *out_process,
 internal void
 ctrl_thread__entry_point(void *p)
 {
-  TCTX tctx_;
-  tctx_init_and_equip(&tctx_);
   ThreadName("[ctrl] thread");
   ProfBeginFunction();
   demon_primary_thread_begin();
@@ -3274,8 +3272,6 @@ ctrl_thread__single_step(CTRL_Msg *msg)
 internal void
 ctrl_mem_stream_thread__entry_point(void *p)
 {
-  TCTX tctx_ = {0};
-  tctx_init_and_equip(&tctx_);
   CTRL_ProcessMemoryCache *cache = &ctrl_state->process_memory_cache;
   for(;;)
   {

src/dasm/dasm.c

@@ -372,8 +372,6 @@ dasm_u2d_dequeue_request(void)
 internal void
 dasm_decode_thread_entry_point(void *p)
 {
-  TCTX tctx_;
-  tctx_init_and_equip(&tctx_);
   for(;;)
   {
     Temp scratch = scratch_begin(0, 0);

src/dbgi/dbgi.c

@@ -632,8 +632,6 @@ dbgi_p2u_pop_events(Arena *arena, U64 endt_us)
 internal void
 dbgi_parse_thread_entry_point(void *p)
 {
-  TCTX tctx_;
-  tctx_init_and_equip(&tctx_);
   ProfThreadName("[dbgi] parse #%I64U", (U64)p);
   for(;;)
   {
@@ -1154,8 +1152,6 @@ dbgi_qsort_compare_fuzzy_search_items(DBGI_FuzzySearchItem *a, DBGI_FuzzySearchI
 internal void
 dbgi_fuzzy_thread__entry_point(void *p)
 {
-  TCTX tctx_;
-  tctx_init_and_equip(&tctx_);
   ProfThreadName("[dbgi] fuzzy search #%I64U", (U64)p);
   DBGI_FuzzySearchThread *thread = &dbgi_shared->fuzzy_threads[(U64)p];
   for(;;)

src/df/gfx/df_gfx.c

@@ -7753,8 +7753,6 @@ df_text_search_little_hash_from_hash(U128 hash)
 internal void
 df_text_search_thread_entry_point(void *p)
 {
-  TCTX tctx_;
-  tctx_init_and_equip(&tctx_);
 #if 0
   // TODO(rjf): [ ] @de2ctrl text searcher -- wound up in DE_Hash
   

src/file_stream/file_stream.c

@@ -140,8 +140,6 @@ fs_u2s_dequeue_path(Arena *arena)
 internal void
 fs_streamer_thread__entry_point(void *p)
 {
-  TCTX tctx_;
-  tctx_init_and_equip(&tctx_);
   ThreadName("[fs] streamer #%I64u", (U64)p);
   for(;;)
   {

src/geo_cache/geo_cache.c

@@ -293,8 +293,6 @@ geo_u2x_dequeue_req(U128 *key_out, U128 *hash_out)
 internal void
 geo_xfer_thread__entry_point(void *p)
 {
-  TCTX tctx_ = {0};
-  tctx_init_and_equip(&tctx_);
   for(;;)
   {
     HS_Scope *scope = hs_scope_open();

src/lib_raddbgi_make/raddbgi_make.c

@@ -1479,6 +1479,7 @@ rdim_bake(RDIM_Arena *arena, RDIM_BakeParams *params)
         RDI_Line *unit_lines      = 0;
         RDI_U16  *unit_cols       = 0;
         RDI_U32   unit_line_count = 0;
+        RDIM_ProfScope("produce combined unit line info")
         {
           RDIM_Temp scratch = rdim_scratch_begin(&arena, 1);
           
@@ -1538,7 +1539,11 @@ rdim_bake(RDIM_Arena *arena, RDIM_BakeParams *params)
           }
           
           //- rjf: sort
-          RDIM_SortKey *sorted_line_keys = rdim_sort_key_array(scratch.arena, line_keys, key_count);
+          RDIM_SortKey *sorted_line_keys = 0;
+          RDIM_ProfScope("sort")
+          {
+            sorted_line_keys = rdim_sort_key_array(scratch.arena, line_keys, key_count);
+          }
           
           // TODO(rjf): do a pass over sorted keys to make sure duplicate keys
           // are sorted with null record first, and no more than one null
@@ -1547,23 +1552,26 @@ rdim_bake(RDIM_Arena *arena, RDIM_BakeParams *params)
           //- rjf: arrange output
           RDI_U64 *arranged_voffs = rdim_push_array_no_zero(arena, RDI_U64, key_count + 1);
           RDI_Line *arranged_lines = rdim_push_array_no_zero(arena, RDI_Line, key_count);
-          for(RDI_U64 i = 0; i < key_count; i += 1)
+          RDIM_ProfScope("arrange output")
           {
-            arranged_voffs[i] = sorted_line_keys[i].key;
-          }
-          arranged_voffs[key_count] = ~0ull;
-          for(RDI_U64 i = 0; i < key_count; i += 1)
-          {
-            RDIM_LineRec *rec = (RDIM_LineRec*)sorted_line_keys[i].val;
-            if(rec != 0)
+            for(RDI_U64 i = 0; i < key_count; i += 1)
             {
-              arranged_lines[i].file_idx = rec->file_id;
-              arranged_lines[i].line_num = rec->line_num;
+              arranged_voffs[i] = sorted_line_keys[i].key;
             }
-            else
+            arranged_voffs[key_count] = ~0ull;
+            for(RDI_U64 i = 0; i < key_count; i += 1)
             {
-              arranged_lines[i].file_idx = 0;
-              arranged_lines[i].line_num = 0;
+              RDIM_LineRec *rec = (RDIM_LineRec*)sorted_line_keys[i].val;
+              if(rec != 0)
+              {
+                arranged_lines[i].file_idx = rec->file_id;
+                arranged_lines[i].line_num = rec->line_num;
+              }
+              else
+              {
+                arranged_lines[i].file_idx = 0;
+                arranged_lines[i].line_num = 0;
+              }
             }
           }
           

src/os/core/linux/os_core_linux.c

@@ -788,8 +788,8 @@ lnx_thread_base(void *ptr){
   
   TCTX tctx_;
   tctx_init_and_equip(&tctx_);
-  
   func(thread_ptr);
+  tctx_release();
   
   // remove my bit
   U32 result = __sync_fetch_and_and(&entity->reference_mask, ~0x2);

src/os/core/win32/os_core_win32.c

@@ -167,7 +167,10 @@ w32_thread_base(void *ptr){
   OS_ThreadFunctionType *func = entity->thread.func;
   void *thread_ptr = entity->thread.ptr;
   
+  TCTX tctx_;
+  tctx_init_and_equip(&tctx_);
   func(thread_ptr);
+  tctx_release();
   
   // remove my bit
   LONG result = InterlockedAnd((LONG*)&entity->reference_mask, ~0x2);
@@ -1238,7 +1241,11 @@ os_thread_wait(OS_Handle handle, U64 endt_us)
 {
   DWORD sleep_ms = w32_sleep_ms_from_endt_us(endt_us);
   W32_Entity *entity = (W32_Entity *)PtrFromInt(handle.u64[0]);
-  DWORD wait_result = WaitForSingleObject(entity->thread.handle, sleep_ms);
+  DWORD wait_result = WAIT_OBJECT_0;
+  if(entity != 0)
+  {
+    wait_result = WaitForSingleObject(entity->thread.handle, sleep_ms);
+  }
   return (wait_result == WAIT_OBJECT_0);
 }
 

src/raddbgi_from_pdb/raddbgi_from_pdb.c

@@ -539,6 +539,33 @@ p2r_location_over_lvar_addr_range(Arena *arena, RDIM_ScopeChunkList *scopes, RDI
   }
 }
 
+////////////////////////////////
+//~ rjf: Initial PDB Parsing Pass Threads
+
+internal void
+p2r_tpi_hash_parse_thread__entry_point(void *p)
+{
+  ThreadName("[p2r] tpi hash parse thread");
+  P2R_TPIHashParseTask *task = (P2R_TPIHashParseTask *)p;
+  task->out = pdb_tpi_hash_from_data(task->out_arena, task->in.strtbl, task->in.tpi, task->in.hash_data, task->in.aux_data);
+}
+
+internal void
+p2r_tpi_leaf_parse_thread__entry_point(void *p)
+{
+  ThreadName("[p2r] tpi leaf parse thread");
+  P2R_TPILeafParseTask *task = (P2R_TPILeafParseTask *)p;
+  task->out = cv_leaf_from_data(task->out_arena, task->in.leaf_data, task->in.itype_first);
+}
+
+internal void
+p2r_exe_hash_thread__entry_point(void *p)
+{
+  ThreadName("[p2r] exe hash thread");
+  P2R_EXEHashTask *task = (P2R_EXEHashTask *)p;
+  ProfScope("hash exe") task->out = rdi_hash(task->in.exe_data.str, task->in.exe_data.size);
+}
+
 ////////////////////////////////
 //~ rjf: Type Forward Resolution Map Build / Thread
 
@@ -665,8 +692,6 @@ p2r_itype_fwd_map_fill(P2R_ITypeFwdMapFillIn *in)
 internal void
 p2r_itype_fwd_map_fill_task_thread__entry_point(void *p)
 {
-  TCTX tctx_;
-  tctx_init_and_equip(&tctx_);
   P2R_ITypeFwdMapFillTaskBatch *batch = (P2R_ITypeFwdMapFillTaskBatch *)p;
   ThreadName("[p2r] itype fwd map fill thread");
   for(;;)
@@ -1398,8 +1423,6 @@ p2r_unit_symbol_convert(Arena *arena, P2R_UnitSymbolConvertIn *in)
 internal void
 p2r_unit_symbol_convert_task_thread__entry_point(void *p)
 {
-  TCTX tctx_;
-  tctx_init_and_equip(&tctx_);
   P2R_UnitSymbolTaskBatch *batch = (P2R_UnitSymbolTaskBatch *)p;
   ThreadName("[p2r] unit symbol thread");
   for(;;)
@@ -1525,45 +1548,88 @@ p2r_convert(Arena *arena, P2R_ConvertIn *in)
   }
   
   //////////////////////////////////////////////////////////////
-  //- rjf: parse tpi hash
+  //- rjf: do independent parsing & preparation passes
   //
+  U64 exe_hash = 0;
   PDB_TpiHashParsed *tpi_hash = 0;
-  if(tpi != 0) ProfScope("parse tpi hash")
-  {
-    String8 hash_data = msf_data_from_stream(msf, tpi->hash_sn);
-    String8 aux_data = msf_data_from_stream(msf, tpi->hash_sn_aux);
-    tpi_hash = pdb_tpi_hash_from_data(arena, strtbl, tpi, hash_data, aux_data);
-  }
-  
-  //////////////////////////////////////////////////////////////
-  //- rjf: parse tpi leaves
-  //
   CV_LeafParsed *tpi_leaf = 0;
-  if(tpi != 0) ProfScope("parse tpi leaves")
-  {
-    String8 leaf_data = pdb_leaf_data_from_tpi(tpi);
-    tpi_leaf = cv_leaf_from_data(arena, leaf_data, tpi->itype_first);
-  }
-  
-  //////////////////////////////////////////////////////////////
-  //- rjf: parse ipi hash
-  //
   PDB_TpiHashParsed *ipi_hash = 0;
-  if(ipi != 0) ProfScope("parse ipi hash")
-  {
-    String8 hash_data = msf_data_from_stream(msf, ipi->hash_sn);
-    String8 aux_data = msf_data_from_stream(msf, ipi->hash_sn_aux);
-    ipi_hash = pdb_tpi_hash_from_data(arena, strtbl, ipi, hash_data, aux_data);
-  }
-  
-  //////////////////////////////////////////////////////////////
-  //- rjf: parse ipi leaves
-  //
   CV_LeafParsed *ipi_leaf = 0;
-  if(ipi != 0) ProfScope("parse ipi leaves")
+  ProfScope("do independent parsing & preparation passess")
   {
-    String8 leaf_data = pdb_leaf_data_from_tpi(ipi);
-    ipi_leaf = cv_leaf_from_data(arena, leaf_data, ipi->itype_first);
+    //- rjf: kick off exe hash
+    OS_Handle exe_hash_thread = {0};
+    P2R_EXEHashTask exe_hash_task = {0};
+    {
+      exe_hash_task.in.exe_data = in->input_exe_data;
+      exe_hash_thread = os_launch_thread(p2r_exe_hash_thread__entry_point, &exe_hash_task, 0);
+    }
+    
+    //- rjf: kick off tpi hash parse
+    OS_Handle tpi_hash_thread = {0};
+    P2R_TPIHashParseTask tpi_hash_task = {0};
+    if(tpi != 0)
+    {
+      tpi_hash_task.in.strtbl    = strtbl;
+      tpi_hash_task.in.tpi       = tpi;
+      tpi_hash_task.in.hash_data = msf_data_from_stream(msf, tpi->hash_sn);
+      tpi_hash_task.in.aux_data  = msf_data_from_stream(msf, tpi->hash_sn_aux);
+      tpi_hash_task.out_arena = arena_alloc();
+      tpi_hash_thread = os_launch_thread(p2r_tpi_hash_parse_thread__entry_point, &tpi_hash_task, 0);
+    }
+    
+    //- rjf: kick off tpi leaf parse
+    OS_Handle tpi_leaf_thread = {0};
+    P2R_TPILeafParseTask tpi_leaf_task = {0};
+    if(tpi != 0)
+    {
+      tpi_leaf_task.in.leaf_data   = pdb_leaf_data_from_tpi(tpi);
+      tpi_leaf_task.in.itype_first = tpi->itype_first;
+      tpi_leaf_task.out_arena = arena_alloc();
+      tpi_leaf_thread = os_launch_thread(p2r_tpi_leaf_parse_thread__entry_point, &tpi_leaf_task, 0);
+    }
+    
+    //- rjf: kick off ipi hash parse
+    OS_Handle ipi_hash_thread = {0};
+    P2R_TPIHashParseTask ipi_hash_task = {0};
+    if(ipi != 0)
+    {
+      ipi_hash_task.in.strtbl    = strtbl;
+      ipi_hash_task.in.tpi       = ipi;
+      ipi_hash_task.in.hash_data = msf_data_from_stream(msf, ipi->hash_sn);
+      ipi_hash_task.in.aux_data  = msf_data_from_stream(msf, ipi->hash_sn_aux);
+      ipi_hash_task.out_arena = arena_alloc();
+      ipi_hash_thread = os_launch_thread(p2r_tpi_hash_parse_thread__entry_point, &ipi_hash_task, 0);
+    }
+    
+    //- rjf: kick off ipi leaf parse
+    OS_Handle ipi_leaf_thread = {0};
+    P2R_TPILeafParseTask ipi_leaf_task = {0};
+    if(ipi != 0)
+    {
+      ipi_leaf_task.in.leaf_data   = pdb_leaf_data_from_tpi(ipi);
+      ipi_leaf_task.in.itype_first = ipi->itype_first;
+      ipi_leaf_task.out_arena = arena_alloc();
+      ipi_leaf_thread = os_launch_thread(p2r_tpi_leaf_parse_thread__entry_point, &ipi_leaf_task, 0);
+    }
+    
+    //- rjf: join all independent task threads
+    os_thread_wait(exe_hash_thread, max_U64);
+    os_thread_wait(tpi_hash_thread, max_U64);
+    os_thread_wait(tpi_leaf_thread, max_U64);
+    os_thread_wait(ipi_hash_thread, max_U64);
+    os_thread_wait(ipi_leaf_thread, max_U64);
+    
+    //- rjf: fill/absorb exports from completed tasks
+    exe_hash = exe_hash_task.out;
+    tpi_hash = tpi_hash_task.out;
+    tpi_leaf = tpi_leaf_task.out;
+    ipi_hash = ipi_hash_task.out;
+    ipi_leaf = ipi_leaf_task.out;
+    arena_absorb(arena, tpi_hash_task.out_arena);
+    arena_absorb(arena, tpi_leaf_task.out_arena);
+    arena_absorb(arena, ipi_hash_task.out_arena);
+    arena_absorb(arena, ipi_leaf_task.out_arena);
   }
   
   //////////////////////////////////////////////////////////////
@@ -1604,7 +1670,7 @@ p2r_convert(Arena *arena, P2R_ConvertIn *in)
   //- rjf: parse syms for each compilation unit
   //
   CV_SymParsed **sym_for_unit = push_array(arena, CV_SymParsed*, comp_unit_count);
-  if(comp_units != 0) ProfScope("parse symbols")
+  if(comp_units != 0) ProfScope("parse syms for each compilation unit")
   {
     PDB_CompUnit **unit_ptr = comp_units->units;
     for(U64 i = 0; i < comp_unit_count; i += 1, unit_ptr += 1)
@@ -1632,15 +1698,6 @@ p2r_convert(Arena *arena, P2R_ConvertIn *in)
     }
   }
   
-  //////////////////////////////////////////////////////////////
-  //- rjf: hash exe
-  //
-  U64 exe_hash = 0;
-  if(in->input_exe_data.size > 0) ProfScope("hash exe")
-  {
-    exe_hash = rdi_hash(in->input_exe_data.str, in->input_exe_data.size);
-  }
-  
   //////////////////////////////////////////////////////////////
   //- rjf: calculate EXE's max voff
   //
@@ -3325,6 +3382,7 @@ p2r_convert(Arena *arena, P2R_ConvertIn *in)
     {
       for(U64 idx = 0; idx < tasks_count; idx += 1)
       {
+        arena_absorb(arena, tasks[idx].out_arena);
         rdim_symbol_chunk_list_concat_in_place(&all_procedures,       &tasks[idx].convert_out->procedures);
         rdim_symbol_chunk_list_concat_in_place(&all_global_variables, &tasks[idx].convert_out->global_variables);
         rdim_symbol_chunk_list_concat_in_place(&all_thread_variables, &tasks[idx].convert_out->thread_variables);

src/raddbgi_from_pdb/raddbgi_from_pdb.h

@@ -53,6 +53,60 @@ struct P2R_ConvertOut
   RDIM_ScopeChunkList scopes;
 };
 
+////////////////////////////////
+//~ rjf: Initial PDB Information Extraction & Conversion Preparation Task Types
+
+//- rjf: tpi hash parsing
+
+typedef struct P2R_TPIHashParseIn P2R_TPIHashParseIn;
+struct P2R_TPIHashParseIn
+{
+  PDB_Strtbl *strtbl;
+  PDB_TpiParsed *tpi;
+  String8 hash_data;
+  String8 aux_data;
+};
+
+typedef struct P2R_TPIHashParseTask P2R_TPIHashParseTask;
+struct P2R_TPIHashParseTask
+{
+  P2R_TPIHashParseIn in;
+  Arena *out_arena;
+  PDB_TpiHashParsed *out;
+};
+
+//- rjf: tpi leaves parsing
+
+typedef struct P2R_TPILeafParseIn P2R_TPILeafParseIn;
+struct P2R_TPILeafParseIn
+{
+  String8 leaf_data;
+  CV_TypeId itype_first;
+};
+
+typedef struct P2R_TPILeafParseTask P2R_TPILeafParseTask;
+struct P2R_TPILeafParseTask
+{
+  P2R_TPILeafParseIn in;
+  Arena *out_arena;
+  CV_LeafParsed *out;
+};
+
+//- rjf: exe hashing
+
+typedef struct P2R_EXEHashIn P2R_EXEHashIn;
+struct P2R_EXEHashIn
+{
+  String8 exe_data;
+};
+
+typedef struct P2R_EXEHashTask P2R_EXEHashTask;
+struct P2R_EXEHashTask
+{
+  P2R_EXEHashIn in;
+  U64 out;
+};
+
 ////////////////////////////////
 //~ rjf: Conversion Data Structure Types
 
@@ -176,6 +230,13 @@ internal CV_EncodedFramePtrReg p2r_cv_encoded_fp_reg_from_frameproc(CV_SymFramep
 internal RDI_RegisterCode p2r_reg_code_from_arch_encoded_fp_reg(RDI_Arch arch, CV_EncodedFramePtrReg encoded_reg);
 internal void p2r_location_over_lvar_addr_range(Arena *arena, RDIM_ScopeChunkList *scopes, RDIM_LocationSet *locset, RDIM_Location *location, CV_LvarAddrRange *range, COFF_SectionHeader *section, CV_LvarAddrGap *gaps, U64 gap_count);
 
+////////////////////////////////
+//~ rjf: Initial Parsing & Preparation Pass Threads
+
+internal void p2r_tpi_hash_parse_thread__entry_point(void *p);
+internal void p2r_tpi_leaf_parse_thread__entry_point(void *p);
+internal void p2r_exe_hash_thread__entry_point(void *p);
+
 ////////////////////////////////
 //~ rjf: Type Forward Resolution Map Build / Thread
 

src/text_cache/text_cache.c

@@ -674,8 +674,6 @@ txt_u2p_dequeue_req(U128 *key_out, U128 *hash_out, TXT_LangKind *lang_out)
 internal void
 txt_parse_thread__entry_point(void *p)
 {
-  TCTX tctx_ = {0};
-  tctx_init_and_equip(&tctx_);
   for(;;)
   {
     HS_Scope *scope = hs_scope_open();

src/texture_cache/texture_cache.c

@@ -311,8 +311,6 @@ tex_u2x_dequeue_req(U128 *key_out, U128 *hash_out, TEX_Topology *top_out)
 internal void
 tex_xfer_thread__entry_point(void *p)
 {
-  TCTX tctx_ = {0};
-  tctx_init_and_equip(&tctx_);
   for(;;)
   {
     HS_Scope *scope = hs_scope_open();

src/txti/txti.c

@@ -940,9 +940,6 @@ txti_set_external_change_detection_enabled(B32 enabled)
 internal void
 txti_mut_thread_entry_point(void *p)
 {
-  TCTX tctx_;
-  tctx_init_and_equip(&tctx_);
-  
   U64 mut_thread_idx = (U64)p;
   ProfThreadName("[txti] mut #%I64u", mut_thread_idx);
   TXTI_MutThread *mut_thread = &txti_state->mut_threads[mut_thread_idx];
@@ -1249,8 +1246,6 @@ txti_mut_thread_entry_point(void *p)
 internal void
 txti_detector_thread_entry_point(void *p)
 {
-  TCTX tctx_;
-  tctx_init_and_equip(&tctx_);
   ProfThreadName("[txti] detector");
   for(;;)
   {