shift to slice-granularity of string map gathering, rather than chunk-granularity - gives a bit more flexibility. set up pdb converter type name gathering tasks for slice subdivision. before going forward with this plan it's imperative that we start sparsely allocating these name maps...

project.4coder

@@ -47,7 +47,7 @@ commands =
 {
   .rjf_f1 =
   {
-    .win = "build raddbgi_from_pdb telemetry debug",
+    .win = "build raddbgi_from_pdb telemetry release",
     .linux = "",
     .out = "*compilation*",
     .footer_panel = true,

src/lib_raddbgi_make/raddbgi_make.c

@@ -1861,47 +1861,47 @@ rdim_bake_string_chunk_list_map_push_path_tree(RDIM_Arena *arena, RDIM_BakeStrin
 //- rjf: chunk-granularity bake string gathering passes
 
 RDI_PROC void
-rdim_bake_string_chunk_list_map_push_src_file_chunk(RDIM_Arena *arena, RDIM_BakeStringChunkListMapTopology *top, RDIM_BakeStringChunkListMap *map, RDIM_SrcFileChunkNode *chunk)
+rdim_bake_string_chunk_list_map_push_src_file_slice(RDIM_Arena *arena, RDIM_BakeStringChunkListMapTopology *top, RDIM_BakeStringChunkListMap *map, RDIM_SrcFile *v, RDI_U64 count)
 {
-  for(RDI_U64 idx = 0; idx < chunk->count; idx += 1)
+  for(RDI_U64 idx = 0; idx < count; idx += 1)
   {
-    rdim_bake_string_chunk_list_map_insert(arena, top, map, 1, chunk->v[idx].normal_full_path);
+    rdim_bake_string_chunk_list_map_insert(arena, top, map, 1, v[idx].normal_full_path);
   }
 }
 
 RDI_PROC void
-rdim_bake_string_chunk_list_map_push_unit_chunk(RDIM_Arena *arena, RDIM_BakeStringChunkListMapTopology *top, RDIM_BakeStringChunkListMap *map, RDIM_UnitChunkNode *chunk)
+rdim_bake_string_chunk_list_map_push_unit_slice(RDIM_Arena *arena, RDIM_BakeStringChunkListMapTopology *top, RDIM_BakeStringChunkListMap *map, RDIM_Unit *v, RDI_U64 count)
 {
-  for(RDI_U64 idx = 0; idx < chunk->count; idx += 1)
+  for(RDI_U64 idx = 0; idx < count; idx += 1)
   {
-    rdim_bake_string_chunk_list_map_insert(arena, top, map, 4, chunk->v[idx].unit_name);
+    rdim_bake_string_chunk_list_map_insert(arena, top, map, 4, v[idx].unit_name);
-    rdim_bake_string_chunk_list_map_insert(arena, top, map, 4, chunk->v[idx].compiler_name);
+    rdim_bake_string_chunk_list_map_insert(arena, top, map, 4, v[idx].compiler_name);
-    rdim_bake_string_chunk_list_map_insert(arena, top, map, 4, chunk->v[idx].source_file);
+    rdim_bake_string_chunk_list_map_insert(arena, top, map, 4, v[idx].source_file);
-    rdim_bake_string_chunk_list_map_insert(arena, top, map, 4, chunk->v[idx].object_file);
+    rdim_bake_string_chunk_list_map_insert(arena, top, map, 4, v[idx].object_file);
-    rdim_bake_string_chunk_list_map_insert(arena, top, map, 4, chunk->v[idx].archive_file);
+    rdim_bake_string_chunk_list_map_insert(arena, top, map, 4, v[idx].archive_file);
-    rdim_bake_string_chunk_list_map_insert(arena, top, map, 4, chunk->v[idx].build_path);
+    rdim_bake_string_chunk_list_map_insert(arena, top, map, 4, v[idx].build_path);
   }
 }
 
 RDI_PROC void
-rdim_bake_string_chunk_list_map_push_type_chunk(RDIM_Arena *arena, RDIM_BakeStringChunkListMapTopology *top, RDIM_BakeStringChunkListMap *map, RDIM_TypeChunkNode *chunk)
+rdim_bake_string_chunk_list_map_push_type_slice(RDIM_Arena *arena, RDIM_BakeStringChunkListMapTopology *top, RDIM_BakeStringChunkListMap *map, RDIM_Type *v, RDI_U64 count)
 {
-  for(RDI_U64 idx = 0; idx < chunk->count; idx += 1)
+  for(RDI_U64 idx = 0; idx < count; idx += 1)
   {
-    rdim_bake_string_chunk_list_map_insert(arena, top, map, 64, chunk->v[idx].name);
+    rdim_bake_string_chunk_list_map_insert(arena, top, map, 64, v[idx].name);
   }
 }
 
 RDI_PROC void
-rdim_bake_string_chunk_list_map_push_udt_chunk(RDIM_Arena *arena, RDIM_BakeStringChunkListMapTopology *top, RDIM_BakeStringChunkListMap *map, RDIM_UDTChunkNode *chunk)
+rdim_bake_string_chunk_list_map_push_udt_slice(RDIM_Arena *arena, RDIM_BakeStringChunkListMapTopology *top, RDIM_BakeStringChunkListMap *map, RDIM_UDT *v, RDI_U64 count)
 {
-  for(RDI_U64 idx = 0; idx < chunk->count; idx += 1)
+  for(RDI_U64 idx = 0; idx < count; idx += 1)
   {
-    for(RDIM_UDTMember *mem = chunk->v[idx].first_member; mem != 0; mem = mem->next)
+    for(RDIM_UDTMember *mem = v[idx].first_member; mem != 0; mem = mem->next)
     {
       rdim_bake_string_chunk_list_map_insert(arena, top, map, 64, mem->name);
     }
-    for(RDIM_UDTEnumVal *mem = chunk->v[idx].first_enum_val; mem != 0; mem = mem->next)
+    for(RDIM_UDTEnumVal *mem = v[idx].first_enum_val; mem != 0; mem = mem->next)
     {
       rdim_bake_string_chunk_list_map_insert(arena, top, map, 64, mem->name);
     }
@@ -1909,21 +1909,21 @@ rdim_bake_string_chunk_list_map_push_udt_chunk(RDIM_Arena *arena, RDIM_BakeStrin
 }
 
 RDI_PROC void
-rdim_bake_string_chunk_list_map_push_symbol_chunk(RDIM_Arena *arena, RDIM_BakeStringChunkListMapTopology *top, RDIM_BakeStringChunkListMap *map, RDIM_SymbolChunkNode *chunk)
+rdim_bake_string_chunk_list_map_push_symbol_slice(RDIM_Arena *arena, RDIM_BakeStringChunkListMapTopology *top, RDIM_BakeStringChunkListMap *map, RDIM_Symbol *v, RDI_U64 count)
 {
-  for(RDI_U64 idx = 0; idx < chunk->count; idx += 1)
+  for(RDI_U64 idx = 0; idx < count; idx += 1)
   {
-    rdim_bake_string_chunk_list_map_insert(arena, top, map, 1, chunk->v[idx].name);
+    rdim_bake_string_chunk_list_map_insert(arena, top, map, 1, v[idx].name);
-    rdim_bake_string_chunk_list_map_insert(arena, top, map, 1, chunk->v[idx].link_name);
+    rdim_bake_string_chunk_list_map_insert(arena, top, map, 1, v[idx].link_name);
   }
 }
 
 RDI_PROC void
-rdim_bake_string_chunk_list_map_push_scope_chunk(RDIM_Arena *arena, RDIM_BakeStringChunkListMapTopology *top, RDIM_BakeStringChunkListMap *map, RDIM_ScopeChunkNode *chunk)
+rdim_bake_string_chunk_list_map_push_scope_slice(RDIM_Arena *arena, RDIM_BakeStringChunkListMapTopology *top, RDIM_BakeStringChunkListMap *map, RDIM_Scope *v, RDI_U64 count)
 {
-  for(RDI_U64 idx = 0; idx < chunk->count; idx += 1)
+  for(RDI_U64 idx = 0; idx < count; idx += 1)
   {
-    for(RDIM_Local *local = chunk->v[idx].first_local; local != 0; local = local->next)
+    for(RDIM_Local *local = v[idx].first_local; local != 0; local = local->next)
     {
       rdim_bake_string_chunk_list_map_insert(arena, top, map, 1, local->name);
     }
@@ -1937,7 +1937,7 @@ rdim_bake_string_chunk_list_map_push_src_files(RDIM_Arena *arena, RDIM_BakeStrin
 {
   for(RDIM_SrcFileChunkNode *n = list->first; n != 0; n = n->next)
   {
-    rdim_bake_string_chunk_list_map_push_src_file_chunk(arena, top, map, n);
+    rdim_bake_string_chunk_list_map_push_src_file_slice(arena, top, map, n->v, n->count);
   }
 }
 
@@ -1946,7 +1946,7 @@ rdim_bake_string_chunk_list_map_push_units(RDIM_Arena *arena, RDIM_BakeStringChu
 {
   for(RDIM_UnitChunkNode *n = list->first; n != 0; n = n->next)
   {
-    rdim_bake_string_chunk_list_map_push_unit_chunk(arena, top, map, n);
+    rdim_bake_string_chunk_list_map_push_unit_slice(arena, top, map, n->v, n->count);
   }
 }
 
@@ -1955,7 +1955,7 @@ rdim_bake_string_chunk_list_map_push_types(RDIM_Arena *arena, RDIM_BakeStringChu
 {
   for(RDIM_TypeChunkNode *n = list->first; n != 0; n = n->next)
   {
-    rdim_bake_string_chunk_list_map_push_type_chunk(arena, top, map, n);
+    rdim_bake_string_chunk_list_map_push_type_slice(arena, top, map, n->v, n->count);
   }
 }
 
@@ -1964,7 +1964,7 @@ rdim_bake_string_chunk_list_map_push_udts(RDIM_Arena *arena, RDIM_BakeStringChun
 {
   for(RDIM_UDTChunkNode *n = list->first; n != 0; n = n->next)
   {
-    rdim_bake_string_chunk_list_map_push_udt_chunk(arena, top, map, n);
+    rdim_bake_string_chunk_list_map_push_udt_slice(arena, top, map, n->v, n->count);
   }
 }
 
@@ -1973,7 +1973,7 @@ rdim_bake_string_chunk_list_map_push_symbols(RDIM_Arena *arena, RDIM_BakeStringC
 {
   for(RDIM_SymbolChunkNode *n = list->first; n != 0; n = n->next)
   {
-    rdim_bake_string_chunk_list_map_push_symbol_chunk(arena, top, map, n);
+    rdim_bake_string_chunk_list_map_push_symbol_slice(arena, top, map, n->v, n->count);
   }
 }
 
@@ -1982,7 +1982,7 @@ rdim_bake_string_chunk_list_map_push_scopes(RDIM_Arena *arena, RDIM_BakeStringCh
 {
   for(RDIM_ScopeChunkNode *n = list->first; n != 0; n = n->next)
   {
-    rdim_bake_string_chunk_list_map_push_scope_chunk(arena, top, map, n);
+    rdim_bake_string_chunk_list_map_push_scope_slice(arena, top, map, n->v, n->count);
   }
 }
 

src/lib_raddbgi_make/raddbgi_make.h

@@ -585,13 +585,6 @@ struct RDIM_TypeChunkList
   RDI_U64 total_count;
 };
 
-typedef struct RDIM_TypeArray RDIM_TypeArray;
-struct RDIM_TypeArray
-{
-  RDIM_Type *v;
-  RDI_U64 count;
-};
-
 ////////////////////////////////
 //~ rjf: User-Defined-Type Info Types
 
@@ -1195,13 +1188,13 @@ RDI_PROC void rdim_bake_string_chunk_list_map_push_top_level_info(RDIM_Arena *ar
 RDI_PROC void rdim_bake_string_chunk_list_map_push_binary_sections(RDIM_Arena *arena, RDIM_BakeStringChunkListMapTopology *top, RDIM_BakeStringChunkListMap *map, RDIM_BinarySectionList *secs);
 RDI_PROC void rdim_bake_string_chunk_list_map_push_path_tree(RDIM_Arena *arena, RDIM_BakeStringChunkListMapTopology *top, RDIM_BakeStringChunkListMap *map, RDIM_BakePathTree *path_tree);
 
-//- rjf: chunk-granularity bake string gathering passes
+//- rjf: slice-granularity bake string gathering passes
-RDI_PROC void rdim_bake_string_chunk_list_map_push_src_file_chunk(RDIM_Arena *arena, RDIM_BakeStringChunkListMapTopology *top, RDIM_BakeStringChunkListMap *map, RDIM_SrcFileChunkNode *chunk);
+RDI_PROC void rdim_bake_string_chunk_list_map_push_src_file_slice(RDIM_Arena *arena, RDIM_BakeStringChunkListMapTopology *top, RDIM_BakeStringChunkListMap *map, RDIM_SrcFile *v, RDI_U64 count);
-RDI_PROC void rdim_bake_string_chunk_list_map_push_unit_chunk(RDIM_Arena *arena, RDIM_BakeStringChunkListMapTopology *top, RDIM_BakeStringChunkListMap *map, RDIM_UnitChunkNode *chunk);
+RDI_PROC void rdim_bake_string_chunk_list_map_push_unit_slice(RDIM_Arena *arena, RDIM_BakeStringChunkListMapTopology *top, RDIM_BakeStringChunkListMap *map, RDIM_Unit *v, RDI_U64 count);
-RDI_PROC void rdim_bake_string_chunk_list_map_push_type_chunk(RDIM_Arena *arena, RDIM_BakeStringChunkListMapTopology *top, RDIM_BakeStringChunkListMap *map, RDIM_TypeChunkNode *chunk);
+RDI_PROC void rdim_bake_string_chunk_list_map_push_type_slice(RDIM_Arena *arena, RDIM_BakeStringChunkListMapTopology *top, RDIM_BakeStringChunkListMap *map, RDIM_Type *v, RDI_U64 count);
-RDI_PROC void rdim_bake_string_chunk_list_map_push_udt_chunk(RDIM_Arena *arena, RDIM_BakeStringChunkListMapTopology *top, RDIM_BakeStringChunkListMap *map, RDIM_UDTChunkNode *chunk);
+RDI_PROC void rdim_bake_string_chunk_list_map_push_udt_slice(RDIM_Arena *arena, RDIM_BakeStringChunkListMapTopology *top, RDIM_BakeStringChunkListMap *map, RDIM_UDT *v, RDI_U64 count);
-RDI_PROC void rdim_bake_string_chunk_list_map_push_symbol_chunk(RDIM_Arena *arena, RDIM_BakeStringChunkListMapTopology *top, RDIM_BakeStringChunkListMap *map, RDIM_SymbolChunkNode *chunk);
+RDI_PROC void rdim_bake_string_chunk_list_map_push_symbol_slice(RDIM_Arena *arena, RDIM_BakeStringChunkListMapTopology *top, RDIM_BakeStringChunkListMap *map, RDIM_Symbol *v, RDI_U64 count);
-RDI_PROC void rdim_bake_string_chunk_list_map_push_scope_chunk(RDIM_Arena *arena, RDIM_BakeStringChunkListMapTopology *top, RDIM_BakeStringChunkListMap *map, RDIM_ScopeChunkNode *chunk);
+RDI_PROC void rdim_bake_string_chunk_list_map_push_scope_slice(RDIM_Arena *arena, RDIM_BakeStringChunkListMapTopology *top, RDIM_BakeStringChunkListMap *map, RDIM_Scope *v, RDI_U64 count);
 
 //- rjf: list-granularity bake string gathering passes
 RDI_PROC void rdim_bake_string_chunk_list_map_push_src_files(RDIM_Arena *arena, RDIM_BakeStringChunkListMapTopology *top, RDIM_BakeStringChunkListMap *map, RDIM_SrcFileChunkList *list);

src/raddbgi_from_pdb/raddbgi_from_pdb.c

@@ -839,6 +839,8 @@ p2r_itype_chain_build_task__entry_point(Arena *arena, void *p)
 {
   Temp scratch = scratch_begin(&arena, 1);
   P2R_ITypeChainBuildIn *in = (P2R_ITypeChainBuildIn *)p;
+  ProfScope("dependency itype chain build")
+  {
     for(CV_TypeId itype = in->itype_first; itype < in->itype_opl; itype += 1)
     {
       //- rjf: push initial itype - should be final-visited-itype for this itype
@@ -1125,6 +1127,7 @@ p2r_itype_chain_build_task__entry_point(Arena *arena, void *p)
         }
       }
     }
+  }
   scratch_end(scratch);
   return 0;
 }
@@ -3506,7 +3509,13 @@ p2r_bake_types_strings_task__entry_point(Arena *arena, void *p)
 {
   P2R_BakeTypesStringsIn *in = (P2R_BakeTypesStringsIn *)p;
   RDIM_BakeStringChunkListMap *map = rdim_bake_string_chunk_list_map_make(arena, in->top);
-  ProfScope("bake type strings") rdim_bake_string_chunk_list_map_push_types(arena, in->top, map, in->list);
+  ProfScope("bake type strings")
+  {
+    for(P2R_BakeTypesStringsInNode *n = in->first; n != 0; n = n->next)
+    {
+      rdim_bake_string_chunk_list_map_push_type_slice(arena, in->top, map, n->v, n->count);
+    }
+  }
   return map;
 }
 
@@ -3790,12 +3799,23 @@ p2r_bake(Arena *arena, P2R_Convert2Bake *in)
     }
     
     // rjf: types
+    {
+      for(RDIM_TypeChunkNode *chunk = params->types.first; chunk != 0; chunk = chunk->next)
+      {
+        U64 types_per_task = chunk->count;
+        U64 tasks_per_this_chunk = (chunk->count+types_per_task-1)/types_per_task;
+        for(U64 task_idx = 0; task_idx < tasks_per_this_chunk; task_idx += 1)
         {
           P2R_BakeTypesStringsIn *in = push_array(scratch.arena, P2R_BakeTypesStringsIn, 1);
           in->top = &bake_string_chunk_list_map_topology;
-      in->list = &params->types;
+          P2R_BakeTypesStringsInNode *n = push_array(scratch.arena, P2R_BakeTypesStringsInNode, 1);
+          SLLQueuePush(in->first, in->last, n);
+          n->v = chunk->v + task_idx*types_per_task;
+          n->count = types_per_task;
           ts_ticket_list_push(scratch.arena, &bake_string_map_build_tickets, ts_kickoff(p2r_bake_types_strings_task__entry_point, 0, in));
         }
+      }
+    }
     
     // rjf: UDTs
     {
@@ -3838,7 +3858,7 @@ p2r_bake(Arena *arena, P2R_Convert2Bake *in)
     }
   }
   
-  //- rjf: kick off pass 1 tasks
+  //- rjf: kick off name map building tasks
   P2R_BuildBakeNameMapIn build_bake_name_map_in[RDI_NameMapKind_COUNT] = {0};
   TS_Ticket build_bake_name_map_ticket[RDI_NameMapKind_COUNT] = {0};
   for(RDI_NameMapKind k = (RDI_NameMapKind)(RDI_NameMapKind_NULL+1);
@@ -3856,8 +3876,12 @@ p2r_bake(Arena *arena, P2R_Convert2Bake *in)
   {
     for(TS_TicketNode *n = bake_string_map_build_tickets.first; n != 0; n = n->next)
     {
+      ProfBegin("waiting...");
       RDIM_BakeStringChunkListMap *map = ts_join_struct(n->v, max_U64, RDIM_BakeStringChunkListMap);
+      ProfEnd();
+      ProfBegin("joining map...");
       rdim_bake_string_chunk_list_map_join_in_place(&bake_string_chunk_list_map_topology, unsorted_bake_string_chunk_list_map, map);
+      ProfEnd();
     }
   }
   
@@ -3893,8 +3917,12 @@ p2r_bake(Arena *arena, P2R_Convert2Bake *in)
   RDIM_BakeStringChunkListMap *sorted_bake_string_chunk_list_map = sorted_bake_string_chunk_list_map__in_progress;
   
   //- rjf: build finalized string map
+  ProfBegin("build finalized string map base indices");
   RDIM_BakeStringChunkListMapBaseIndices bake_string_chunk_list_map_base_idxes = rdim_bake_string_chunk_list_base_indices_from_map(arena, &bake_string_chunk_list_map_topology, sorted_bake_string_chunk_list_map);
+  ProfEnd();
+  ProfBegin("build finalized string map");
   RDIM_BakeStringMapFinal bake_strings = rdim_bake_string_map_final_from_chunk_list_map(arena, &bake_string_chunk_list_map_topology, &bake_string_chunk_list_map_base_idxes, sorted_bake_string_chunk_list_map);
+  ProfEnd();
   
   //- rjf: kick off pass 2 tasks
   P2R_BakeUnitsTopLevelIn bake_units_top_level_in = {&bake_strings, path_tree, params};

src/raddbgi_from_pdb/raddbgi_from_pdb.h

@@ -259,11 +259,20 @@ struct P2R_BakeUnitsStringsIn
   RDIM_UnitChunkList *list;
 };
 
+typedef struct P2R_BakeTypesStringsInNode P2R_BakeTypesStringsInNode;
+struct P2R_BakeTypesStringsInNode
+{
+  P2R_BakeTypesStringsInNode *next;
+  RDIM_Type *v;
+  RDI_U64 count;
+};
+
 typedef struct P2R_BakeTypesStringsIn P2R_BakeTypesStringsIn;
 struct P2R_BakeTypesStringsIn
 {
   RDIM_BakeStringChunkListMapTopology *top;
-  RDIM_TypeChunkList *list;
+  P2R_BakeTypesStringsInNode *first;
+  P2R_BakeTypesStringsInNode *last;
 };
 
 typedef struct P2R_BakeUDTsStringsIn P2R_BakeUDTsStringsIn;