plug in new vmap baking

src/rdi_make/rdi_make_local.c

@@ -169,9 +169,204 @@ rdim_bake(Arena *arena, RDIM_BakeParams *params)
     lane_sync();
     
     ////////////////////////////
-    //- rjf: sort vmap records
+    //- rjf: gather unsorted global vmap records
     //
-    ProfScope("sort vmap records")
+    VMapRecord *global_vmap_records = 0;
+    U64 global_vmap_records_count = 0;
+    ProfScope("gather unsorted global vmap records")
+    {
+      //- rjf: calculate per-lane-chunk counts
+      U64 *lane_chunk_range_counts = 0;
+      if(lane_idx() == 0)
+      {
+        lane_chunk_range_counts = push_array(scratch.arena, U64, params->global_variables.chunk_count * lane_count());
+      }
+      lane_sync_u64(&lane_chunk_range_counts, 0);
+      {
+        U64 chunk_idx = 0;
+        for EachNode(n, RDIM_SymbolChunkNode, params->global_variables.first)
+        {
+          U64 slot_idx = lane_idx()*params->global_variables.chunk_count + chunk_idx;
+          Rng1U64 range = lane_range(n->count);
+          lane_chunk_range_counts[slot_idx] += dim_1u64(range);
+          chunk_idx += 1;
+        }
+      }
+      lane_sync();
+      
+      //- rjf: calculate per-lane-chunk offsets
+      U64 *lane_chunk_range_offs = 0;
+      U64 total_range_count = 0;
+      if(lane_idx() == 0)
+      {
+        lane_chunk_range_offs = push_array(scratch.arena, U64, params->global_variables.chunk_count * lane_count());
+        U64 off = 0;
+        U64 chunk_idx = 0;
+        for EachNode(n, RDIM_SymbolChunkNode, params->global_variables.first)
+        {
+          for EachIndex(l_idx, lane_count())
+          {
+            U64 slot_idx = l_idx*params->global_variables.chunk_count + chunk_idx;
+            lane_chunk_range_offs[slot_idx] = off;
+            off += lane_chunk_range_counts[slot_idx];
+          }
+          chunk_idx += 1;
+        }
+        total_range_count = off;
+      }
+      lane_sync_u64(&lane_chunk_range_offs, 0);
+      lane_sync_u64(&total_range_count, 0);
+      
+      //- rjf: allocate records
+      if(lane_idx() == 0)
+      {
+        global_vmap_records_count = total_range_count;
+        global_vmap_records = push_array_no_zero(scratch.arena, VMapRecord, global_vmap_records_count);
+      }
+      lane_sync_u64(&global_vmap_records, 0);
+      lane_sync_u64(&global_vmap_records_count, 0);
+      
+      //- rjf: fill records
+      {
+        U64 chunk_idx = 0;
+        for EachNode(n, RDIM_SymbolChunkNode, params->global_variables.first)
+        {
+          U64 slot_idx = lane_idx()*params->global_variables.chunk_count + chunk_idx;
+          U64 off = lane_chunk_range_offs[slot_idx];
+          Rng1U64 range = lane_range(n->count);
+          for EachInRange(n_idx, range)
+          {
+            RDI_U32 global_idx  = (RDI_U32)rdim_idx_from_symbol(&n->v[n_idx]); // TODO(rjf): @u64_to_u32
+            RDI_U32 global_size = (RDI_U32)(n->v[n_idx].type ? n->v[n_idx].type->byte_size : 1);
+            RDI_U64 global_voff = n->v[n_idx].offset;
+            global_vmap_records[off].key.voff = global_voff;
+            global_vmap_records[off].key.negative_size = -global_size;
+            global_vmap_records[off].idx = global_idx;
+            off += 1;
+          }
+          chunk_idx += 1;
+        }
+      }
+    }
+    lane_sync();
+    
+    ////////////////////////////
+    //- rjf: gather unsorted unit vmap records
+    //
+    VMapRecord *unit_vmap_records = 0;
+    U64 unit_vmap_records_count = 0;
+    ProfScope("gather unsorted unit vmap records")
+    {
+      //- rjf: calculate per-lane-chunk counts
+      U64 *lane_chunk_range_counts = 0;
+      if(lane_idx() == 0)
+      {
+        lane_chunk_range_counts = push_array(scratch.arena, U64, params->units.chunk_count * lane_count());
+      }
+      lane_sync_u64(&lane_chunk_range_counts, 0);
+      {
+        U64 chunk_idx = 0;
+        for EachNode(n, RDIM_UnitChunkNode, params->units.first)
+        {
+          U64 slot_idx = lane_idx()*params->units.chunk_count + chunk_idx;
+          Rng1U64 range = lane_range(n->count);
+          for EachInRange(n_idx, range)
+          {
+            lane_chunk_range_counts[slot_idx] += n->v[n_idx].voff_ranges.total_count;
+          }
+          chunk_idx += 1;
+        }
+      }
+      lane_sync();
+      
+      //- rjf: calculate per-lane-chunk offsets
+      U64 *lane_chunk_range_offs = 0;
+      U64 total_range_count = 0;
+      if(lane_idx() == 0)
+      {
+        lane_chunk_range_offs = push_array(scratch.arena, U64, params->units.chunk_count * lane_count());
+        U64 off = 0;
+        U64 chunk_idx = 0;
+        for EachNode(n, RDIM_UnitChunkNode, params->units.first)
+        {
+          for EachIndex(l_idx, lane_count())
+          {
+            U64 slot_idx = l_idx*params->units.chunk_count + chunk_idx;
+            lane_chunk_range_offs[slot_idx] = off;
+            off += lane_chunk_range_counts[slot_idx];
+          }
+          chunk_idx += 1;
+        }
+        total_range_count = off;
+      }
+      lane_sync_u64(&lane_chunk_range_offs, 0);
+      lane_sync_u64(&total_range_count, 0);
+      
+      //- rjf: allocate records
+      if(lane_idx() == 0)
+      {
+        unit_vmap_records_count = total_range_count;
+        unit_vmap_records = push_array_no_zero(scratch.arena, VMapRecord, unit_vmap_records_count);
+      }
+      lane_sync_u64(&unit_vmap_records, 0);
+      lane_sync_u64(&unit_vmap_records_count, 0);
+      
+      //- rjf: fill records
+      {
+        U64 chunk_idx = 0;
+        for EachNode(n, RDIM_UnitChunkNode, params->units.first)
+        {
+          U64 slot_idx = lane_idx()*params->units.chunk_count + chunk_idx;
+          U64 off = lane_chunk_range_offs[slot_idx];
+          Rng1U64 range = lane_range(n->count);
+          for EachInRange(n_idx, range)
+          {
+            RDI_U32 unit_idx = (RDI_U32)rdim_idx_from_unit(&n->v[n_idx]); // TODO(rjf): @u64_to_u32
+            for EachNode(rng_n, RDIM_Rng1U64ChunkNode, n->v[n_idx].voff_ranges.first)
+            {
+              for EachIndex(rng_n_idx, rng_n->count)
+              {
+                unit_vmap_records[off].key.voff = rng_n->v[rng_n_idx].min;
+                unit_vmap_records[off].key.negative_size = -(RDI_U32)(rng_n->v[rng_n_idx].max - rng_n->v[rng_n_idx].min);
+                unit_vmap_records[off].idx = unit_idx;
+                off += 1;
+              }
+            }
+          }
+          chunk_idx += 1;
+        }
+      }
+    }
+    lane_sync();
+    
+    ////////////////////////////
+    //- rjf: sort & bake all vmaps
+    //
+    struct
+    {
+      String8 name;
+      VMapRecord *records;
+      U64 records_count;
+      RDI_VMapEntry **vmap_out;
+      U32 *vmap_count_out;
+    }
+    vmap_tasks[] =
+    {
+      {str8_lit_comp("scopes"),  scope_vmap_records,   scope_vmap_records_count,  &rdim_shared->baked_scope_vmap.vmap.vmap,  &rdim_shared->baked_scope_vmap.vmap.count},
+      {str8_lit_comp("globals"), global_vmap_records,  global_vmap_records_count, &rdim_shared->baked_global_vmap.vmap.vmap, &rdim_shared->baked_global_vmap.vmap.count},
+      {str8_lit_comp("units"),   unit_vmap_records,    unit_vmap_records_count,   &rdim_shared->baked_unit_vmap.vmap.vmap,   &rdim_shared->baked_unit_vmap.vmap.count},
+    };
+    ProfScope("sort & bake all vmaps")
+    {
+      for EachElement(vmap_task_idx, vmap_tasks) ProfScope("sort & bake vmap for %.*s", str8_varg(vmap_tasks[vmap_task_idx].name))
+      {
+        VMapRecord *records = vmap_tasks[vmap_task_idx].records;
+        U64 records_count = vmap_tasks[vmap_task_idx].records_count;
+        
+        ////////////////////////
+        //- rjf: sort
+        //
+        ProfScope("sort")
         {
           //- rjf: set up constants
           U64 bytes_per_digit = 1;
@@ -179,8 +374,6 @@ rdim_bake(Arena *arena, RDIM_BakeParams *params)
           U64 digits_count = sizeof(((VMapRecord *)0)->key)/bytes_per_digit;
           
           //- rjf: set up swap buffer / lane counters
-      U64 records_count = scope_vmap_records_count;
-      VMapRecord *records = scope_vmap_records;
           VMapRecord *records__swap = 0;
           U32 **lane_digit_counts = 0;
           U32 **lane_digit_offs = 0;
@@ -270,15 +463,12 @@ rdim_bake(Arena *arena, RDIM_BakeParams *params)
         }
         lane_sync();
         
-    ////////////////////////////
+        ////////////////////////
-    //- rjf: produce vmap
+        //- rjf: bake
         //
         RDI_VMapEntry *vmap = 0;
         RDI_U64 vmap_count = 0;
         {
-      U64 records_count = scope_vmap_records_count;
-      VMapRecord *records = scope_vmap_records;
-      
           //- rjf: allocate vmap
           RDI_U64 vmap_count__cap = records_count*2 + 1;
           if(lane_idx() == 0)
@@ -372,12 +562,25 @@ rdim_bake(Arena *arena, RDIM_BakeParams *params)
         }
         lane_sync();
         
+        ////////////////////////
+        //- rjf: store
+        //
+        if(lane_idx() == 0)
+        {
+          vmap_tasks[vmap_task_idx].vmap_out[0] = vmap;
+          vmap_tasks[vmap_task_idx].vmap_count_out[0] = vmap_count;
+        }
+      }
+    }
+    
     scratch_end(scratch);
   }
+  lane_sync();
   
   //////////////////////////////////////////////////////////////
   //- rjf: @rdim_bake_stage gather unsorted vmap keys/markers
   //
+#if 0
   ProfScope("gather unsorted vmap keys/markers")
   {
     //- rjf: gather scope vmap keys/markers
@@ -553,10 +756,12 @@ rdim_bake(Arena *arena, RDIM_BakeParams *params)
     }
   }
   lane_sync();
+#endif
   
   //////////////////////////////////////////////////////////////
   //- rjf: @rdim_bake_stage sort all vmap keys
   //
+#if 0
   ProfScope("sort all vmap keys")
   {
     // rjf: set up
@@ -666,10 +871,12 @@ rdim_bake(Arena *arena, RDIM_BakeParams *params)
     }
   }
   lane_sync();
+#endif
   
   //////////////////////////////////////////////////////////////
   //- rjf: @rdim_bake_stage bake all vmaps
   //
+#if 0
   ProfScope("bake all vmaps")
   {
     Temp scratch = scratch_begin(&arena, 1);
@@ -850,6 +1057,7 @@ rdim_bake(Arena *arena, RDIM_BakeParams *params)
     scratch_end(scratch);
   }
   lane_sync();
+#endif
   
   //////////////////////////////////////////////////////////////
   //- rjf: @rdim_bake_stage build interned path tree