more non-determinism & bug fixes in name map & idx run map building

src/lib_rdi_make/rdi_make.c

@@ -1707,10 +1707,6 @@ rdim_bake_idx_run_chunk_list_push(RDIM_Arena *arena, RDIM_BakeIdxRunChunkList *l
 RDI_PROC void
 rdim_bake_idx_run_chunk_list_concat_in_place(RDIM_BakeIdxRunChunkList *dst, RDIM_BakeIdxRunChunkList *to_push)
 {
-  for(RDIM_BakeIdxRunChunkNode *n = to_push->first; n != 0; n = n->next)
-  {
-    n->base_idx += dst->total_idx_count;
-  }
   if(dst->last != 0 && to_push->first != 0)
   {
     dst->last->next = to_push->first;
@@ -1782,7 +1778,10 @@ rdim_bake_idx_run_chunk_list_sorted_from_unsorted(RDIM_Arena *arena, RDIM_BakeId
     RDI_U64 last_idx = 0;
     for(RDI_U64 idx = 1; idx < dst.first->count; idx += 1)
     {
-      if(dst.first->v[last_idx].hash == dst.first->v[idx].hash)
+      if(dst.first->v[last_idx].count == dst.first->v[idx].count &&
+         MemoryMatch(dst.first->v[last_idx].idxes,
+                     dst.first->v[idx].idxes,
+                     sizeof(dst.first->v[idx].idxes[0]) * dst.first->v[idx].count))
       {
         rdim_memzero_struct(&dst.first->v[idx]);
         num_duplicates += 1;
@@ -1854,7 +1853,9 @@ rdim_bake_idx_run_map_loose_insert(RDIM_Arena *arena, RDIM_BakeIdxRunMapTopology
     {
       for(RDI_U64 idx = 0; idx < n->count; idx += 1)
       {
-        if(n->v[idx].hash == hash)
+        if(n->v[idx].hash == hash &&
+           n->v[idx].count == count &&
+           MemoryMatch(n->v[idx].idxes, idxes, sizeof(idxes[0])*count))
         {
           is_duplicate = 1;
           goto break_all;
@@ -1883,17 +1884,22 @@ rdim_bake_idx_from_idx_run_2(RDIM_BakeIdxRunMap2 *map, RDI_U32 *idxes, RDI_U32 c
   {
     RDI_U64 hash = rdim_hash_from_idx_run(idxes, count);
     RDI_U64 slot_idx = hash%map->slots_count;
+    RDI_U64 off = 0;
     for(RDIM_BakeIdxRunChunkNode *n = map->slots[slot_idx].first; n != 0; n = n->next)
     {
       for(RDI_U64 chunk_idx = 0; chunk_idx < n->count; chunk_idx += 1)
       {
-        if(n->v[chunk_idx].hash == hash)
+        if(n->v[chunk_idx].hash == hash &&
+           n->v[chunk_idx].count == count &&
+           MemoryMatch(n->v[chunk_idx].idxes, idxes, sizeof(idxes[0])*count))
         {
-          idx = map->slots_base_idxs[slot_idx] + n->base_idx + n->v[chunk_idx].encoding_idx;
+          idx = (RDI_U32)(map->slots_base_idxs[slot_idx] + off); // TODO(rjf): @u64_to_u32
-          break;
+          goto end_lookup;
         }
+        off += n->v[chunk_idx].count;
       }
     }
+    end_lookup:;
   }
   return idx;
 }
@@ -1946,7 +1952,19 @@ rdim_bake_name_chunk_list_concat_in_place(RDIM_BakeNameChunkList *dst, RDIM_Bake
 RSFORCEINLINE int
 rdim_bake_name_is_before(void *l, void *r)
 {
-  return str8_is_before(((RDIM_BakeName *)l)->string, ((RDIM_BakeName *)r)->string);
+  RDIM_BakeName *lhs = (RDIM_BakeName *)l;
+  RDIM_BakeName *rhs = (RDIM_BakeName *)r;
+  B32 lhs_name_lt = str8_is_before(lhs->string, rhs->string);
+  B32 is_before = lhs_name_lt;
+  if(!lhs_name_lt)
+  {
+    B32 rhs_name_lt = str8_is_before(rhs->string, lhs->string);
+    if(!rhs_name_lt)
+    {
+      is_before = (lhs->idx > rhs->idx);
+    }
+  }
+  return is_before;
 }
 
 RDI_PROC RDIM_BakeNameChunkList
@@ -1977,7 +1995,8 @@ rdim_bake_name_chunk_list_sorted_from_unsorted(RDIM_Arena *arena, RDIM_BakeNameC
     RDI_U64 last_idx = 0;
     for(RDI_U64 idx = 1; idx < dst.first->count; idx += 1)
     {
-      if(rdim_str8_match(dst.first->v[last_idx].string, dst.first->v[idx].string, 0))
+      if(rdim_str8_match(dst.first->v[last_idx].string, dst.first->v[idx].string, 0) &&
+         dst.first->v[last_idx].idx == dst.first->v[idx].idx)
       {
         rdim_memzero_struct(&dst.first->v[idx]);
         num_duplicates += 1;
@@ -2050,7 +2069,8 @@ rdim_bake_name_map_2_insert(RDIM_Arena *arena, RDIM_BakeNameMapTopology *map_top
     {
       for(RDI_U64 idx = 0; idx < n->count; idx += 1)
       {
-        if(rdim_str8_match(n->v[idx].string, string, 0))
+        if(rdim_str8_match(n->v[idx].string, string, 0) &&
+           n->v[idx].idx == idx)
         {
           is_duplicate = 1;
           goto break_all;

src/lib_rdi_make/rdi_make.h

@@ -1183,7 +1183,6 @@ struct RDIM_BakeIdxRun
   RDI_U64 hash;
   RDI_U64 count;
   RDI_U32 *idxes;
-  RDI_U64 encoding_idx;
 };
 
 typedef struct RDIM_BakeIdxRunChunkNode RDIM_BakeIdxRunChunkNode;
@@ -1193,7 +1192,6 @@ struct RDIM_BakeIdxRunChunkNode
   RDIM_BakeIdxRun *v;
   RDI_U64 count;
   RDI_U64 cap;
-  RDI_U64 base_idx;
 };
 
 typedef struct RDIM_BakeIdxRunChunkList RDIM_BakeIdxRunChunkList;
@@ -1203,7 +1201,6 @@ struct RDIM_BakeIdxRunChunkList
   RDIM_BakeIdxRunChunkNode *last;
   RDI_U64 chunk_count;
   RDI_U64 total_count;
-  RDI_U64 total_idx_count;
 };
 
 typedef struct RDIM_BakeIdxRunMapTopology RDIM_BakeIdxRunMapTopology;

src/rdi/rdi_local.c

@@ -1201,9 +1201,10 @@ rdi_dump_list_from_parsed(Arena *arena, RDI_Parsed *rdi, RDI_DumpSubsetFlags fla
             for(U32 idx_i = 0; idx_i < idx_count; idx_i += 1)
             {
               U32 idx = idx_array[idx_i];
-              str8_list_pushf(temp.arena, &idx_strings, "%u");
+              str8_list_pushf(temp.arena, &idx_strings, "%u", idx);
             }
-            indices = str8_list_join(scratch.arena, &idx_strings, &(StringJoin){.sep = str8_lit(", ")});
+            String8 extra = push_str8f(temp.arena, " // idx_run[%u]", node_ptr->match_idx_or_idx_run_first);
+            indices = str8_list_join(scratch.arena, &idx_strings, &(StringJoin){.sep = str8_lit(", "), .post = extra});
           }
           dumpf("      \"%S\": %S\n", str, indices);
           temp_end(temp);

src/rdi_from_pdb/rdi_from_pdb_2.c

@@ -3311,10 +3311,6 @@ p2r2_convert(Arena *arena, P2R_ConvertParams *params)
               // rjf: build local
               RDIM_Scope *scope = top_scope_node->scope;
               RDIM_Local *local = rdim_scope_push_local(arena, sym_scopes, scope);
-              if(str8_match(name, str8_lit("example_color_struct"), 0))
-              {
-                int x = 0;
-              }
               local->kind = local_kind;
               local->name = name;
               local->type = type;

src/rdi_make/rdi_make_local_2.c

@@ -864,23 +864,6 @@ rdim2_bake(Arena *arena, RDIM_BakeParams *params)
     }
     lane_sync();
     
-    // TODO(rjf): debugging:
-    if(lane_idx() == 0)
-    {
-      for EachIndex(slot_idx, rdim2_shared->bake_string_map_topology.slots_count)
-      {
-        if(rdim2_shared->bake_string_map__loose->slots[slot_idx] == 0) { continue; }
-        for EachNode(n, RDIM_BakeStringChunkNode, rdim2_shared->bake_string_map__loose->slots[slot_idx]->first)
-        {
-          for EachIndex(n_idx, n->count)
-          {
-            // printf("%.*s\n", str8_varg(n->v[n_idx].string));
-          }
-        }
-      }
-      fflush(stdout);
-    }
-    
     //- rjf: sort
     ProfScope("sort")
     {
@@ -925,22 +908,6 @@ rdim2_bake(Arena *arena, RDIM_BakeParams *params)
   }
   lane_sync();
   RDIM_BakeStringMapTight *bake_strings = &rdim2_shared->bake_strings;
-  // TODO(rjf): debugging:
-  if(lane_idx() == 0)
-  {
-    for EachIndex(slot_idx, bake_strings->slots_count)
-    {
-      for EachNode(n, RDIM_BakeStringChunkNode, bake_strings->slots[slot_idx].first)
-      {
-        for EachIndex(n_idx, n->count)
-        {
-          // printf("%.*s\n", str8_varg(n->v[n_idx].string));
-        }
-      }
-    }
-    fflush(stdout);
-  }
-  
   
   //////////////////////////////////////////////////////////////
   //- rjf: @rdim_bake_stage build name maps
@@ -1164,7 +1131,7 @@ rdim2_bake(Arena *arena, RDIM_BakeParams *params)
             IdxRunNode *first_idx_run_node = 0;
             IdxRunNode *last_idx_run_node = 0;
             U64 active_idx_count = 0;
-            U64 active_hash = 0;
+            String8 active_string = {0};
             RDIM_BakeNameChunkNode *n = slot->first;
             U64 n_idx = 0;
             for(;;)
@@ -1177,16 +1144,16 @@ rdim2_bake(Arena *arena, RDIM_BakeParams *params)
               }
               
               // rjf: grab next element
-              U64 hash = 0;
+              String8 string = {0};
               U64 idx = 0;
               if(n != 0)
               {
-                hash = n->v[n_idx].hash;
+                string = n->v[n_idx].string;
                 idx  = n->v[n_idx].idx;
               }
               
               // rjf: next element hash doesn't match the active? -> push index run, clear active list, start new list
-              if(hash != active_hash)
+              if(!str8_match(string, active_string, 0))
               {
                 if(active_idx_count > 1)
                 {
@@ -1202,14 +1169,15 @@ rdim2_bake(Arena *arena, RDIM_BakeParams *params)
                   }
                   rdim_bake_idx_run_map_loose_insert(arena, lane_map_top, lane_map, 4, idxs, idxs_count);
                 }
-                active_hash = hash;
+                active_string = string;
                 first_idx_run_node = 0;
                 last_idx_run_node = 0;
+                active_idx_count = 0;
                 temp_end(scratch);
               }
               
-              // rjf: hash matches the active list -> push
+              // rjf: new element matches the active list -> push
-              if(hash != 0 && hash == active_hash)
+              if(active_string.size != 0 && str8_match(string, active_string, 0))
               {
                 IdxRunNode *idx_run_n = push_array(scratch.arena, IdxRunNode, 1);
                 idx_run_n->idx = idx;
@@ -1243,6 +1211,7 @@ rdim2_bake(Arena *arena, RDIM_BakeParams *params)
         {
           RDIM_BakeIdxRunMapLoose *src_map = rdim2_shared->lane_bake_idx_run_maps__loose[src_lane_idx];
           RDIM_BakeIdxRunMapLoose *dst_map = rdim2_shared->bake_idx_run_map__loose;
+          dst_map->slots_idx_counts[slot_idx] += src_map->slots_idx_counts[slot_idx];
           if(dst_map->slots[slot_idx] == 0 && src_map->slots[slot_idx] != 0)
           {
             dst_map->slots[slot_idx] = src_map->slots[slot_idx];
@@ -1266,6 +1235,14 @@ rdim2_bake(Arena *arena, RDIM_BakeParams *params)
         if(map->slots[slot_idx] != 0 && map->slots[slot_idx]->total_count > 1)
         {
           *map->slots[slot_idx] = rdim_bake_idx_run_chunk_list_sorted_from_unsorted(arena, map->slots[slot_idx]);
+          map->slots_idx_counts[slot_idx] = 0;
+          for EachNode(n, RDIM_BakeIdxRunChunkNode, map->slots[slot_idx]->first)
+          {
+            for EachIndex(idx, n->count)
+            {
+              map->slots_idx_counts[slot_idx] += n->v[idx].count;
+            }
+          }
         }
       }
     }
@@ -1287,9 +1264,10 @@ rdim2_bake(Arena *arena, RDIM_BakeParams *params)
           rdim2_shared->bake_idx_runs.slots_base_idxs[slot_idx] = encoding_idx_off;
           if(map->slots[slot_idx] != 0)
           {
-            encoding_idx_off += map->slots[slot_idx]->total_idx_count;
+            encoding_idx_off += map->slots_idx_counts[slot_idx];
           }
         }
+        rdim2_shared->bake_idx_runs.slots_base_idxs[map_top->slots_count] = encoding_idx_off;
       }
       lane_sync();
       ProfScope("fill tight map")
@@ -1358,16 +1336,6 @@ rdim2_bake(Arena *arena, RDIM_BakeParams *params)
   }
   lane_sync();
   RDIM_StringBakeResult baked_strings = rdim2_shared->baked_strings;
-  // TODO(rjf): debugging:
-  if(lane_idx() == 0)
-  {
-    for EachIndex(string_idx, baked_strings.string_offs_count)
-    {
-      String8 string = str8(baked_strings.string_data + baked_strings.string_offs[string_idx], baked_strings.string_offs[string_idx+1]-baked_strings.string_offs[string_idx]);
-      // printf("%.*s\n", str8_varg(string));
-    }
-    fflush(stdout);
-  }
   
   //////////////////////////////////////////////////////////////
   //- rjf: @rdim_bake_stage bake idx runs
@@ -1393,7 +1361,7 @@ rdim2_bake(Arena *arena, RDIM_BakeParams *params)
           StaticAssert(sizeof(rdim2_shared->baked_idx_runs.idx_runs[0]) == sizeof(n->v[0].idxes[0]), idx_run_size_check);
           for EachIndex(n_idx, n->count)
           {
-            rdim_memcpy(rdim2_shared->baked_idx_runs.idx_runs + off, n->v[n_idx].idxes, sizeof(RDI_U32) * n->v[n_idx].count);
+            rdim_memcpy(rdim2_shared->baked_idx_runs.idx_runs + off, n->v[n_idx].idxes, sizeof(n->v[n_idx].idxes[0]) * n->v[n_idx].count);
             off += n->v[n_idx].count;
           }
         }
@@ -1531,7 +1499,6 @@ rdim2_bake(Arena *arena, RDIM_BakeParams *params)
             IdxRunNode *first_idx_run_node = 0;
             IdxRunNode *last_idx_run_node = 0;
             U64 active_idx_count = 0;
-            U64 active_hash = 0;
             String8 active_string = {0};
             RDIM_BakeNameChunkNode *n = src_slot->first;
             U64 n_idx = 0;
@@ -1545,21 +1512,19 @@ rdim2_bake(Arena *arena, RDIM_BakeParams *params)
               }
               
               // rjf: grab next element
-              U64 hash = 0;
               U64 idx = 0;
               String8 string = {0};
               if(n != 0)
               {
-                hash   = n->v[n_idx].hash;
                 idx    = n->v[n_idx].idx;
                 string = n->v[n_idx].string;
               }
               
-              // rjf: next element hash doesn't match the active? -> push index run, clear active list, start new list
+              // rjf: next element doesn't match the active list? -> push index run, clear active list, start new list
-              if(hash != active_hash)
+              if(!str8_match(active_string, string, 0))
               {
-                // rjf: has active hash -> flatten & serialize
+                // rjf: has active run -> flatten & serialize
-                if(active_hash != 0)
+                if(active_string.size != 0)
                 {
                   // rjf: flatten idxes
                   RDI_U64 idxs_count = active_idx_count;
@@ -1590,15 +1555,15 @@ rdim2_bake(Arena *arena, RDIM_BakeParams *params)
                 }
                 
                 // rjf: start new list
-                active_hash = hash;
                 active_string = string;
                 first_idx_run_node = 0;
                 last_idx_run_node = 0;
+                active_idx_count = 0;
                 temp_end(scratch);
               }
               
               // rjf: hash matches the active list -> push
-              if(hash != 0 && hash == active_hash)
+              if(active_string.size != 0 && str8_match(active_string, string, 0))
               {
                 IdxRunNode *idx_run_n = push_array(scratch.arena, IdxRunNode, 1);
                 idx_run_n->idx = idx;