From e65bc121ed4d8866ff394ddc93caa458d1586b48 Mon Sep 17 00:00:00 2001
From: Ryan Fleury <ryan.j.fleury@gmail.com>
Date: Wed, 16 Oct 2024 08:52:27 -0700
Subject: [PATCH] more work on pdb -> rdi determinism; almost done hitting all
 of the issues...

---
 src/lib_rdi_make/rdi_make.c     | 53 ++++++++++++++++++++++++++++++++-
 src/rdi_from_pdb/rdi_from_pdb.c |  2 +-
 src/tester/tester_main.c        |  2 +-
 3 files changed, 54 insertions(+), 3 deletions(-)

diff --git a/src/lib_rdi_make/rdi_make.c b/src/lib_rdi_make/rdi_make.c
index 539a45e5..59961a01 100644
--- a/src/lib_rdi_make/rdi_make.c
+++ b/src/lib_rdi_make/rdi_make.c
@@ -1345,7 +1345,7 @@ rdim_bake_string_chunk_list_sorted_from_unsorted(RDIM_Arena *arena, RDIM_BakeStr
       // build new sort tasks for buckets with many elements
       {
         RDI_U64 write_idx = 0;
-        for(U64 bucket_idx = 0; bucket_idx < 256; bucket_idx += 1)
+        for(RDI_U64 bucket_idx = 0; bucket_idx < 256; bucket_idx += 1)
         {
           // rjf: write each chunk node's array into original array, detect if there is size left to sort
           RDI_U64 bucket_base_idx = write_idx;
@@ -1379,6 +1379,57 @@ rdim_bake_string_chunk_list_sorted_from_unsorted(RDIM_Arena *arena, RDIM_BakeStr
     scratch_end(scratch);
   }
   
+  //- rjf: iterate sorted chunk node, remove duplicates, count # to pop
+  RDI_U64 num_to_pop = 0;
+  RDI_U64 arena_pos_pre_pop = rdim_arena_pos(arena);
+  if(dst.first != 0)
+  {
+    RDI_U64 last_idx = 0;
+    for(RDI_U64 idx = 1; idx < dst.first->count; idx += 1)
+    {
+      if(rdim_str8_match(dst.first->v[last_idx].string, dst.first->v[idx].string, 0))
+      {
+        rdim_memzero_struct(&dst.first->v[idx]);
+        num_to_pop += 1;
+      }
+      else
+      {
+        last_idx = idx;
+      }
+    }
+  }
+  
+  //- rjf: iterate sorted chunk node, make non-empty elements contiguous
+  if(num_to_pop != 0)
+  {
+    RDI_U64 last_idx = 0;
+    for(RDI_U64 idx = 1; idx < dst.first->count; idx += 1)
+    {
+      if(dst.first->v[idx].string.RDIM_String8_SizeMember == 0 &&
+         dst.first->v[idx].hash == 0)
+      {
+        last_idx = idx;
+      }
+      else if(last_idx != 0 && last_idx != idx)
+      {
+        rdim_memcpy_struct(&dst.first->v[last_idx], &dst.first->v[idx]);
+        rdim_memzero_struct(&dst.first->v[idx]);
+        idx = last_idx;
+        last_idx = 0;
+      }
+    }
+    
+    //- rjf: pop extras
+    if(num_to_pop != 0)
+    {
+      rdim_arena_pop_to(arena, arena_pos_pre_pop - num_to_pop*sizeof(dst.first->v[0]));
+      dst.first->count -= num_to_pop;
+      dst.first->cap -= num_to_pop;
+      dst.total_count -= num_to_pop;
+    }
+  }
+  
+  
   return dst;
 }
 
diff --git a/src/rdi_from_pdb/rdi_from_pdb.c b/src/rdi_from_pdb/rdi_from_pdb.c
index 686d472d..f6e5b697 100644
--- a/src/rdi_from_pdb/rdi_from_pdb.c
+++ b/src/rdi_from_pdb/rdi_from_pdb.c
@@ -4639,7 +4639,7 @@ p2r_bake(Arena *arena, P2R_Convert2Bake *in)
             {
               for EachIndex(idx, n->count)
               {
-                printf("string: %.*s\n", str8_varg(n->v[idx].string));
+                printf("string: \"%.*s\"\n", str8_varg(n->v[idx].string));
               }
             }
           }
diff --git a/src/tester/tester_main.c b/src/tester/tester_main.c
index 8b7060c8..5e6befd7 100644
--- a/src/tester/tester_main.c
+++ b/src/tester/tester_main.c
@@ -45,7 +45,7 @@ entry_point(CmdLine *cmdline)
   String8List out = {0};
   {
     name = str8_lit("pdb2rdi_determinism");
-    U64 num_repeats_per_pdb = 4;
+    U64 num_repeats_per_pdb = 16;
     String8 pdb_paths[] =
     {
       // str8_lit_comp("odintest/test.pdb"),