d2r2 type deduplication: further work on type content hashing; correct .debug_str_offsets handling; sketch out type dependency chain build

src/base/base_math.c

@@ -758,34 +758,32 @@ rng1u64_array_from_list(Arena *arena, Rng1U64List *list)
 }
 
 internal U64
-rng_1u64_array_bsearch(Rng1U64Array arr, U64 value)
+rng1u64_array_num_from_value__binary_search(Rng1U64Array *array, U64 value)
 {
-  if(arr.count > 0 && arr.v[0].min < value && value < arr.v[arr.count-1].max)
+  U64 result = 0;
+  if(array->count > 0 && array->v[0].min <= value && value < array->v[array->count-1].max)
   {
-    U64 l = 0;
+    U64 min_idx = 0;
-    U64 r = arr.count - 1;
+    U64 max_idx = array->count - 1;
-    for(; l <= r; )
+    for(;min_idx <= max_idx;)
     {
-      U64 m = l + (r - l) / 2;
+      U64 mid_idx = min_idx + (max_idx - min_idx) / 2;
-      if(contains_1u64(arr.v[m], value))
+      if(contains_1u64(array->v[mid_idx], value))
       {
-        return m;
+        result = mid_idx+1;
+        break;
       }
-      else if(arr.v[m].min < value)
+      else if(array->v[mid_idx].min < value)
       {
-        l = m + 1;
+        min_idx = mid_idx + 1;
       }
       else
       {
-        r = m - 1;
+        max_idx = mid_idx - 1;
       }
     }
   }
-  else if(arr.count == 1 && contains_1u64(arr.v[0], value))
+  return result;
-  {
-    return 0;
-  }
-  return max_U64;
 }
 
 internal void

src/base/base_math.h

@@ -682,7 +682,7 @@ internal void rng1u64_list_push_node(Rng1U64List *list, Rng1U64Node *n);
 internal Rng1U64Node * rng1u64_list_push(Arena *arena, Rng1U64List *list, Rng1U64 rng);
 internal void rng1u64_list_concat(Rng1U64List *list, Rng1U64List *to_concat);
 internal Rng1U64Array rng1u64_array_from_list(Arena *arena, Rng1U64List *list);
-internal U64 rng_1u64_array_bsearch(Rng1U64Array arr, U64 value);
+internal U64 rng1u64_array_num_from_value__binary_search(Rng1U64Array *array, U64 value);
 
 internal void rng1s64_list_push(Arena *arena, Rng1S64List *list, Rng1S64 rng);
 internal Rng1S64Array rng1s64_array_from_list(Arena *arena, Rng1S64List *list);

src/dwarf/dwarf.c

@@ -280,11 +280,11 @@ internal U64
 dw_size_from_format(DW_Format format)
 {
   U64 result = 0;
-  switch (format) {
+  switch(format)
-    case DW_Format_Null: break;
+  {
-    case DW_Format_32Bit: result = 4; break;
+    case DW_Format_Null:{}break;
-    case DW_Format_64Bit: result = 8; break;
+    case DW_Format_32Bit:{result = 4;}break;
-    default: InvalidPath; break;
+    case DW_Format_64Bit:{result = 8;}break;
   }
   return result;
 }

src/dwarf/dwarf_parse.c

@@ -2072,16 +2072,16 @@ dw_cu_from_info_off(Arena *arena, DW_Raw *input, DW_ListUnitInput lu_input, U64
     U64 loclists_sec_off    = dw_interp_sec_offset(loclists_base_attrib->form   );
     
     // map section offset to unit index
-    U64 addr_lu_idx        = rng_1u64_array_bsearch(lu_input.addr_ranges,       addr_sec_off       );
+    U64 addr_lu_num        = rng1u64_array_num_from_value__binary_search(&lu_input.addr_ranges,       addr_sec_off       );
-    U64 str_offsets_lu_idx = rng_1u64_array_bsearch(lu_input.str_offset_ranges, str_offsets_sec_off);
+    U64 str_offsets_lu_num = rng1u64_array_num_from_value__binary_search(&lu_input.str_offset_ranges, str_offsets_sec_off);
-    U64 rnglists_lu_idx    = rng_1u64_array_bsearch(lu_input.rnglist_ranges,    rnglists_sec_off   );
+    U64 rnglists_lu_num    = rng1u64_array_num_from_value__binary_search(&lu_input.rnglist_ranges,    rnglists_sec_off   );
-    U64 loclists_lu_idx    = rng_1u64_array_bsearch(lu_input.loclist_ranges,    loclists_sec_off   );
+    U64 loclists_lu_num    = rng1u64_array_num_from_value__binary_search(&lu_input.loclist_ranges,    loclists_sec_off   );
     
     // map index to unit
-    DW_ListUnit *addr_lu        = addr_lu_idx        < lu_input.addr_count       ? &lu_input.addrs[addr_lu_idx]              : 0;
+    DW_ListUnit *addr_lu        = 0 < (addr_lu_num && addr_lu_num <= lu_input.addr_count)                     ? &lu_input.addrs[addr_lu_num-1]              : 0;
-    DW_ListUnit *str_offsets_lu = str_offsets_lu_idx < lu_input.str_offset_count ? &lu_input.str_offsets[str_offsets_lu_idx] : 0;
+    DW_ListUnit *str_offsets_lu = 0 < (str_offsets_lu_num && str_offsets_lu_num <= lu_input.str_offset_count) ? &lu_input.str_offsets[str_offsets_lu_num-1] : 0;
-    DW_ListUnit *rnglists_lu    = rnglists_lu_idx    < lu_input.rnglist_count    ? &lu_input.rnglists[rnglists_lu_idx]       : 0;
+    DW_ListUnit *rnglists_lu    = 0 < (rnglists_lu_num && rnglists_lu_num <= lu_input.rnglist_count)          ? &lu_input.rnglists[rnglists_lu_num-1]       : 0;
-    DW_ListUnit *loclists_lu    = loclists_lu_idx    < lu_input.loclist_count    ? &lu_input.loclists[loclists_lu_idx]       : 0;
+    DW_ListUnit *loclists_lu    = 0 < (loclists_lu_num && loclists_lu_num <= lu_input.loclist_count)          ? &lu_input.loclists[loclists_lu_num-1]       : 0;
     
     // find compile unit base address
     DW_Attrib *low_pc_attrib = dw_attrib_from_tag(0, 0, cu_tag, DW_AttribKind_LowPc);

src/dwarf/dwarf_parse_2.c

@@ -399,8 +399,19 @@ dw2_read_form_val(DW2_ParseCtx *ctx, String8 data, U64 off, DW_FormKind form_kin
         case DW_Form_Strx3:
         case DW_Form_Strx4:
         case DW_Form_Strx:
+        if(ctx->str_offsets_table != 0)
         {
-          // TODO(rjf)
+          U64 entry_idx = val.u128.u64[0];
+          if(entry_idx < ctx->str_offsets_table->entries_count)
+          {
+            U64 entry_size = ctx->str_offsets_table->entry_size;
+            U64 entry_off = entry_idx * entry_size;
+            U64 string_data_off = 0;
+            MemoryCopy(&string_data_off, (U8 *)ctx->str_offsets_table->entries + entry_off, entry_size);
+            String8 string_section_data = ctx->raw->sec[DW_Section_Str].data;
+            val.string = str8_cstring_capped(string_section_data.str + string_data_off,
+                                             string_section_data.str + string_section_data.size);
+          }
         }break;
         case DW_Form_LineStrp:
         {
@@ -545,6 +556,25 @@ dw2_attrib_from_kind(DW2_Tag *tag, DW_AttribKind kind)
   return result;
 }
 
+internal U64
+dw2_reference_info_off_from_form_val(DW2_ParseCtx *ctx, DW2_FormVal *v)
+{
+  U64 result = 0;
+  switch(v->kind)
+  {
+    default:{}break;
+    case DW_Form_Ref1:
+    case DW_Form_Ref2:
+    case DW_Form_Ref4:
+    case DW_Form_Ref8:
+    {
+      result = v->u128.u64[0];
+    }break;
+    // TODO(rjf): DW_Form_RefAddr, DW_Form_RefUData, DW_Form_RefSig8, DW_Form_RefSup8, etc.
+  }
+  return result;
+}
+
 ////////////////////////////////
 //~ rjf: Line Table Parsing
 
@@ -831,3 +861,42 @@ dw2_read_line_table_header(Arena *arena, DW2_ParseCtx *ctx, String8 data, U64 of
   scratch_end(scratch);
   return bytes_read;
 }
+
+////////////////////////////////
+//~ rjf: String Offset Table Parsing (.debug_str_offsets)
+
+internal U64
+dw2_read_str_offsets_table(String8 data, U64 off, DW2_StrOffsetsTable *out)
+{
+  U64 start_off = off;
+  {
+    // rjf: read data length / format
+    U64 unit_data_length = 0;
+    DW_Format format = DW_Format_Null;
+    off += dw2_read_initial_length(data, off, &unit_data_length, &format);
+    U64 unit_data_off_opl = off + unit_data_length;
+    
+    // rjf: read version
+    DW_Version version = DW_Version_Null;
+    off += str8_deserial_read_struct(data, off, &version);
+    
+    // rjf: version 5: read rest (this section only exists in 5+)
+    if(version == DW_Version_5)
+    {
+      // rjf: skip padding
+      off += sizeof(U16);
+      
+      // rjf: fill table info
+      out->format        = format;
+      out->version       = version;
+      out->entry_size    = dw_size_from_format(format);
+      out->entries_count = (unit_data_off_opl - off) / out->entry_size;
+      out->entries       = data.str + off;
+      
+      // rjf: skip table
+      off = unit_data_off_opl;
+    }
+  }
+  U64 bytes_read = (off - start_off);
+  return bytes_read;
+}

src/dwarf/dwarf_parse_2.h

@@ -36,6 +36,19 @@ struct DW2_AbbrevMap
   U64 slots_count;
 };
 
+////////////////////////////////
+//~ rjf: String Offset Table (.debug_str_offsets)
+
+typedef struct DW2_StrOffsetsTable DW2_StrOffsetsTable;
+struct DW2_StrOffsetsTable
+{
+  DW_Format format;
+  DW_Version version;
+  U64 entry_size;
+  U64 entries_count;
+  void *entries;
+};
+
 ////////////////////////////////
 //~ rjf: Parsing Context Bundle
 
@@ -49,6 +62,7 @@ struct DW2_ParseCtx
   DW_Format format;
   U64 addr_size;
   DW2_AbbrevMap *abbrev_map;
+  DW2_StrOffsetsTable *str_offsets_table;
   String8 unit_dir;
   String8 unit_file;
 };
@@ -210,10 +224,16 @@ internal U64 dw2_read_form_val(DW2_ParseCtx *ctx, String8 data, U64 off, DW_Form
 
 internal U64 dw2_read_tag(Arena *arena, DW2_ParseCtx *ctx, String8 data, U64 off, DW2_Tag *tag_out);
 internal DW2_Attrib *dw2_attrib_from_kind(DW2_Tag *tag, DW_AttribKind kind);
+internal U64 dw2_reference_info_off_from_form_val(DW2_ParseCtx *ctx, DW2_FormVal *v);
 
 ////////////////////////////////
 //~ rjf: Line Table Parsing
 
 internal U64 dw2_read_line_table_header(Arena *arena, DW2_ParseCtx *ctx, String8 data, U64 off, DW2_LineTableHeader *out);
 
+////////////////////////////////
+//~ rjf: String Offset Table Parsing (.debug_str_offsets)
+
+internal U64 dw2_read_str_offsets_table(String8 data, U64 off, DW2_StrOffsetsTable *out);
+
 #endif // DWARF_PARSE_2_H

src/linker/lnk_debug_info.c

@@ -2415,7 +2415,8 @@ THREAD_POOL_TASK_FUNC(lnk_push_dbi_sec_contrib_task)
       if (obj_sect_header->vsize == 0) {
         continue;
       }
-      sect_number = rng_1u64_array_bsearch(task->image_section_virt_ranges, obj_sect_header->voff);
+      U64 sect_num = rng1u64_array_num_from_value__binary_search(&task->image_section_virt_ranges, obj_sect_header->voff);
+      sect_number = sect_num-1;
       Assert(sect_number < task->image_section_virt_ranges.count);
       sect_data   = str8_zero();
       sect_off    = obj_sect_header->voff - task->image_section_virt_ranges.v[sect_number].min;
@@ -2424,7 +2425,8 @@ THREAD_POOL_TASK_FUNC(lnk_push_dbi_sec_contrib_task)
       if (obj_sect_header->fsize == 0) {
         continue;
       }
-      sect_number = rng_1u64_array_bsearch(task->image_section_file_ranges, obj_sect_header->foff);
+      U64 sect_num = rng1u64_array_num_from_value__binary_search(&task->image_section_file_ranges, obj_sect_header->foff);
+      sect_number = sect_num-1;
       Assert(sect_number < task->image_section_file_ranges.count);
       sect_data   = str8_substr(task->image_data, rng_1u64(obj_sect_header->foff, obj_sect_header->foff + obj_sect_header->fsize));
       sect_off    = obj_sect_header->foff - task->image_section_file_ranges.v[sect_number].min;

src/rdi_from_dwarf/rdi_from_dwarf_2.c

@@ -1,25 +1,6 @@
 // Copyright (c) Epic Games Tools
 // Licensed under the MIT license (https://opensource.org/license/mit/)
 
-////////////////////////////////
-//~ rjf: Helpers
-
-internal U64
-d2r2_hash_from_seed_tag(U64 seed, DW2_Tag *tag)
-{
-  U64 result = seed;
-  result = u64_hash_from_seed_str8(result, str8_struct(&tag->kind));
-  for EachNode(n, DW2_AttribNode, tag->attribs.first)
-  {
-    DW2_Attrib *attrib = &n->v;
-    result = u64_hash_from_seed_str8(result, str8_struct(&attrib->attrib_kind));
-    result = u64_hash_from_seed_str8(result, str8_struct(&attrib->val.kind));
-    result = u64_hash_from_seed_str8(result, attrib->val.string);
-    result = u64_hash_from_seed_str8(result, str8_struct(&attrib->val.u128));
-  }
-  return result;
-}
-
 ////////////////////////////////
 //~ rjf: Main Conversion Entry Point (New)
 
@@ -276,6 +257,86 @@ d2r2_convert(Arena *arena, D2R2_ConvertParams *params)
   }
   lane_sync();
   
+  ////////////////////////////
+  //- rjf: parse all string offsets tables
+  //
+  // in an incredible twist of fate, DWARF decided to decouple these from
+  // compilation units. compilation units *do* contain a
+  // DW_AttribKind_StrOffsetsBase attribute. but this base offset does *not*
+  // point to the beginning of a table in .debug_str_offsets! it instead points
+  // PAST THE INITIAL VARIABLE-WIDTH LENGTH AND FORMAT ENCODING! this means
+  // you can't actually use the StrOffsetsBase attribute for ANYTHING other
+  // than correllating a unit to its associated string offset table - but you
+  // *necessarily needed to have parsed that table beforehand*, completely
+  // independently from units.
+  //
+  // so, we have to parse all the string offset tables up-front, then
+  // *binary search* their ranges to determine which unit has which table.
+  //
+  // of course, in practice, it's perhaps likely/expected that these match
+  // one-to-one with units, and in the same order, because that is what is
+  // most natural for generators. but, the format does not *guarantee this*,
+  // and instead specced something far more arbitrary.
+  //
+  // thank you, again, DWARF.
+  //
+  U64 str_offsets_tables_count = 0;
+  DW2_StrOffsetsTable *str_offsets_tables = 0;
+  Rng1U64 *str_offsets_tables_ranges = 0;
+  ProfScope("parse all string offsets tables") if(lane_idx() == 0)
+  {
+    Temp scratch2 = scratch_begin(&scratch.arena, 1);
+    
+    //- rjf: gather all tables (loose)
+    typedef struct TableNode TableNode;
+    struct TableNode
+    {
+      TableNode *next;
+      DW2_StrOffsetsTable v;
+      Rng1U64 range;
+    };
+    TableNode *first_table = 0;
+    TableNode *last_table = 0;
+    U64 table_count = 0;
+    for(U64 off = 0; off < raw->sec[DW_Section_StrOffsets].data.size;)
+    {
+      U64 start_off = off;
+      DW2_StrOffsetsTable table = {0};
+      off += dw2_read_str_offsets_table(raw->sec[DW_Section_StrOffsets].data, off, &table);
+      if(table.entries != 0)
+      {
+        TableNode *n = push_array(scratch2.arena, TableNode, 1);
+        SLLQueuePush(first_table, last_table, n);
+        n->v = table;
+        n->range = r1u64(start_off, off);
+        table_count += 1;
+      }
+      if(off == start_off)
+      {
+        break;
+      }
+    }
+    
+    //- rjf: tighten
+    str_offsets_tables_count = table_count;
+    str_offsets_tables = push_array(scratch.arena, DW2_StrOffsetsTable, str_offsets_tables_count);
+    str_offsets_tables_ranges = push_array(scratch.arena, Rng1U64, str_offsets_tables_count);
+    {
+      U64 idx = 0;
+      for EachNode(n, TableNode, first_table)
+      {
+        str_offsets_tables[idx] = n->v;
+        str_offsets_tables_ranges[idx] = n->range;
+        idx += 1;
+      }
+    }
+    
+    scratch_end(scratch2);
+  }
+  lane_sync_u64(&str_offsets_tables_count, 0);
+  lane_sync_u64(&str_offsets_tables, 0);
+  lane_sync_u64(&str_offsets_tables_ranges, 0);
+  
   ////////////////////////////
   //- rjf: build per-unit parsing contexts
   //
@@ -301,7 +362,58 @@ d2r2_convert(Arena *arena, D2R2_ConvertParams *params)
   lane_sync();
   
   ////////////////////////////
-  //- rjf: parse each unit's root-level tag
+  //- rjf: do initial parse of each unit's root-level tag
+  //
+  // an incredible NOTE: this is actually not sufficient. because this tag parse
+  // is what informs us which .debug_str_offsets table each unit should be associated
+  // with (via the StrOffsetsBase attribute), we actually don't have the right string
+  // offset table *before* we parse this, which means we can't resolve some of the
+  // string attribute values.
+  //
+  // so, in another incredible twist of fate, we actually must do this *twice*. first,
+  // to find *just the StrOffsetsBase*, and then, to actually fully resolve everything.
+  //
+  DW2_Tag *unit_root_tags__pre_str_offsets = 0;
+  {
+    if(lane_idx() == 0)
+    {
+      unit_root_tags__pre_str_offsets = push_array(scratch.arena, DW2_Tag, unit_count);
+    }
+    lane_sync_u64(&unit_root_tags__pre_str_offsets, 0);
+    Rng1U64 range = lane_range(unit_count);
+    for EachInRange(unit_idx, range)
+    {
+      dw2_read_tag(scratch.arena, &unit_parse_ctxs[unit_idx], raw->sec[DW_Section_Info].data, unit_info_tag_ranges[unit_idx].min, &unit_root_tags__pre_str_offsets[unit_idx]);
+    }
+  }
+  lane_sync();
+  
+  ////////////////////////////
+  //- rjf: look up string offset tables for each unit (.debug_str_offsets);
+  // equip to per-unit parsing contexts
+  //
+  {
+    Rng1U64Array str_offsets_tables_ranges_array = {str_offsets_tables_ranges, str_offsets_tables_count};
+    Rng1U64 range = lane_range(unit_count);
+    for EachInRange(unit_idx, range)
+    {
+      DW2_Tag *unit_root_tag = &unit_root_tags__pre_str_offsets[unit_idx];
+      DW2_Attrib *str_offsets_base_off_attrib = dw2_attrib_from_kind(unit_root_tag, DW_AttribKind_StrOffsetsBase);
+      U64 str_offsets_base_off = str_offsets_base_off_attrib->val.u128.u64[0];
+      U64 str_offsets_table_num = rng1u64_array_num_from_value__binary_search(&str_offsets_tables_ranges_array, str_offsets_base_off);
+      if(0 < str_offsets_table_num && str_offsets_table_num <= str_offsets_tables_ranges_array.count)
+      {
+        DW2_StrOffsetsTable *table = &str_offsets_tables[str_offsets_table_num-1];
+        unit_parse_ctxs[unit_idx].str_offsets_table = table;
+      }
+    }
+  }
+  lane_sync();
+  
+  ////////////////////////////
+  //- rjf: do parse of each unit's root tag AFTER finding the right .debug_str_offsets table
+  //
+  // (excellent work everyone - thank you for reminding me why RDI is necessary)
   //
   DW2_Tag *unit_root_tags = 0;
   {
@@ -888,9 +1000,9 @@ d2r2_convert(Arena *arena, D2R2_ConvertParams *params)
   U64 total_tag_count_estimate = 1;
   {
     U64 tag_size_estimate = 32;
-    for EachIndex(unit_idx, unit_info_ranges->count)
+    for EachIndex(unit_idx, unit_count)
     {
-      total_tag_count_estimate += dim_1u64(unit_info_ranges->v[unit_idx]) / tag_size_estimate;
+      total_tag_count_estimate += dim_1u64(unit_info_tag_ranges[unit_idx]) / tag_size_estimate;
     }
   }
   
@@ -921,6 +1033,7 @@ d2r2_convert(Arena *arena, D2R2_ConvertParams *params)
   UniqueTypeTagNode **unique_type_tag_slots = 0;
   U64 unique_type_tag_slots_count = total_tag_count_estimate/8 + 1;
   UnitTypeMap *unit_type_maps = 0;
+  ProfScope("gather all unique type tags across all units")
   {
     if(lane_idx() == 0)
     {
@@ -935,108 +1048,173 @@ d2r2_convert(Arena *arena, D2R2_ConvertParams *params)
     for(;;)
     {
       //- rjf: take next unit
-      U64 unit_idx = ins_atomic_u64_inc_eval(unit_take_idx_ptr) - 1;
+      U64 origin_unit_idx = ins_atomic_u64_inc_eval(unit_take_idx_ptr) - 1;
-      if(unit_idx >= unit_count)
+      if(origin_unit_idx >= unit_count)
       {
         break;
       }
       
-      //- rjf: unpack unit
+      //- rjf: unpack unit info
-      DW2_ParseCtx *unit_parse_ctx = &unit_parse_ctxs[unit_idx];
+      Rng1U64 origin_unit_info_tag_range = unit_info_tag_ranges[origin_unit_idx];
-      Rng1U64 unit_info_range = unit_info_ranges->v[unit_idx];
       
       //- rjf: set up type map for this unit
-      unit_type_maps[unit_idx].slots_count = dim_1u64(unit_info_range) / 256 + 1;
+      unit_type_maps[origin_unit_idx].slots_count = dim_1u64(origin_unit_info_tag_range) / 256 + 1;
-      unit_type_maps[unit_idx].slots = push_array(scratch.arena, UnitTypeNode *, unit_type_maps[unit_idx].slots_count);
+      unit_type_maps[origin_unit_idx].slots = push_array(scratch.arena, UnitTypeNode *, unit_type_maps[origin_unit_idx].slots_count);
       
-      //- rjf: find all type tags in this unit
+      //- rjf: hash all type content from tags in this unit; record if unique
-      for(U64 off = unit_info_range.min; off < unit_info_range.max;)
+      for(U64 off = origin_unit_info_tag_range.min; off < origin_unit_info_tag_range.max;)
       {
         Temp scratch2 = scratch_begin(&scratch.arena, 1);
         U64 start_off = off;
         
-        // rjf: read next tag
+        //- rjf: hash type tags - this requires a hash of not only type tags'
-        DW2_Tag tag = {0};
+        // attributes, but also a walk of all other type tags this tag references,
-        off += dw2_read_tag(scratch2.arena, unit_parse_ctx, raw->sec[DW_Section_Info].data, off, &tag);
+        // and a hash of them too. so we produce a list of tasks for
+        // parsing/hashing tags, in order to find the full comprehensive hash
+        // for each type tag.
+        //
+        B32 is_type_tag_tree = 0;
+        U64 hash = 0;
+        {
+          typedef struct TypeTagTask TypeTagTask;
+          struct TypeTagTask
+          {
+            TypeTagTask *next;
+            U64 unit_idx;
+            U64 off;
+          };
+          TypeTagTask start_task = {0, origin_unit_idx, off};
+          TypeTagTask *first_task = &start_task;
+          TypeTagTask *last_task = first_task;
+          TypeTagTask *free_task = 0;
+          for(TypeTagTask *t = first_task; t != 0; t = t->next)
+          {
+            U64 t_off = t->off;
             
-        // rjf: if offset not advanced -> increment
+            // rjf: unpack unit
+            Rng1U64 unit_info_range = unit_info_ranges->v[t->unit_idx];
+            DW2_ParseCtx *unit_parse_ctx = &unit_parse_ctxs[t->unit_idx];
+            
+            // rjf: read/hash the full tag tree at `t_off`; kick off additional
+            // tasks for referenced dependency types
+            U64 depth = 0;
+            for(;unit_info_range.min <= t_off && t_off < unit_info_range.max;)
+            {
+              U64 t_start_off = t_off;
+              
+              // rjf: read tag
+              DW2_Tag tag = {0};
+              t_off += dw2_read_tag(scratch2.arena, unit_parse_ctx, raw->sec[DW_Section_Info].data, t_off, &tag);
+              
+              // rjf: record top-level info about this tag tree
+              if(t_start_off == t->off && t == &start_task)
+              {
+                is_type_tag_tree = (tag.kind == DW_TagKind_ArrayType || tag.kind == DW_TagKind_ClassType ||
+                                    tag.kind == DW_TagKind_EnumerationType || tag.kind == DW_TagKind_PointerType ||
+                                    tag.kind == DW_TagKind_ReferenceType || tag.kind == DW_TagKind_StringType ||
+                                    tag.kind == DW_TagKind_StructureType || tag.kind == DW_TagKind_SubroutineType ||
+                                    tag.kind == DW_TagKind_Typedef || tag.kind == DW_TagKind_UnionType ||
+                                    tag.kind == DW_TagKind_PtrToMemberType || tag.kind == DW_TagKind_SetType ||
+                                    tag.kind == DW_TagKind_SubrangeType || tag.kind == DW_TagKind_BaseType ||
+                                    tag.kind == DW_TagKind_ConstType || tag.kind == DW_TagKind_FileType ||
+                                    tag.kind == DW_TagKind_PackedType || tag.kind == DW_TagKind_VolatileType ||
+                                    tag.kind == DW_TagKind_RestrictType || tag.kind == DW_TagKind_InterfaceType ||
+                                    tag.kind == DW_TagKind_UnspecifiedType || tag.kind == DW_TagKind_SharedType ||
+                                    tag.kind == DW_TagKind_RValueReferenceType || tag.kind == DW_TagKind_CoarrayType ||
+                                    tag.kind == DW_TagKind_DynamicType || tag.kind == DW_TagKind_AtomicType ||
+                                    tag.kind == DW_TagKind_ImmutableType);
+              }
+              
+              // rjf: is type -> combine tag's content into hash
+              if(is_type_tag_tree)
+              {
+                // rjf: combine tag's kind
+                hash = u64_hash_from_seed_str8(hash, str8_struct(&tag.kind));
+                
+                // rjf: combine non-reference attributes (references could be different,
+                // because of deduping, but they could match ultimately). for any
+                // referenced dependency types, kick them off
+                for(DW2_AttribNode *n = tag.attribs.first; n != 0; n = n->next)
+                {
+                  if(n->v.val.kind != DW_Form_RefAddr &&
+                     n->v.val.kind != DW_Form_Ref1 &&
+                     n->v.val.kind != DW_Form_Ref2 &&
+                     n->v.val.kind != DW_Form_Ref4 &&
+                     n->v.val.kind != DW_Form_Ref8 &&
+                     n->v.val.kind != DW_Form_RefUData &&
+                     n->v.val.kind != DW_Form_RefSup4 &&
+                     n->v.val.kind != DW_Form_RefSig8)
+                  {
+                    hash = u64_hash_from_seed_str8(hash, str8_struct(&n->v.val.kind));
+                    hash = u64_hash_from_seed_str8(hash, str8_struct(&n->v.val.u128));
+                    hash = u64_hash_from_seed_str8(hash, n->v.val.string);
+                  }
+                  if(n->v.attrib_kind == DW_AttribKind_Type)
+                  {
+                    TypeTagTask *dependency_task = free_task;
+                    if(dependency_task != 0)
+                    {
+                      SLLStackPop(free_task);
+                    }
+                    else
+                    {
+                      dependency_task = push_array(scratch2.arena, TypeTagTask, 1);
+                    }
+                    dependency_task->off = dw2_reference_info_off_from_form_val(unit_parse_ctx, &n->v.val);
+                    dependency_task->unit_idx = t->unit_idx;
+                    if(!contains_1u64(unit_info_range, dependency_task->off))
+                    {
+                      U64 new_unit_num = rng1u64_array_num_from_value__binary_search(unit_info_ranges, dependency_task->off);
+                      if(0 < new_unit_num && new_unit_num <= unit_count)
+                      {
+                        dependency_task->unit_idx = new_unit_num-1;
+                      }
+                    }
+                  }
+                }
+              }
+              
+              // rjf: is type tag tree, has children -> descend
+              if(is_type_tag_tree && tag.has_children)
+              {
+                depth += 1;
+              }
+              
+              // rjf: zero tag kind -> ascend
+              if(is_type_tag_tree && tag.kind == DW_TagKind_Null && depth > 0)
+              {
+                depth -= 1;
+              }
+              
+              // rjf: no advancing? -> +1
+              if(t_off == t_start_off)
+              {
+                t_off += 1;
+              }
+              
+              // rjf: depth == 0? -> done
+              if(depth == 0)
+              {
+                break;
+              }
+            }
+            
+            // rjf: is this the starter task? -> advance base reading offset
+            if(t == &start_task)
+            {
+              off = t_off;
+            }
+          }
+        }
+        
+        //- rjf: if offset not advanced -> increment
         if(off == start_off)
         {
           off += 1;
         }
         
-        // rjf: if type tag -> read full tree, hash, gather
+        //- rjf: atomically gather this hash if not already gathered
-        switch(tag.kind)
+        if(is_type_tag_tree)
-        {
-          default:{}break;
-          case DW_TagKind_ArrayType:
-          case DW_TagKind_ClassType:
-          case DW_TagKind_EnumerationType:
-          case DW_TagKind_PointerType:
-          case DW_TagKind_ReferenceType:
-          case DW_TagKind_StringType:
-          case DW_TagKind_StructureType:
-          case DW_TagKind_SubroutineType:
-          case DW_TagKind_Typedef:
-          case DW_TagKind_UnionType:
-          case DW_TagKind_PtrToMemberType:
-          case DW_TagKind_SetType:
-          case DW_TagKind_SubrangeType:
-          case DW_TagKind_BaseType:
-          case DW_TagKind_ConstType:
-          case DW_TagKind_FileType:
-          case DW_TagKind_PackedType:
-          case DW_TagKind_VolatileType:
-          case DW_TagKind_RestrictType:
-          case DW_TagKind_InterfaceType:
-          case DW_TagKind_UnspecifiedType:
-          case DW_TagKind_SharedType:
-          case DW_TagKind_RValueReferenceType:
-          case DW_TagKind_CoarrayType:
-          case DW_TagKind_DynamicType:
-          case DW_TagKind_AtomicType:
-          case DW_TagKind_ImmutableType:
-          {
-            // rjf: form hash from top-level tag
-            U64 hash = d2r2_hash_from_seed_tag(5381, &tag);
-            
-            // rjf: if this tag has children -> hash all children, combine into `hash`
-            if(tag.has_children)
-            {
-              U64 depth = 1;
-              for(;off < unit_info_range.max && depth > 0;)
-              {
-                Temp temp = temp_begin(scratch2.arena);
-                U64 start_off_2 = off;
-                
-                // rjf: read descendant tag
-                DW2_Tag descendant_tag = {0};
-                off += dw2_read_tag(temp.arena, unit_parse_ctx, raw->sec[DW_Section_Info].data, off, &descendant_tag);
-                
-                // rjf: if offset not advanced -> increment
-                if(off == start_off_2)
-                {
-                  off += 1;
-                }
-                
-                // rjf: combine hash of descendant tag to main hash
-                hash = d2r2_hash_from_seed_tag(hash, &descendant_tag);
-                
-                // rjf: navigate the tree
-                if(descendant_tag.kind == DW_TagKind_Null)
-                {
-                  depth -= 1;
-                }
-                else if(descendant_tag.has_children)
-                {
-                  depth += 1;
-                }
-                
-                temp_end(temp);
-              }
-            }
-            
-            // rjf: atomically gather this hash if not already gathered
         {
           B32 gathered = 0;
           U64 slot_idx = hash%unique_type_tag_slots_count;
@@ -1069,10 +1247,10 @@ d2r2_convert(Arena *arena, D2R2_ConvertParams *params)
               UniqueTypeTagNode *n = push_array(scratch.arena, UniqueTypeTagNode, 1);
               n->next = (UniqueTypeTagNode *)slot_head_val;
               n->hash = hash;
-                  n->unit_idx = unit_idx;
+              n->unit_idx = origin_unit_idx;
               n->info_off = start_off;
               U64 new_head_val = (U64)n;
-                  if(slot_head_val == ins_atomic_u64_eval_cond_assign(&unique_type_tag_slots[slot_idx], slot_head_val, new_head_val))
+              if(slot_head_val == ins_atomic_u64_eval_cond_assign(&unique_type_tag_slots[slot_idx], new_head_val, slot_head_val))
               {
                 gathered = 1;
               }
@@ -1084,18 +1262,17 @@ d2r2_convert(Arena *arena, D2R2_ConvertParams *params)
           }
         }
         
-            // rjf: record this (info_off -> hash) mapping, so that when we have
+        //- rjf: record this (info_off -> hash) mapping, so that when we have
         // later references to this type, we can redirect to the deduplicated
         // type with the right hash later.
+        if(is_type_tag_tree)
         {
           U64 info_off_hash = u64_hash_from_str8(str8_struct(&start_off));
-              U64 info_off_slot_idx = info_off_hash%unit_type_maps[unit_idx].slots_count;
+          U64 info_off_slot_idx = info_off_hash%unit_type_maps[origin_unit_idx].slots_count;
           UnitTypeNode *n = push_array(scratch.arena, UnitTypeNode, 1);
           n->src_info_off = start_off;
           n->dst_hash = hash;
-              SLLStackPush(unit_type_maps[unit_idx].slots[info_off_slot_idx], n);
+          SLLStackPush(unit_type_maps[origin_unit_idx].slots[info_off_slot_idx], n);
-            }
-          }break;
         }
         
         scratch_end(scratch2);
@@ -1104,6 +1281,93 @@ d2r2_convert(Arena *arena, D2R2_ConvertParams *params)
     lane_sync();
   }
   
+  ////////////////////////////
+  //- rjf: produce [0...n) -> unique-tag-node mapping for all types
+  //
+  U64 type_count = 0;
+  UniqueTypeTagNode **type_tag_nodes = 0;
+  ProfScope("produce [0...n) -> hash mapping for all types") if(lane_idx() == 0)
+  {
+    for EachIndex(slot_idx, unique_type_tag_slots_count)
+    {
+      for EachNode(n, UniqueTypeTagNode, unique_type_tag_slots[slot_idx])
+      {
+        type_count += 1;
+      }
+    }
+    type_tag_nodes = push_array(scratch.arena, UniqueTypeTagNode *, type_count);
+    U64 idx = 0;
+    for EachIndex(slot_idx, unique_type_tag_slots_count)
+    {
+      for EachNode(n, UniqueTypeTagNode, unique_type_tag_slots[slot_idx])
+      {
+        type_tag_nodes[idx] = n;
+        idx += 1;
+      }
+    }
+  }
+  lane_sync_u64(&type_count, 0);
+  lane_sync_u64(&type_tag_nodes, 0);
+  
+  ////////////////////////////
+  //- rjf: gather per-type dependency chains
+  //
+  typedef struct TypeDepChain TypeDepChain;
+  struct TypeDepChain
+  {
+    TypeDepChain *next;
+    U64 hash;
+  };
+  TypeDepChain **type_dep_chains = 0;
+  ProfScope("gather per-type dependency chains")
+  {
+    if(lane_idx() == 0)
+    {
+      type_dep_chains = push_array(scratch.arena, TypeDepChain *, type_count);
+    }
+    lane_sync_u64(&type_dep_chains, 0);
+    Rng1U64 range = lane_range(type_count);
+    for EachInRange(type_idx, range)
+    {
+      Temp scratch2 = scratch_begin(&scratch.arena, 1);
+      
+      // rjf: unpack type's tag node
+      UniqueTypeTagNode *type_tag_node = type_tag_nodes[type_idx];
+      U64 info_off = type_tag_node->info_off;
+      U64 unit_idx = type_tag_node->unit_idx;
+      
+      // rjf: unpack unit
+      DW2_ParseCtx *unit_parse_ctx = &unit_parse_ctxs[unit_idx];
+      Rng1U64 unit_info_range = unit_info_ranges->v[unit_idx];
+      
+      // rjf: parse this type's tag
+      DW2_Tag tag = {0};
+      dw2_read_tag(scratch2.arena, unit_parse_ctx, raw->sec[DW_Section_Info].data, info_off, &tag);
+      
+      // rjf: find direct type, if one exists
+      switch(tag.kind)
+      {
+        default:{}break;
+        case DW_TagKind_PointerType:
+        case DW_TagKind_ReferenceType:
+        case DW_TagKind_RValueReferenceType:
+        case DW_TagKind_RestrictType:
+        case DW_TagKind_VolatileType:
+        case DW_TagKind_ConstType:
+        case DW_TagKind_ArrayType:
+        case DW_TagKind_SubrangeType:
+        case DW_TagKind_Typedef:
+        {
+          DW2_Attrib *direct_type_attrib = dw2_attrib_from_kind(&tag, DW_AttribKind_Type);
+          U64 direct_type_info_off = dw2_reference_info_off_from_form_val(unit_parse_ctx, &direct_type_attrib->val);
+          
+        }break;
+      }
+      
+      scratch_end(scratch2);
+    }
+  }
+  
   ////////////////////////////
   //- rjf: fill result
   //

src/rdi_from_dwarf/rdi_from_dwarf_2.h

@@ -21,11 +21,6 @@ struct D2R2_ConvertParams
   B32 deterministic;
 };
 
-////////////////////////////////
-//~ rjf: Helpers
-
-internal U64 d2r2_hash_from_seed_tag(U64 seed, DW2_Tag *tag);
-
 ////////////////////////////////
 //~ rjf: Main Conversion Entry Point (New)