pass over PDB converter

- moved incomplete type resolution from the converter to RDI make helper
  layer so we don't need to duplicate code in DWARF converter
- changed code for building basic types to handle various data models

src/lib_rdi_make/rdi_make.c

@@ -438,6 +438,144 @@ rdim_rng1u64_list_push(RDIM_Arena *arena, RDIM_Rng1U64List *list, RDIM_Rng1U64 r
   }
 }
 
+////////////////////////////////
+//~ Data Model
+
+RDI_PROC RDI_TypeKind
+rdim_short_type_from_data_model(RDIM_DataModel data_model)
+{
+  switch (data_model) {
+  case RDIM_DataModel_Null  : break;
+  case RDIM_DataModel_ILP32 : return RDI_TypeKind_S16;
+  case RDIM_DataModel_LLP64 : return RDI_TypeKind_S16;
+  case RDIM_DataModel_LP64  : return RDI_TypeKind_S16;
+  case RDIM_DataModel_ILP64 : return RDI_TypeKind_S16;
+  case RDIM_DataModel_SILP64: return RDI_TypeKind_S64;
+  default: InvalidPath;
+  }
+  return RDI_TypeKind_NULL;
+}
+
+RDI_PROC RDI_TypeKind
+rdim_unsigned_short_type_from_data_model(RDIM_DataModel data_model)
+{
+  switch (data_model) {
+  case RDIM_DataModel_Null  : break;
+  case RDIM_DataModel_ILP32 : return RDI_TypeKind_U16;
+  case RDIM_DataModel_LLP64 : return RDI_TypeKind_U16;
+  case RDIM_DataModel_LP64  : return RDI_TypeKind_U16;
+  case RDIM_DataModel_ILP64 : return RDI_TypeKind_U16;
+  case RDIM_DataModel_SILP64: return RDI_TypeKind_U64;
+  default: InvalidPath;
+  }
+  return RDI_TypeKind_NULL;
+}
+
+RDI_PROC RDI_TypeKind
+rdim_int_type_from_data_model(RDIM_DataModel data_model)
+{
+  switch (data_model) {
+  case RDIM_DataModel_Null  : break;
+  case RDIM_DataModel_ILP32 : return RDI_TypeKind_S32;
+  case RDIM_DataModel_LLP64 : return RDI_TypeKind_S32;
+  case RDIM_DataModel_LP64  : return RDI_TypeKind_S32;
+  case RDIM_DataModel_ILP64 : return RDI_TypeKind_S64;
+  case RDIM_DataModel_SILP64: return RDI_TypeKind_S64;
+  default: InvalidPath;
+  }
+  return RDI_TypeKind_NULL;
+}
+
+RDI_PROC RDI_TypeKind
+rdim_unsigned_int_type_from_data_model(RDIM_DataModel data_model)
+{
+  switch (data_model) {
+  case RDIM_DataModel_Null  : break;
+  case RDIM_DataModel_ILP32 : return RDI_TypeKind_U32;
+  case RDIM_DataModel_LLP64 : return RDI_TypeKind_U32;
+  case RDIM_DataModel_LP64  : return RDI_TypeKind_U32;
+  case RDIM_DataModel_ILP64 : return RDI_TypeKind_U64;
+  case RDIM_DataModel_SILP64: return RDI_TypeKind_U64;
+  default: InvalidPath;
+  }
+  return RDI_TypeKind_NULL;
+}
+
+RDI_PROC RDI_TypeKind
+rdim_long_type_from_data_model(RDIM_DataModel data_model)
+{
+  switch (data_model) {
+  case RDIM_DataModel_Null  : break;
+  case RDIM_DataModel_ILP32 : return RDI_TypeKind_S32;
+  case RDIM_DataModel_LLP64 : return RDI_TypeKind_S32;
+  case RDIM_DataModel_LP64  : return RDI_TypeKind_S64;
+  case RDIM_DataModel_ILP64 : return RDI_TypeKind_S64;
+  case RDIM_DataModel_SILP64: return RDI_TypeKind_S64;
+  default: InvalidPath;
+  }
+  return RDI_TypeKind_NULL;
+}
+
+RDI_PROC RDI_TypeKind
+rdim_unsigned_long_type_from_data_model(RDIM_DataModel data_model)
+{
+  switch (data_model) {
+  case RDIM_DataModel_Null  : break;
+  case RDIM_DataModel_ILP32 : return RDI_TypeKind_U32;
+  case RDIM_DataModel_LLP64 : return RDI_TypeKind_U32;
+  case RDIM_DataModel_LP64  : return RDI_TypeKind_U64;
+  case RDIM_DataModel_ILP64 : return RDI_TypeKind_U64;
+  case RDIM_DataModel_SILP64: return RDI_TypeKind_U64;
+  default: InvalidPath;
+  }
+  return RDI_TypeKind_NULL;
+}
+
+RDI_PROC RDI_TypeKind
+rdim_long_long_type_from_data_model(RDIM_DataModel data_model)
+{
+  switch (data_model) {
+  case RDIM_DataModel_Null  : break;
+  case RDIM_DataModel_ILP32 : return RDI_TypeKind_S64;
+  case RDIM_DataModel_LLP64 : return RDI_TypeKind_S64;
+  case RDIM_DataModel_LP64  : return RDI_TypeKind_S64;
+  case RDIM_DataModel_ILP64 : return RDI_TypeKind_S64;
+  case RDIM_DataModel_SILP64: return RDI_TypeKind_S64;
+  default: InvalidPath;
+  }
+  return RDI_TypeKind_NULL;
+}
+
+RDI_PROC RDI_TypeKind
+rdim_unsigned_long_long_type_from_data_model(RDIM_DataModel data_model)
+{
+  switch (data_model) {
+  case RDIM_DataModel_Null  : break;
+  case RDIM_DataModel_ILP32 : return RDI_TypeKind_U64;
+  case RDIM_DataModel_LLP64 : return RDI_TypeKind_U64;
+  case RDIM_DataModel_LP64  : return RDI_TypeKind_U64;
+  case RDIM_DataModel_ILP64 : return RDI_TypeKind_U64;
+  case RDIM_DataModel_SILP64: return RDI_TypeKind_U64;
+  default: InvalidPath;
+  }
+  return RDI_TypeKind_NULL;
+}
+
+RDI_PROC RDI_TypeKind
+rdim_pointer_size_t_type_from_data_model(RDIM_DataModel data_model)
+{
+  switch (data_model) {
+  case RDIM_DataModel_Null  : break;
+  case RDIM_DataModel_ILP32 : return RDI_TypeKind_U32;
+  case RDIM_DataModel_LLP64 : return RDI_TypeKind_U64;
+  case RDIM_DataModel_LP64  : return RDI_TypeKind_U64;
+  case RDIM_DataModel_ILP64 : return RDI_TypeKind_U64;
+  case RDIM_DataModel_SILP64: return RDI_TypeKind_U64;
+  default: InvalidPath;
+  }
+  return RDI_TypeKind_NULL;
+}
+
 ////////////////////////////////
 //~ rjf: [Building] Binary Section List Building
 
@@ -1163,6 +1301,13 @@ RDI_PROC void
 rdim_assign_type_index(RDIM_Type *type, U64 *type_indices, U64 *curr_type_idx)
 {
   RDI_U64 type_pos = rdim_idx_from_type(type);
+
+  if(type->kind == RDI_TypeKind_NULL)
+  {
+    type_indices[type_pos] = 0;
+    return;
+  }
+
   if(type_indices[type_pos] == 0)
   {
     if(type->param_types)
@@ -1186,6 +1331,8 @@ rdim_assign_type_index(RDIM_Type *type, U64 *type_indices, U64 *curr_type_idx)
 RDI_PROC RDI_U64 *
 rdim_make_type_indices(RDIM_Arena *arena, RDIM_TypeChunkList *types)
 {
+  ProfBeginFunction();
+
   RDI_U64 *type_indices       = rdim_push_array(arena, RDI_U64, types->total_count + 1);
   RDI_U64  type_indices_count = 1;
 
@@ -1197,6 +1344,7 @@ rdim_make_type_indices(RDIM_Arena *arena, RDIM_TypeChunkList *types)
     }
   }
 
+  ProfEnd();
   return type_indices;
 }
 
@@ -3038,6 +3186,11 @@ rdim_bake_types(RDIM_Arena *arena, RDIM_BakeStringMapTight *strings, RDIM_BakeId
       U64           dst_idx = rdim_final_idx_from_type(type_indices, src);
       RDI_TypeNode *dst     = &type_nodes[dst_idx];
 
+      if(src->kind == RDI_TypeKind_NULL)
+      {
+        continue;
+      }
+      
       //- rjf: fill shared type node info
       dst->kind      = src->kind;
       dst->flags     = (RDI_U16)src->flags; // TODO(rjf): @u32_to_u16
@@ -3327,7 +3480,7 @@ rdim_bake_thread_variables(RDIM_Arena *arena, RDIM_BakeStringMapTight *strings,
 }
 
 RDI_PROC U64
-rdim_bake_location(Arena *arena, RDIM_String8List *location_data_blobs, RDIM_Location *src_location)
+rdim_bake_location(RDIM_Arena *arena, RDIM_String8List *location_data_blobs, RDIM_Location *src_location)
 {
   U64 location_data_off = location_data_blobs->total_size;
 

src/lib_rdi_make/rdi_make.h

@@ -595,6 +595,17 @@ struct RDIM_UnitChunkList
 ////////////////////////////////
 //~ rjf: Type System Node Types
 
+typedef RDI_U32 RDIM_DataModel;
+enum RDIM_DataModelEnum
+{
+  RDIM_DataModel_Null,
+  RDIM_DataModel_ILP32,
+  RDIM_DataModel_LLP64,
+  RDIM_DataModel_LP64,
+  RDIM_DataModel_ILP64,
+  RDIM_DataModel_SILP64
+};
+
 typedef struct RDIM_Type RDIM_Type;
 struct RDIM_Type
 {
@@ -605,6 +616,7 @@ struct RDIM_Type
   RDI_U32 off;
   RDI_U32 count;
   RDIM_String8 name;
+  RDIM_String8 link_name;
   RDIM_Type *direct_type;
   RDIM_Type **param_types;
   struct RDIM_UDT *udt;
@@ -1348,6 +1360,19 @@ RDI_PROC RDIM_SortKey *rdim_sort_key_array(RDIM_Arena *arena, RDIM_SortKey *keys
 //- rjf: rng1u64 list
 RDI_PROC void rdim_rng1u64_list_push(RDIM_Arena *arena, RDIM_Rng1U64List *list, RDIM_Rng1U64 r);
 
+////////////////////////////////
+//~ Data Model
+
+RDI_PROC RDI_TypeKind rdim_short_type_from_data_model(RDIM_DataModel data_model);
+RDI_PROC RDI_TypeKind rdim_unsigned_short_type_from_data_model(RDIM_DataModel data_model);
+RDI_PROC RDI_TypeKind rdim_int_type_from_data_model(RDIM_DataModel data_model);
+RDI_PROC RDI_TypeKind rdim_unsigned_int_type_from_data_model(RDIM_DataModel data_model);
+RDI_PROC RDI_TypeKind rdim_long_type_from_data_model(RDIM_DataModel data_model);
+RDI_PROC RDI_TypeKind rdim_unsigned_long_type_from_data_model(RDIM_DataModel data_model);
+RDI_PROC RDI_TypeKind rdim_long_long_type_from_data_model(RDIM_DataModel data_model);
+RDI_PROC RDI_TypeKind rdim_unsigned_long_long_type_from_data_model(RDIM_DataModel data_model);
+RDI_PROC RDI_TypeKind rdim_pointer_size_t_type_from_data_model(RDIM_DataModel data_model);
+
 ////////////////////////////////
 //~ rjf: [Building] Binary Section Info Building
 

src/rdi_from_pdb/rdi_from_pdb.h

@@ -201,37 +201,6 @@ struct P2R_LinkNameMapBuildIn
   P2R_LinkNameMap *link_name_map;
 };
 
-//- rjf: type forward resolution map build
-
-typedef struct P2R_ITypeFwdMapFillIn P2R_ITypeFwdMapFillIn;
-struct P2R_ITypeFwdMapFillIn
-{
-  PDB_TpiHashParsed *tpi_hash;
-  CV_LeafParsed *tpi_leaf;
-  CV_TypeId itype_first;
-  CV_TypeId itype_opl;
-  CV_TypeId *itype_fwd_map;
-};
-
-//- rjf: itype chain build
-
-typedef struct P2R_TypeIdChain P2R_TypeIdChain;
-struct P2R_TypeIdChain
-{
-  P2R_TypeIdChain *next;
-  CV_TypeId itype;
-};
-
-typedef struct P2R_ITypeChainBuildIn P2R_ITypeChainBuildIn;
-struct P2R_ITypeChainBuildIn
-{
-  CV_LeafParsed *tpi_leaf;
-  CV_TypeId itype_first;
-  CV_TypeId itype_opl;
-  CV_TypeId *itype_fwd_map;
-  P2R_TypeIdChain **itype_chains;
-};
-
 //- rjf: udt conversion
 
 typedef struct P2R_UDTConvertIn P2R_UDTConvertIn;
@@ -240,7 +209,6 @@ struct P2R_UDTConvertIn
   CV_LeafParsed *tpi_leaf;
   CV_TypeId itype_first;
   CV_TypeId itype_opl;
-  CV_TypeId *itype_fwd_map;
   RDIM_Type **itype_type_ptrs;
 };
 
@@ -257,7 +225,6 @@ struct P2R_SymbolStreamConvertIn
   CV_SymParsed *sym;
   U64 sym_ranges_first;
   U64 sym_ranges_opl;
-  CV_TypeId *itype_fwd_map;
   RDIM_Type **itype_type_ptrs;
   P2R_LinkNameMap *link_name_map;
   RDIM_LineTable *first_inline_site_line_table;
@@ -341,12 +308,6 @@ ASYNC_WORK_DEF(p2r_units_convert_work);
 
 ASYNC_WORK_DEF(p2r_link_name_map_build_work);
 
-////////////////////////////////
-//~ rjf: Type Parsing/Conversion Tasks
-
-ASYNC_WORK_DEF(p2r_itype_fwd_map_fill_work);
-ASYNC_WORK_DEF(p2r_itype_chain_build_work);
-
 ////////////////////////////////
 //~ rjf: UDT Conversion Tasks
 

src/rdi_from_pdb/rdi_from_pdb_main.c

@@ -15,6 +15,9 @@
 #include "lib_rdi_format/rdi_format.c"
 #include "third_party/rad_lzb_simple/rad_lzb_simple.h"
 #include "third_party/rad_lzb_simple/rad_lzb_simple.c"
+#define XXH_STATIC_LINKING_ONLY
+#include "third_party/xxHash/xxhash.h"
+#include "third_party/xxHash/xxhash.c"
 
 //- rjf: [h]
 #include "base/base_inc.h"

src/rdi_make/rdi_make_help.c

@@ -1,6 +1,40 @@
 // Copyright (c) 2024 Epic Games Tools
 // Licensed under the MIT license (https://opensource.org/license/mit/)
 
+////////////////////////////////
+
+internal RDIM_DataModel
+rdim_infer_data_model(OperatingSystem os, RDI_Arch arch)
+{
+  RDIM_DataModel data_model = RDIM_DataModel_Null;
+  switch (os) {
+  case OperatingSystem_Null: break;
+  case OperatingSystem_Windows: {
+    switch (arch) {
+    case RDI_Arch_X86:
+    case RDI_Arch_X64:
+      data_model = RDIM_DataModel_LLP64; break;
+    default: NotImplemented;
+    }
+  } break;
+  case OperatingSystem_Linux: {
+    switch (arch) {
+    case RDI_Arch_X86: data_model = RDIM_DataModel_ILP32; break;
+    case RDI_Arch_X64: data_model = RDIM_DataModel_LLP64; break;
+    default: NotImplemented;
+    }
+  } break;
+  case OperatingSystem_Mac: {
+    switch (arch) {
+    case RDI_Arch_X86: NotImplemented; break;
+    case RDI_Arch_X64: data_model = RDIM_DataModel_LP64; break;
+    }
+  } break;
+  default: InvalidPath;
+  }
+  return data_model;
+}
+
 ////////////////////////////////
 //~ rjf: Baking Stage Tasks
 
@@ -382,6 +416,154 @@ ASYNC_WORK_DEF(rdim_bake_idx_runs_work)
   return out;
 }
 
+internal U64
+rdim_help_hash(RDIM_String8 string)
+{
+  return XXH3_64bits(string.str, string.size);
+}
+
+internal void
+rdim_help_resolve_incomplete_types(RDIM_TypeChunkList *types)
+{
+  ProfBeginFunction();
+
+  Temp scratch = scratch_begin(0,0);
+
+  ProfBegin("Build Hash Table");
+  RDIM_Type **name_ht = rdim_push_array(scratch.arena, RDIM_Type *, types->total_count);
+  for(RDIM_TypeChunkNode *chunk = types->first; chunk != 0; chunk = chunk->next)
+  {
+    for(RDI_U64 i = 0; i < chunk->count; i += 1)
+    {
+      RDIM_Type *type = &chunk->v[i];
+      if(RDI_TypeKind_FirstUserDefined <= type->kind && type->kind <= RDI_TypeKind_LastRecord)
+      {
+        RDIM_String8 name = type->link_name.size ? type->link_name : type->name;
+        RDI_U64      hash = rdim_help_hash(name);
+
+        RDI_U64 best_slot = hash % types->total_count;
+        RDI_U64 slot      = best_slot;
+        do
+        {
+          RDIM_Type *s = name_ht[slot];
+          if(s == 0)
+          {
+            break;
+          }
+
+          if(s->link_name.size)
+          {
+            if(str8_match(s->link_name, name, 0))
+            {
+              break;
+            }
+          }
+          else if(s->name.size)
+          {
+            if(str8_match(s->name, type->name, 0))
+            {
+              break;
+            }
+          }
+
+          slot = (slot + 1) % types->total_count;
+        } while (slot != best_slot);
+
+        if(name_ht[slot] == 0)
+        {
+          name_ht[slot] = type;
+        }
+      }
+    }
+  }
+  ProfEnd();
+
+  ProfBegin("Make Fwd Map");
+  RDIM_Type **fwd_map = rdim_push_array(scratch.arena, RDIM_Type *, types->total_count);
+  for(RDIM_TypeChunkNode *chunk = types->first; chunk != 0; chunk = chunk->next)
+  {
+    for(RDI_U64 i = 0; i < chunk->count; i += 1)
+    {
+      RDIM_Type *type = &chunk->v[i];
+
+      if(RDI_TypeKind_FirstIncomplete <= type->kind && type->kind <= RDI_TypeKind_LastIncomplete)
+      {
+        RDIM_String8 name      = type->link_name.size ? type->link_name : type->name;
+        RDI_U64      hash      = rdim_help_hash(name);
+        RDI_U64      best_slot = hash % types->total_count;
+        RDI_U64      slot      = best_slot;
+
+        RDIM_Type *match = 0;
+        do
+        {
+          if(name_ht[slot] == 0)
+          {
+            break;
+          }
+          RDIM_Type *s = name_ht[slot];
+          if(s->link_name.size)
+          {
+            if(str8_match(s->link_name, type->link_name, 0))
+            {
+              match = s;
+              break;
+            }
+          }
+          else
+          {
+            if(str8_match(s->name, type->name, 0))
+            {
+              match = s;
+              break;
+            }
+          }
+        } while(slot != best_slot);
+
+        if(match)
+        {
+          type->kind = RDI_TypeKind_NULL;
+
+          RDI_U64 type_idx = rdim_idx_from_type(type);
+          fwd_map[type_idx] = match;
+        }
+      }
+    }
+  }
+  ProfEnd();
+
+  ProfBegin("Resolve Types");
+  for(RDIM_TypeChunkNode *chunk = types->first; chunk != 0; chunk = chunk->next)
+  {
+    for(RDI_U64 i = 0; i < chunk->count; ++i)
+    {
+      RDIM_Type *t = &chunk->v[i];
+      if(t->direct_type)
+      {
+        RDI_U64 direct_idx = rdim_idx_from_type(t->direct_type);
+        if(fwd_map[direct_idx])
+        {
+          t->direct_type = fwd_map[direct_idx];
+        }
+      }
+      if(t->param_types)
+      {
+        for(RDI_U64 param_idx = 0; param_idx < t->count; param_idx += 1)
+        {
+          RDI_U64 type_idx = rdim_idx_from_type(t->param_types[param_idx]);
+          if(fwd_map[type_idx])
+          {
+            t->param_types[param_idx] = fwd_map[type_idx];
+          }
+        }
+      }
+    }
+  }
+  ProfEnd();
+
+  scratch_end(scratch);
+  ProfEnd();
+}
+
 internal RDIM_HelpState *
 rdim_help_init(void)
 {
@@ -405,6 +587,11 @@ rdim_bake(RDIM_HelpState *state, RDIM_BakeParams *in_params)
 
   rdim_help_state = state;
 
+  ////////////////////////////////
+  // resolve incomplete types
+
+  rdim_help_resolve_incomplete_types(&in_params->types);
+
   ////////////////////////////////
   // compute type indices
 

src/rdi_make/rdi_make_help.h

@@ -281,6 +281,10 @@ struct RDIM_BakeIdxRunsIn
   RDIM_BakeIdxRunMap *idx_runs;
 };
 
+////////////////////////////////
+
+internal RDIM_DataModel rdim_infer_data_model(OperatingSystem os, RDI_Arch arch);
+
 ////////////////////////////////
 //~ rjf: Baking Stage Tasks