rdim bake parameter joining, source rdi bake parameters from *all* inputs, and not mutually-exclusive dwarf vs. pdb cases

project.4coder

@@ -46,8 +46,8 @@ load_paths =
 commands =
 {
   //- rjf: [raddbg]
-  .f1 = { .win = "raddbg_stable --ipc kill_all && build raddbg telemetry", .linux = "", .out = "*compilation*", .footer_panel = true, .save_dirty_files = true, .cursor_at_end = false, },
+  // .f1 = { .win = "raddbg_stable --ipc kill_all && build raddbg telemetry", .linux = "", .out = "*compilation*", .footer_panel = true, .save_dirty_files = true, .cursor_at_end = false, },
-  // .f1 = { .win = "raddbg_stable --ipc kill_all && build radbin debug telemetry", .linux = "", .out = "*compilation*", .footer_panel = true, .save_dirty_files = true, .cursor_at_end = false, },
+  .f1 = { .win = "raddbg_stable --ipc kill_all && build radbin debug telemetry", .linux = "", .out = "*compilation*", .footer_panel = true, .save_dirty_files = true, .cursor_at_end = false, },
   
   //- rjf: [raddbg wsl]
   // .f1 = { .win = "wsl ./build.sh raddbg", .linux = "", .out = "*compilation*", .footer_panel = true, .save_dirty_files = true, .cursor_at_end = false, },

src/lib_rdi_make/rdi_make.c

@@ -955,7 +955,7 @@ rdim_bytecode_push_op(RDIM_Arena *arena, RDIM_EvalBytecode *bytecode, RDI_EvalOp
   RDIM_SLLQueuePush(bytecode->first_op, bytecode->last_op, node);
   bytecode->op_count += 1;
   bytecode->encoded_size += 1 + p_size;
-
+  
   return node;
 }
 
@@ -1171,6 +1171,89 @@ rdim_local_push_location_case(RDIM_Arena *arena, RDIM_ScopeChunkList *scopes, RD
   return rdim_push_location_case(arena, scopes, &local->location_cases, location, voff_range);
 }
 
+////////////////////////////////
+//~ rjf: [Building] Bake Parameter Joining
+
+RDI_PROC void
+rdim_bake_params_concat_in_place(RDIM_BakeParams *dst, RDIM_BakeParams *src)
+{
+  // rjf: join top-level info (deduplicate - throw away conflicts)
+  {
+    if(dst->top_level_info.arch == RDI_Arch_NULL)
+    {
+      dst->top_level_info.arch = src->top_level_info.arch;
+    }
+    if(dst->top_level_info.exe_name.size == 0)
+    {
+      dst->top_level_info.exe_name = src->top_level_info.exe_name;
+    }
+    if(dst->top_level_info.exe_hash == 0)
+    {
+      dst->top_level_info.exe_hash = src->top_level_info.exe_hash;
+    }
+    if(dst->top_level_info.voff_max == 0)
+    {
+      dst->top_level_info.voff_max = src->top_level_info.voff_max;
+    }
+    if(dst->top_level_info.producer_name.size == 0)
+    {
+      dst->top_level_info.producer_name = src->top_level_info.producer_name;
+    }
+  }
+  
+  // rjf: join binary sections (deduplicate)
+  {
+    RDIM_Temp scratch = rdim_scratch_begin(0, 0);
+    RDI_U64 slots_count = 256;
+    RDIM_BinarySectionNode **slots = rdim_push_array(scratch.arena, RDIM_BinarySectionNode *, slots_count);
+    for(RDIM_BinarySectionNode *n = dst->binary_sections.first; n != 0; n = n->next)
+    {
+      RDIM_BinarySectionNode *hash_node = rdim_push_array(scratch.arena, RDIM_BinarySectionNode, 1);
+      RDI_U64 hash = rdi_hash(n->v.name.str, n->v.name.size);
+      RDI_U64 slot_idx = hash%slots_count;
+      RDIM_SLLStackPush(slots[slot_idx], hash_node);
+      hash_node->v = n->v;
+    }
+    for(RDIM_BinarySectionNode *n = src->binary_sections.first, *next = 0; n != 0; n = next)
+    {
+      next = n->next;
+      RDI_U64 hash = rdi_hash(n->v.name.str, n->v.name.size);
+      RDI_U64 slot_idx = hash%slots_count;
+      RDI_S32 is_duplicate = 0;
+      for(RDIM_BinarySectionNode *hash_n = slots[slot_idx]; hash_n != 0; hash_n = hash_n->next)
+      {
+        if(rdim_str8_match(hash_n->v.name, n->v.name, 0))
+        {
+          is_duplicate = 1;
+          break;
+        }
+      }
+      if(!is_duplicate)
+      {
+        RDIM_SLLQueuePush(dst->binary_sections.first, dst->binary_sections.last, n);
+        dst->binary_sections.count += 1;
+      }
+    }
+    rdim_scratch_end(scratch);
+  }
+  
+  // rjf: join non-top-level chunk lists
+  {
+    rdim_unit_chunk_list_concat_in_place(&dst->units, &src->units);
+    rdim_type_chunk_list_concat_in_place(&dst->types, &src->types);
+    rdim_udt_chunk_list_concat_in_place(&dst->udts, &src->udts);
+    rdim_src_file_chunk_list_concat_in_place(&dst->src_files, &src->src_files);
+    rdim_line_table_chunk_list_concat_in_place(&dst->line_tables, &src->line_tables);
+    rdim_location_chunk_list_concat_in_place(&dst->locations, &src->locations);
+    rdim_symbol_chunk_list_concat_in_place(&dst->global_variables, &src->global_variables);
+    rdim_symbol_chunk_list_concat_in_place(&dst->thread_variables, &src->thread_variables);
+    rdim_symbol_chunk_list_concat_in_place(&dst->constants, &src->constants);
+    rdim_symbol_chunk_list_concat_in_place(&dst->procedures, &src->procedures);
+    rdim_scope_chunk_list_concat_in_place(&dst->scopes, &src->scopes);
+    rdim_inline_site_chunk_list_concat_in_place(&dst->inline_sites, &src->inline_sites);
+  }
+}
+
 ////////////////////////////////
 //~ rjf: [Baking Helpers] Deduplicated String Baking Map
 

src/lib_rdi_make/rdi_make.h

@@ -1671,6 +1671,11 @@ RDI_PROC void rdim_scope_push_voff_range(RDIM_Arena *arena, RDIM_ScopeChunkList
 RDI_PROC RDIM_Local *rdim_scope_push_local(RDIM_Arena *arena, RDIM_ScopeChunkList *scopes, RDIM_Scope *scope);
 RDI_PROC RDIM_LocationCase *rdim_local_push_location_case(RDIM_Arena *arena, RDIM_ScopeChunkList *scopes, RDIM_Local *local, RDIM_Location *location, RDIM_Rng1U64 voff_range);
 
+////////////////////////////////
+//~ rjf: [Building] Bake Parameter Joining
+
+RDI_PROC void rdim_bake_params_concat_in_place(RDIM_BakeParams *dst, RDIM_BakeParams *src);
+
 ////////////////////////////////
 //~ rjf: [Baking Helpers] Deduplicated String Baking Map
 

src/radbin/radbin.c

@@ -604,17 +604,17 @@ rb_thread_entry_point(void *p)
       
       //- rjf: convert inputs to RDI info
       B32 convert_done = 0;
-      RDIM_BakeParams bake_params = {0};
+      RDIM_BakeParams pdb_bake_params = {0};
+      RDIM_BakeParams dwarf_bake_params = {0};
       {
         //- rjf: PE inputs w/ DWARF, or ELF inputs => DWARF -> RDI conversion
-        if(!convert_done &&
+        if(((input_files_from_format_table[RB_FileFormat_PE].count != 0 &&
-           ((input_files_from_format_table[RB_FileFormat_PE].count != 0 &&
              input_files_from_format_table[RB_FileFormat_PE].first->v->format_flags & RB_FileFormatFlag_HasDWARF) ||
             (input_files_from_format_table[RB_FileFormat_ELF32].count != 0 ||
              input_files_from_format_table[RB_FileFormat_ELF64].count != 0)))
         {
           convert_done = 1;
-          log_infof("PEs w/ DWARF, or ELFs specified; producing RDI by converting DWARF data\n");
+          log_infof("PEs w/ DWARF, or ELFs specified; converting DWARF data to RDI\n");
           
           // rjf: convert
           D2R_ConvertParams convert_params = {0};
@@ -695,21 +695,14 @@ rb_thread_entry_point(void *p)
             convert_params.subset_flags   = subset_flags;
             convert_params.deterministic  = cmd_line_has_flag(cmdline, str8_lit("deterministic"));
           }
-          ProfScope("convert") bake_params = d2r_convert(arena, &convert_params);
+          ProfScope("convert") dwarf_bake_params = d2r_convert(arena, &convert_params);
-          
-          // rjf: no output path? -> pick one based on debug
-          if(output_path.size == 0)
-          {
-            output_path = push_str8f(arena, "%S.rdi", str8_chop_last_dot(convert_params.dbg_name));
-          }
         }
         
         //- rjf: PDB inputs => PDB -> RDI conversion
-        if(!convert_done &&
+        if(input_files_from_format_table[RB_FileFormat_PDB].count != 0)
-           input_files_from_format_table[RB_FileFormat_PDB].count != 0)
         {
           convert_done = 1;
-          log_infof("PDBs specified; producing RDI by converting PDB data\n");
+          log_infof("PDBs specified; converting PDB data to RDI\n");
           
           // rjf: get EXE/PDB file data
           RB_File *exe_file = rb_file_list_first(&input_files_from_format_table[RB_FileFormat_PE]);
@@ -729,21 +722,40 @@ rb_thread_entry_point(void *p)
             convert_params.subset_flags   = subset_flags;
             convert_params.deterministic  = cmd_line_has_flag(cmdline, str8_lit("deterministic"));
           }
-          ProfScope("convert") bake_params = p2r_convert(arena, &convert_params);
+          ProfScope("convert") pdb_bake_params = p2r_convert(arena, &convert_params);
-          
+        }
-          // rjf: no output path? -> pick one based on PDB
+      }
-          if(output_path.size == 0) switch(output_kind)
+      lane_sync();
+      
+      //- rjf: join conversion artifacts
+      RDIM_BakeParams *bake_params = 0;
+      if(lane_idx() == 0)
+      {
+        bake_params = push_array(arena, RDIM_BakeParams, 1);
+        rdim_bake_params_concat_in_place(bake_params, &pdb_bake_params);
+        rdim_bake_params_concat_in_place(bake_params, &dwarf_bake_params);
+      }
+      lane_sync_u64(&bake_params, 0);
+      
+      //- rjf: no output path? -> pick one based on input files
+      if(output_path.size == 0)
+      {
+        String8 output_path__noext = {0};
+        if(output_path__noext.size == 0) { output_path__noext = str8_chop_last_dot(rb_file_list_first(&input_files_from_format_table[RB_FileFormat_PDB])->path); }
+        if(output_path__noext.size == 0) { output_path__noext = str8_chop_last_dot(rb_file_list_first(&input_files_from_format_table[RB_FileFormat_PE])->path); }
+        if(output_path__noext.size == 0) { output_path__noext = str8_chop_last_dot(rb_file_list_first(&input_files_from_format_table[RB_FileFormat_ELF64])->path); }
+        if(output_path__noext.size == 0) { output_path__noext = str8_chop_last_dot(rb_file_list_first(&input_files_from_format_table[RB_FileFormat_ELF32])->path); }
+        switch(output_kind)
+        {
+          default:{}break;
+          case OutputKind_RDI:
           {
-            default:{}break;
+            output_path = push_str8f(arena, "%S.rdi", output_path__noext);
-            case OutputKind_RDI:
+          }break;
-            {
+          case OutputKind_Breakpad:
-              output_path = push_str8f(arena, "%S.rdi", str8_chop_last_dot(convert_params.input_pdb_name));
+          {
-            }break;
+            output_path = push_str8f(arena, "%S.psym", output_path__noext);
-            case OutputKind_Breakpad:
+          }break;
-            {
-              output_path = push_str8f(arena, "%S.psym", str8_chop_last_dot(convert_params.input_pdb_name));
-            }break;
-          }
         }
       }
       
@@ -757,7 +769,7 @@ rb_thread_entry_point(void *p)
       RDIM_BakeResults bake_results = {0};
       if(convert_done) ProfScope("bake")
       {
-        bake_results = rdim_bake(arena, &bake_params);
+        bake_results = rdim_bake(arena, bake_params);
       }
       
       //- rjf: convert done => generate output
@@ -799,8 +811,8 @@ rb_thread_entry_point(void *p)
           if(lane_idx() == 0)
           {
             p2b_shared = push_array(arena, P2B_Shared, 1);
-            p2b_shared->lane_chunk_file_dumps = push_array(arena, String8List, lane_count()*bake_params.src_files.chunk_count);
+            p2b_shared->lane_chunk_file_dumps = push_array(arena, String8List, lane_count()*bake_params->src_files.chunk_count);
-            p2b_shared->lane_chunk_func_dumps = push_array(arena, String8List, lane_count()*bake_params.procedures.chunk_count);
+            p2b_shared->lane_chunk_func_dumps = push_array(arena, String8List, lane_count()*bake_params->procedures.chunk_count);
           }
           lane_sync();
           
@@ -808,7 +820,7 @@ rb_thread_entry_point(void *p)
           if(lane_idx() == 0)
           {
             // rjf: pick name to identify module
-            String8 module_name_string = bake_params.top_level_info.exe_name;
+            String8 module_name_string = bake_params->top_level_info.exe_name;
             if(module_name_string.size == 0 && input_files.first != 0)
             {
               module_name_string = input_files.first->v->path;
@@ -816,9 +828,9 @@ rb_thread_entry_point(void *p)
             
             // rjf: pick string for unique code
             String8 unique_identifier_string = {0};
-            if(unique_identifier_string.size == 0 && bake_params.top_level_info.exe_hash != 0)
+            if(unique_identifier_string.size == 0 && bake_params->top_level_info.exe_hash != 0)
             {
-              unique_identifier_string = str8f(arena, "%I64x", bake_params.top_level_info.exe_hash);
+              unique_identifier_string = str8f(arena, "%I64x", bake_params->top_level_info.exe_hash);
             }
             if(unique_identifier_string.size == 0 && input_files.first != 0 && input_files.first->v->format == RB_FileFormat_PDB)
             {
@@ -856,14 +868,14 @@ rb_thread_entry_point(void *p)
           ProfScope("dump FILE records")
           {
             U64 chunk_idx = 0;
-            for EachNode(n, RDIM_SrcFileChunkNode, bake_params.src_files.first)
+            for EachNode(n, RDIM_SrcFileChunkNode, bake_params->src_files.first)
             {
               Rng1U64 range = lane_range(n->count);
               for EachInRange(idx, range)
               {
                 U64 file_idx = rdim_idx_from_src_file(&n->v[idx]);
                 String8 src_path = n->v[idx].path;
-                str8_list_pushf(arena, &p2b_shared->lane_chunk_file_dumps[lane_idx()*bake_params.src_files.chunk_count + chunk_idx], "FILE %I64u %S\n", file_idx, src_path);
+                str8_list_pushf(arena, &p2b_shared->lane_chunk_file_dumps[lane_idx()*bake_params->src_files.chunk_count + chunk_idx], "FILE %I64u %S\n", file_idx, src_path);
               }
               chunk_idx += 1;
             }
@@ -873,9 +885,9 @@ rb_thread_entry_point(void *p)
           ProfScope("dump FUNC records")
           {
             U64 chunk_idx = 0;
-            for EachNode(n, RDIM_SymbolChunkNode, bake_params.procedures.first)
+            for EachNode(n, RDIM_SymbolChunkNode, bake_params->procedures.first)
             {
-              String8List *out = &p2b_shared->lane_chunk_func_dumps[lane_idx()*bake_params.procedures.chunk_count + chunk_idx];
+              String8List *out = &p2b_shared->lane_chunk_func_dumps[lane_idx()*bake_params->procedures.chunk_count + chunk_idx];
               Rng1U64 range = lane_range(n->count);
               for EachInRange(idx, range)
               {
@@ -942,18 +954,18 @@ rb_thread_entry_point(void *p)
           lane_sync();
           if(lane_idx() == 0)
           {
-            for EachIndex(chunk_idx, bake_params.src_files.chunk_count)
+            for EachIndex(chunk_idx, bake_params->src_files.chunk_count)
             {
               for EachIndex(ln_idx, lane_count())
               {
-                str8_list_concat_in_place(&p2b_shared->dump, &p2b_shared->lane_chunk_file_dumps[ln_idx*bake_params.src_files.chunk_count + chunk_idx]);
+                str8_list_concat_in_place(&p2b_shared->dump, &p2b_shared->lane_chunk_file_dumps[ln_idx*bake_params->src_files.chunk_count + chunk_idx]);
               }
             }
-            for EachIndex(chunk_idx, bake_params.procedures.chunk_count)
+            for EachIndex(chunk_idx, bake_params->procedures.chunk_count)
             {
               for EachIndex(ln_idx, lane_count())
               {
-                str8_list_concat_in_place(&p2b_shared->dump, &p2b_shared->lane_chunk_func_dumps[ln_idx*bake_params.procedures.chunk_count + chunk_idx]);
+                str8_list_concat_in_place(&p2b_shared->dump, &p2b_shared->lane_chunk_func_dumps[ln_idx*bake_params->procedures.chunk_count + chunk_idx]);
               }
             }
           }

src/rdi_from_dwarf/rdi_from_dwarf.c

@@ -1697,7 +1697,10 @@ d2r_tag_iterator_init(Arena *arena, DW_TagNode *root)
   iter->free_list            = 0;
   iter->stack                = push_array(arena, D2R_TagFrame, 1);
   iter->stack->node          = push_array(arena, DW_TagNode, 1);
-  *iter->stack->node         = *root;
+  if(root != 0)
+  {
+    *iter->stack->node         = *root;
+  }
   iter->stack->node->sibling = 0;
   iter->visit_children       = 1;
   iter->tag_node             = root;
@@ -1797,7 +1800,10 @@ d2r_find_or_convert_type(Arena *arena, D2R_TypeTable *type_table, DW_Input *inpu
         
         // if we do not have a converted type at this point then debug info is malformed
         type = d2r_type_from_offset(type_table, ref.info_off);
-        Assert(type);
+        if(type == 0)
+        {
+          type = type_table->builtin_types[RDI_TypeKind_NULL];
+        }
       }
     } else {
       Assert(!"unexpected attrib class");
@@ -2054,7 +2060,8 @@ d2r_convert_types(Arena         *arena,
         Assert(!dw_tag_has_attrib(input, cu, tag, DW_AttribKind_Allocated));
         Assert(!dw_tag_has_attrib(input, cu, tag, DW_AttribKind_Associated));
         Assert(!dw_tag_has_attrib(input, cu, tag, DW_AttribKind_Alignment));
-        Assert(!dw_tag_has_attrib(input, cu, tag, DW_AttribKind_Name));
+        // TODO(rjf): this is not an invalid case; it shows up in `mule_main` pointer types
+        // Assert(!dw_tag_has_attrib(input, cu, tag, DW_AttribKind_Name));
         Assert(!dw_tag_has_attrib(input, cu, tag, DW_AttribKind_AddressClass));
         
         U64 byte_size = arch_addr_size;
@@ -2192,11 +2199,14 @@ d2r_convert_types(Arena         *arena,
         }
         
         RDIM_Type      *parent = d2r_type_from_offset(type_table, parent_tag.info_off);
-        RDIM_Type      *type   = d2r_find_or_convert_type(arena, type_table, input, cu, cu_lang, arch_addr_size, tag, DW_AttribKind_Type);
+        if(parent->udt != 0)
-        RDIM_UDTMember *member = rdim_udt_push_member(arena, &udts, parent->udt);
+        {
-        member->kind           = RDI_MemberKind_Base;
+          RDIM_Type      *type   = d2r_find_or_convert_type(arena, type_table, input, cu, cu_lang, arch_addr_size, tag, DW_AttribKind_Type);
-        member->type           = type;
+          RDIM_UDTMember *member = rdim_udt_push_member(arena, &udts, parent->udt);
-        member->off            = safe_cast_u32(dw_const_u32_from_tag_attrib_kind(input, cu, tag, DW_AttribKind_DataMemberLocation));
+          member->kind           = RDI_MemberKind_Base;
+          member->type           = type;
+          member->off            = safe_cast_u32(dw_const_u32_from_tag_attrib_kind(input, cu, tag, DW_AttribKind_DataMemberLocation));
+        }
       } break;
     }
   }
@@ -2550,7 +2560,10 @@ d2r_convert_symbols(Arena         *arena,
       case DW_TagKind_ReferenceType: {
         // TODO:
       } break;
-      default: NotImplemented; break;
+      default:
+      {
+        // NotImplemented;
+      }break;
     }
   }
   scratch_end(scratch);