first pass over raddbgi make baking phase rewrite/cleanup pass, getting off of old cons API

src/raddbgi_from_pdb/raddbgi_from_pdb.c

@@ -25,15 +25,16 @@ p2r_end_of_cplusplus_container_name(String8 str)
   }
   return(result);
 }
-////////////////////////////////
-//~ rjf: Command Line -> Conversion Parameters
 
-internal P2R_Params*
-p2r_params_from_cmd_line(Arena *arena, CmdLine *cmdline)
+////////////////////////////////
+//~ rjf: Command Line -> Conversion Inputs
+
+internal P2R_ConvertIn *
+p2r_convert_in_from_cmd_line(Arena *arena, CmdLine *cmdline)
 {
-  P2R_Params *result = push_array(arena, P2R_Params, 1);
+  P2R_ConvertIn *result = push_array(arena, P2R_ConvertIn, 1);
   
-  // get input pdb
+  //- rjf: get input pdb
   {
     String8 input_name = cmd_line_string(cmdline, str8_lit("pdb"));
     if(input_name.size == 0)
@@ -55,7 +56,7 @@ p2r_params_from_cmd_line(Arena *arena, CmdLine *cmdline)
     }
   }
   
-  // get input exe
+  //- rjf: get input exe
   {
     String8 input_name = cmd_line_string(cmdline, str8_lit("exe"));
     if(input_name.size > 0)
@@ -73,12 +74,12 @@ p2r_params_from_cmd_line(Arena *arena, CmdLine *cmdline)
     }
   }
   
-  // get output name
+  //- rjf: get output name
   {
     result->output_name = cmd_line_string(cmdline, str8_lit("out"));
   }
   
-  // error options
+  //- rjf: error options
   if(cmd_line_has_flag(cmdline, str8_lit("hide_errors")))
   {
     String8List vals = cmd_line_strings(cmdline, str8_lit("hide_errors"));
@@ -116,7 +117,7 @@ p2r_params_from_cmd_line(Arena *arena, CmdLine *cmdline)
     }
   }
   
-  // dump options
+  //- rjf: dump options
   if(cmd_line_has_flag(cmdline, str8_lit("dump")))
   {
     result->dump = 1;
@@ -169,7 +170,7 @@ p2r_params_from_cmd_line(Arena *arena, CmdLine *cmdline)
     }
   }
   
-  return(result);
+  return result;
 }
 
 ////////////////////////////////
@@ -3433,20 +3434,18 @@ p2r_link_name_find(P2R_LinkNameMap *map, U64 voff)
 ////////////////////////////////
 //~ rjf: Top-Level Conversion Entry Point
 
-internal P2R_Out *
-p2r_convert(Arena *arena, P2R_Params *params)
+internal P2R_ConvertOut *
+p2r_convert(Arena *arena, P2R_ConvertIn *in)
 {
   Temp scratch = scratch_begin(&arena, 1);
-  P2R_Out *out = push_array(arena, P2R_Out, 1);
-  out->good_parse = 1;
   
   //////////////////////////////////////////////////////////////
   //- rjf: parse MSF structure
   //
   MSF_Parsed *msf = 0;
-  if(params->errors.node_count == 0) ProfScope("parse MSF structure")
+  if(in->input_pdb_data.size != 0) ProfScope("parse MSF structure")
   {
-    msf = msf_parsed_from_data(arena, params->input_pdb_data);
+    msf = msf_parsed_from_data(arena, in->input_pdb_data);
   }
   
   //////////////////////////////////////////////////////////////
@@ -3651,9 +3650,9 @@ p2r_convert(Arena *arena, P2R_Params *params)
   //- rjf: hash exe
   //
   U64 exe_hash = 0;
-  if(params->input_exe_data.size > 0) ProfScope("hash exe")
+  if(in->input_exe_data.size > 0) ProfScope("hash exe")
   {
-    exe_hash = rdi_hash(params->input_exe_data.str, params->input_exe_data.size);
+    exe_hash = rdi_hash(in->input_exe_data.str, in->input_exe_data.size);
   }
   
   //////////////////////////////////////////////////////////////
@@ -3708,7 +3707,7 @@ p2r_convert(Arena *arena, P2R_Params *params)
   RDIM_TopLevelInfo top_level_info = {0};
   {
     top_level_info.arch     = arch;
-    top_level_info.exe_name = params->input_exe_name;
+    top_level_info.exe_name = in->input_exe_name;
     top_level_info.exe_hash = exe_hash;
     top_level_info.voff_max = exe_voff_max;
   }
@@ -3844,11 +3843,11 @@ p2r_convert(Arena *arena, P2R_Params *params)
   //
   U64 type_fwd_map_count = 0;
   CV_TypeId *type_fwd_map = 0;
+  CV_TypeId itype_first = 0;
+  CV_TypeId itype_opl   = tpi_leaf->itype_opl;
   ProfScope("types pass 1: produce type forward resolution map")
   {
-    CV_TypeId itype_first = tpi_leaf->itype_first;
-    CV_TypeId itype_opl   = tpi_leaf->itype_opl;
-    type_fwd_map_count = (U64)(itype_opl-itype_first);
+    type_fwd_map_count = (U64)itype_opl;
     type_fwd_map = push_array(arena, CV_TypeId, type_fwd_map_count);
     for(CV_TypeId itype = itype_first; itype < itype_opl; itype += 1)
     {
@@ -3985,8 +3984,6 @@ p2r_convert(Arena *arena, P2R_Params *params)
   };
   P2R_TypeIdRevisitTask *first_itype_revisit_task = 0;
   P2R_TypeIdRevisitTask *last_itype_revisit_task = 0;
-  CV_TypeId itype_first = tpi_leaf->itype_first;
-  CV_TypeId itype_opl   = tpi_leaf->itype_opl;
   RDIM_TypeArray itype_types = {0};     // root type for per-TPI-itype
   RDIM_TypeChunkList extra_types = {0}; // extra supplementary types we build, which do not have any itypes
   ProfScope("types pass 2: construct all root/stub types from TPI")
@@ -4384,8 +4381,6 @@ p2r_convert(Arena *arena, P2R_Params *params)
   ProfScope("types pass 3: attach cross-itype-relationship data to all types, build UDTs")
   {
     RDI_U64 udts_chunk_cap = 1024;
-    CV_TypeId itype_first = tpi_leaf->itype_first;
-    CV_TypeId itype_opl   = tpi_leaf->itype_opl;
 #define p2r_type_ptr_from_itype(itype) ((itype_first <= (itype) && (itype) < itype_opl) ? (&itype_types.v[(type_fwd_map[(itype)-itype_first] ? type_fwd_map[(itype)-itype_first] : (itype))-itype_first]) : 0)
     for(P2R_TypeIdRevisitTask *task = first_itype_revisit_task; task != 0; task = task->next)
     {
@@ -4483,7 +4478,7 @@ p2r_convert(Arena *arena, P2R_Params *params)
         //- rjf: arrays -> calculate array count based on direct type size
         //
         case RDI_TypeKind_Array:
-        if(dst_type->direct_type != 0)
+        if(dst_type->direct_type != 0 && dst_type->direct_type->byte_size != 0)
         {
           dst_type->count = dst_type->byte_size/dst_type->direct_type->byte_size;
         }break;
@@ -5086,6 +5081,10 @@ p2r_convert(Arena *arena, P2R_Params *params)
   //////////////////////////////////////////////////////////////
   //- rjf: produce symbols from all sym streams
   //
+  RDIM_SymbolChunkList all_procedures = {0};
+  RDIM_SymbolChunkList all_global_variables = {0};
+  RDIM_SymbolChunkList all_thread_variables = {0};
+  RDIM_ScopeChunkList all_scopes = {0};
   ProfScope("produce symbols from all sym streams")
   {
 #define p2r_type_ptr_from_itype(itype) ((itype_first <= (itype) && (itype) < itype_opl) ? (&itype_types.v[(type_fwd_map[(itype)-itype_first] ? type_fwd_map[(itype)-itype_first] : (itype))-itype_first]) : 0)
@@ -5239,8 +5238,11 @@ p2r_convert(Arena *arena, P2R_Params *params)
             case CV_SymKind_END:
             {
               P2R_ScopeNode *n = top_scope_node;
-              SLLStackPop(top_scope_node);
-              SLLStackPush(free_scope_node, n);
+              if(n != 0)
+              {
+                SLLStackPop(top_scope_node);
+                SLLStackPush(free_scope_node, n);
+              }
               defrange_target = 0;
               defrange_target_is_param = 0;
             }break;
@@ -5785,19 +5787,42 @@ p2r_convert(Arena *arena, P2R_Params *params)
         }
       }
       
+      //////////////////////////
+      //- rjf: merge this stream's outputs with collated list
+      //
+      rdim_symbol_chunk_list_concat_in_place(&all_procedures, &sym_procedures);
+      rdim_symbol_chunk_list_concat_in_place(&all_global_variables, &sym_global_variables);
+      rdim_symbol_chunk_list_concat_in_place(&all_thread_variables, &sym_thread_variables);
+      rdim_scope_chunk_list_concat_in_place(&all_scopes, &sym_scopes);
+      
       scratch_end(scratch);
     }
     
 #undef p2r_type_ptr_from_itype
   }
   
+  //////////////////////////////////////////////////////////////
+  //- rjf: fill output
+  //
+  P2R_ConvertOut *out = push_array(arena, P2R_ConvertOut, 1);
+  {
+    out->top_level_info   = top_level_info;
+    out->binary_sections  = binary_sections;
+    rdim_unit_chunk_list_push_array(arena, &out->units, &units);
+    rdim_type_chunk_list_push_array(arena, &out->types, &itype_types);
+    rdim_type_chunk_list_concat_in_place(&out->types, &extra_types);
+    out->global_variables = all_global_variables;
+    out->thread_variables = all_thread_variables;
+    out->procedures       = all_procedures;
+    out->scopes           = all_scopes;
+  }
   
   //~ TODO(rjf): OLD vvvvvvvvvvvvvvvvvvv
 #if 0
   
   // output generation
   P2R_Ctx *p2r_ctx = 0;
-  if(params->output_name.size > 0)
+  if(in->output_name.size > 0)
   {
     // setup root
     RDIM_RootParams root_params = {0};
@@ -5830,7 +5855,7 @@ p2r_convert(Arena *arena, P2R_Params *params)
       // set top level info
       RDIM_TopLevelInfo tli = {0};
       tli.architecture = architecture;
-      tli.exe_name = params->input_exe_name;
+      tli.exe_name = in->input_exe_name;
       tli.exe_hash = exe_hash;
       tli.voff_max = voff_max;
       
@@ -5977,7 +6002,7 @@ p2r_convert(Arena *arena, P2R_Params *params)
     }
     
     // conversion errors
-    if(!params->hide_errors.converting)
+    if(!in->hide_errors.converting)
     {
       for(RDIM_Msg *msg = rdim_first_msg_from_root(root);
           msg != 0;
@@ -5989,7 +6014,7 @@ p2r_convert(Arena *arena, P2R_Params *params)
   }
   
   // dump
-  if(params->dump) ProfScope("dump")
+  if(in->dump) ProfScope("dump")
   {
     String8List dump = {0};
     
@@ -6007,7 +6032,7 @@ p2r_convert(Arena *arena, P2R_Params *params)
     }
     
     // MSF
-    if(params->dump_msf)
+    if(in->dump_msf)
     {
       if(msf != 0)
       {
@@ -6033,7 +6058,7 @@ p2r_convert(Arena *arena, P2R_Params *params)
     }
     
     // DBI
-    if(params->dump_sym)
+    if(in->dump_sym)
     {
       if(sym != 0)
       {
@@ -6047,7 +6072,7 @@ p2r_convert(Arena *arena, P2R_Params *params)
     }
     
     // TPI
-    if(params->dump_tpi_hash)
+    if(in->dump_tpi_hash)
     {
       if(tpi_hash != 0)
       {
@@ -6071,7 +6096,7 @@ p2r_convert(Arena *arena, P2R_Params *params)
     }
     
     // LEAF
-    if(params->dump_leaf)
+    if(in->dump_leaf)
     {
       if(tpi_leaf != 0)
       {
@@ -6095,7 +6120,7 @@ p2r_convert(Arena *arena, P2R_Params *params)
     }
     
     // BINARY SECTIONS
-    if(params->dump_coff_sections)
+    if(in->dump_coff_sections)
     {
       if(coff_sections != 0)
       {
@@ -6130,8 +6155,8 @@ p2r_convert(Arena *arena, P2R_Params *params)
     // UNITS
     if(comp_units != 0)
     {
-      B32 dump_sym = params->dump_sym;
-      B32 dump_c13 = params->dump_c13;
+      B32 dump_sym = in->dump_sym;
+      B32 dump_c13 = in->dump_c13;
       
       B32 dump_units = (dump_sym || dump_c13);
       
@@ -6163,7 +6188,7 @@ p2r_convert(Arena *arena, P2R_Params *params)
     // UNIT CONTRIBUTIONS
     if(comp_unit_contributions != 0)
     {
-      if(params->dump_contributions)
+      if(in->dump_contributions)
       {
         str8_list_push(arena, &dump,
                        str8_lit("################################"
@@ -6181,7 +6206,7 @@ p2r_convert(Arena *arena, P2R_Params *params)
     }
     
     // rjf: dump table diagnostics
-    if(params->dump_table_diagnostics)
+    if(in->dump_table_diagnostics)
     {
       str8_list_push(arena, &dump,
                      str8_lit("################################"

src/raddbgi_from_pdb/raddbgi_from_pdb.h

@@ -5,10 +5,10 @@
 #define RDI_FROM_PDB_H
 
 ////////////////////////////////
-//~ rjf: Conversion Parameters Type
+//~ rjf: Conversion Inputs/Outputs
 
-typedef struct P2R_Params P2R_Params;
-struct P2R_Params
+typedef struct P2R_ConvertIn P2R_ConvertIn;
+struct P2R_ConvertIn
 {
   String8 input_pdb_name;
   String8 input_pdb_data;
@@ -39,6 +39,19 @@ struct P2R_Params
   String8List errors;
 };
 
+typedef struct P2R_ConvertOut P2R_ConvertOut;
+struct P2R_ConvertOut
+{
+  RDIM_TopLevelInfo top_level_info;
+  RDIM_BinarySectionList binary_sections;
+  RDIM_UnitChunkList units;
+  RDIM_TypeChunkList types;
+  RDIM_SymbolChunkList global_variables;
+  RDIM_SymbolChunkList thread_variables;
+  RDIM_SymbolChunkList procedures;
+  RDIM_ScopeChunkList scopes;
+};
+
 ////////////////////////////////
 //~ rjf: Conversion Data Structure Types
 
@@ -217,7 +230,6 @@ struct P2R_Ctx
   P2R_KnownGlobalSet known_globals;
   P2R_LinkNameMap link_names;
 };
-#endif
 
 ////////////////////////////////
 //~ Conversion Output Type
@@ -230,6 +242,7 @@ struct P2R_Out
   String8List dump;
   String8List errors;
 };
+#endif
 
 ////////////////////////////////
 //~ rjf: Basic Helpers
@@ -237,9 +250,9 @@ struct P2R_Out
 internal U64 p2r_end_of_cplusplus_container_name(String8 str);
 
 ////////////////////////////////
-//~ rjf: Command Line -> Conversion Parameters
+//~ rjf: Command Line -> Conversion Inputs
 
-internal P2R_Params *p2r_params_from_cmd_line(Arena *arena, CmdLine *cmdline);
+internal P2R_ConvertIn *p2r_convert_in_from_cmd_line(Arena *arena, CmdLine *cmdline);
 
 ////////////////////////////////
 //~ rjf: COFF => RADDBGI Canonical Conversions
@@ -263,6 +276,11 @@ internal RDI_RegisterCode p2r_reg_code_from_arch_encoded_fp_reg(RDI_Arch arch, C
 internal void p2r_location_over_lvar_addr_range(Arena *arena, RDIM_LocationSet *locset, RDIM_Location *location, CV_LvarAddrRange *range, COFF_SectionHeader *section, CV_LvarAddrGap *gaps, U64 gap_count);
 
 
+////////////////////////////////
+//~ rjf: Top-Level Conversion Entry Point
+
+internal P2R_ConvertOut *p2r_convert(Arena *arena, P2R_ConvertIn *in);
+
 #if 0
 
 ////////////////////////////////
@@ -371,9 +389,4 @@ internal String8 p2r_link_name_find(P2R_LinkNameMap *map, U64 voff);
 
 #endif
 
-////////////////////////////////
-//~ rjf: Top-Level Conversion Entry Point
-
-internal P2R_Out *p2r_convert(Arena *arena, P2R_Params *params);
-
 #endif // RDI_FROM_PDB_H

src/raddbgi_from_pdb/raddbgi_from_pdb_main.c

@@ -1,9 +1,11 @@
 // Copyright (c) 2024 Epic Games Tools
 // Licensed under the MIT license (https://opensource.org/license/mit/)
 
+//- rjf: [lib]
 #include "lib_raddbgi_format/raddbgi_format.h"
 #include "lib_raddbgi_format/raddbgi_format.c"
 
+//- rjf: [h]
 #include "base/base_inc.h"
 #include "os/os_inc.h"
 #include "raddbgi_make_local/raddbgi_make_local.h"
@@ -13,9 +15,9 @@
 #include "msf/msf.h"
 #include "pdb/pdb.h"
 #include "pdb/pdb_stringize.h"
-
 #include "raddbgi_from_pdb.h"
 
+//- rjf: [c]
 #include "base/base_inc.c"
 #include "os/os_inc.c"
 #include "raddbgi_make_local/raddbgi_make_local.c"
@@ -25,11 +27,13 @@
 #include "msf/msf.c"
 #include "pdb/pdb.c"
 #include "pdb/pdb_stringize.c"
-
 #include "raddbgi_from_pdb.c"
 
+//- rjf: entry point
+
 int
-main(int argc, char **argv){
+main(int argc, char **argv)
+{
   local_persist TCTX main_thread_tctx = {0};
   tctx_init_and_equip(&main_thread_tctx);
 #if PROFILE_TELEMETRY
@@ -40,69 +44,76 @@ main(int argc, char **argv){
   tmSetMaxThreadCount(1024);
   tmInitialize(tm_data_size, tm_data);
 #endif
-  
   ThreadName("[main]");
   
+  //- rjf: initialize dependencies
+  os_init(argc, argv);
+  
+  //- rjf: initialize state, parse command line
   Arena *arena = arena_alloc();
   String8List args = os_string_list_from_argcv(arena, argc, argv);
   CmdLine cmdline = cmd_line_from_string_list(arena, args);
+  B32 should_capture = cmd_line_has_flag(&cmdline, str8_lit("capture"));
+  P2R_ConvertIn *convert_in = p2r_convert_in_from_cmd_line(arena, &cmdline);
   
-  ProfBeginCapture("raddbgi_from_pdb");
-  
-  //- rjf: parse arguments
-  P2R_Params *params = p2r_params_from_cmd_line(arena, &cmdline);
-  
-  //- rjf: show input errors
-  if (params->errors.node_count > 0 &&
-      !params->hide_errors.input){
-    for (String8Node *node = params->errors.first;
-         node != 0;
-         node = node->next){
-      fprintf(stderr, "error(input): %.*s\n", str8_varg(node->string));
-    }
+  //- rjf: begin capture
+  if(should_capture)
+  {
+    ProfBeginCapture(argv[0]);
   }
   
-  //- rjf: open output file
-  String8 output_name = push_str8_copy(arena, params->output_name);
-  FILE *out_file = fopen((char*)output_name.str, "wb");
-  if(out_file == 0 && !params->hide_errors.output)
+  //- rjf: display errors with input
+  if(convert_in->errors.node_count > 0 && !convert_in->hide_errors.input)
   {
-    fprintf(stderr, "error(output): could not open output file\n");
+    for(String8Node *n = convert_in->errors.first; n != 0; n = n->next)
+    {
+      fprintf(stderr, "error(input): %.*s\n", str8_varg(n->string));
+    }
   }
   
   //- rjf: convert
-  P2R_Out *out = 0;
-  if(out_file != 0)
+  P2R_ConvertOut *convert_out = 0;
+  ProfScope("convert")
   {
-    out = p2r_convert(arena, params);
+    convert_out = p2r_convert(arena, convert_in);
   }
   
-  //- rjf: print dump
-  if(out != 0)
+  //- rjf: bake
+  String8List bake_strings = {0};
+  ProfScope("bake")
   {
-    for(String8Node *node = out->dump.first; node != 0; node = node->next)
+    RDIM_BakeParams bake_params = {0};
     {
-      fwrite(node->string.str, 1, node->string.size, stdout);
+      bake_params.top_level_info   = convert_out->top_level_info;
+      bake_params.binary_sections  = convert_out->binary_sections;
+      bake_params.types            = convert_out->types;
+      bake_params.global_variables = convert_out->global_variables;
+      bake_params.thread_variables = convert_out->thread_variables;
+      bake_params.procedures       = convert_out->procedures;
+      bake_params.scopes           = convert_out->scopes;
     }
+    bake_strings = rdim_bake(arena, &bake_params);
   }
   
-  //- rjf: bake file
-  if(out != 0 && out->good_parse && params->output_name.size > 0 && out->good_parse)
+  //- rjf: write
+  ProfScope("write")
   {
-    String8List baked = {0};
-    rdim_bake_file(arena, out->root, &baked);
-    for(String8Node *node = baked.first; node != 0; node = node->next)
+    OS_Handle output_file = os_file_open(OS_AccessFlag_Read|OS_AccessFlag_Write, convert_in->output_name);
+    U64 off = 0;
+    for(String8Node *n = bake_strings.first; n != 0; n = n->next)
     {
-      fwrite(node->string.str, node->string.size, 1, out_file);
+      os_file_write(output_file, r1u64(off, off+n->string.size), n->string.str);
+      off += n->string.size;
     }
+    os_file_close(output_file);
   }
   
-  //- rjf: close output file
-  if(out_file != 0)
+  
+  //- rjf: end capture
+  if(should_capture)
   {
-    fclose(out_file);
+    ProfEndCapture();
   }
   
-  ProfEndCapture();
   return(0);
 }