ed/raddebugger
1
0
Fork 0
mirror of https://github.com/Ed94/raddebugger.git synced 2026-07-10 11:31:37 -07:00
Code Issues Packages Projects Releases Wiki Activity

p2r: wide/async per-unit src file gather/dedup; wide/async line (& inline) table conversion

Browse Source
This commit is contained in:
Ryan Fleury
2025-06-03 17:10:36 -07:00
parent ad0ead92c3
commit 0512b61d95
3 changed files with 852 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib_rdi_make/rdi_make.c
+4
View File
@@ -646,6 +646,8 @@ rdim_src_file_chunk_list_concat_in_place(RDIM_SrcFileChunkList *dst, RDIM_SrcFil
dst->last = to_push->last;
dst->chunk_count += to_push->chunk_count;
dst->total_count += to_push->total_count;
dst->source_line_map_count += to_push->source_line_map_count;
dst->total_line_count += to_push->total_line_count;
}
else if(dst->first == 0)
{
@@ -714,7 +716,9 @@ rdim_line_table_chunk_list_concat_in_place(RDIM_LineTableChunkList *dst, RDIM_Li
dst->last = to_push->last;
dst->chunk_count += to_push->chunk_count;
dst->total_count += to_push->total_count;
dst->total_seq_count += to_push->total_seq_count;
dst->total_line_count += to_push->total_line_count;
dst->total_col_count += to_push->total_col_count;
}
else if(dst->first == 0)
{
src/rdi_from_pdb/rdi_from_pdb.c
+780 -44
View File
@@ -577,6 +577,624 @@ ASYNC_WORK_DEF(p2r_comp_unit_contributions_parse_work)
return out;
}
////////////////////////////////
//~ rjf: Unit Source File Gathering Tasks
ASYNC_WORK_DEF(p2r_gather_unit_src_file_work)
{
ProfBeginFunction();
Arena *arena = p2r_state->work_thread_arenas[thread_idx];
Temp scratch = scratch_begin(&arena, 1);
P2R_GatherUnitSrcFilesIn *in = (P2R_GatherUnitSrcFilesIn *)input;
P2R_GatherUnitSrcFilesOut *out = push_array(arena, P2R_GatherUnitSrcFilesOut, 1);
PDB_CompUnit *pdb_unit = in->comp_unit;
CV_SymParsed *pdb_unit_sym = in->comp_unit_syms;
CV_C13Parsed *pdb_unit_c13 = in->comp_unit_c13s;
CV_RecRange *rec_ranges_first = pdb_unit_sym->sym_ranges.ranges;
CV_RecRange *rec_ranges_opl = rec_ranges_first+pdb_unit_sym->sym_ranges.count;
String8List src_file_paths = {0};
{
//- rjf: build local hash table to dedup files within this unit
U64 hit_path_slots_count = 4096;
String8Node **hit_path_slots = push_array(scratch.arena, String8Node *, hit_path_slots_count);
//- rjf: produce obj name/path
String8 obj_name = pdb_unit->obj_name;
if(str8_match(obj_name, str8_lit("* Linker *"), 0) ||
str8_match(obj_name, str8_lit("Import:"), StringMatchFlag_RightSideSloppy))
{
MemoryZeroStruct(&obj_name);
}
String8 obj_folder_path = lower_from_str8(scratch.arena, str8_chop_last_slash(obj_name));
//- rjf: find all files in this unit's (non-inline) line info
ProfScope("find all files in this unit's (non-inline) line info")
for(CV_C13SubSectionNode *node = pdb_unit_c13->first_sub_section;
node != 0;
node = node->next)
{
if(node->kind == CV_C13SubSectionKind_Lines)
{
for(CV_C13LinesParsedNode *lines_n = node->lines_first;
lines_n != 0;
lines_n = lines_n->next)
{
// rjf: file name -> normalized file path
String8 file_path = lines_n->v.file_name;
String8 file_path_normalized = lower_from_str8(scratch.arena, str8_skip_chop_whitespace(file_path));
{
PathStyle file_path_normalized_style = path_style_from_str8(file_path_normalized);
String8List file_path_normalized_parts = str8_split_path(scratch.arena, file_path_normalized);
if(file_path_normalized_style == PathStyle_Relative)
{
String8List obj_folder_path_parts = str8_split_path(scratch.arena, obj_folder_path);
str8_list_concat_in_place(&obj_folder_path_parts, &file_path_normalized_parts);
file_path_normalized_parts = obj_folder_path_parts;
file_path_normalized_style = path_style_from_str8(obj_folder_path);
}
str8_path_list_resolve_dots_in_place(&file_path_normalized_parts, file_path_normalized_style);
file_path_normalized = str8_path_list_join_by_style(scratch.arena, &file_path_normalized_parts, file_path_normalized_style);
}
// rjf: normalized file path -> source file node
U64 file_path_normalized_hash = rdi_hash(file_path_normalized.str, file_path_normalized.size);
U64 hit_path_slot = file_path_normalized_hash%hit_path_slots_count;
String8Node *hit_path_node = 0;
for(String8Node *n = hit_path_slots[hit_path_slot]; n != 0; n = n->next)
{
if(str8_match(n->string, file_path_normalized, 0))
{
hit_path_node = n;
break;
}
}
if(hit_path_node == 0)
{
hit_path_node = push_array(scratch.arena, String8Node, 1);
SLLStackPush(hit_path_slots[hit_path_slot], hit_path_node);
hit_path_node->string = file_path_normalized;
str8_list_push(scratch.arena, &src_file_paths, push_str8_copy(arena, file_path_normalized));
}
}
}
}
//- rjf: find all files in unit's inline line info
ProfScope("find all files in unit's inline line info")
{
U64 base_voff = 0;
for(CV_RecRange *rec_range = rec_ranges_first;
rec_range < rec_ranges_opl;
rec_range += 1)
{
//- rjf: rec range -> symbol info range
U64 sym_off_first = rec_range->off + 2;
U64 sym_off_opl = rec_range->off + rec_range->hdr.size;
//- rjf: skip invalid ranges
if(sym_off_opl > pdb_unit_sym->data.size || sym_off_first > pdb_unit_sym->data.size || sym_off_first > sym_off_opl)
{
continue;
}
//- rjf: unpack symbol info
CV_SymKind kind = rec_range->hdr.kind;
U64 sym_header_struct_size = cv_header_struct_size_from_sym_kind(kind);
void *sym_header_struct_base = pdb_unit_sym->data.str + sym_off_first;
void *sym_data_opl = pdb_unit_sym->data.str + sym_off_opl;
//- rjf: skip bad sizes
if(sym_off_first + sym_header_struct_size > sym_off_opl)
{
continue;
}
//- rjf: process symbol
switch(kind)
{
default:{}break;
//- rjf: LPROC32/GPROC32 (gather base address)
case CV_SymKind_LPROC32:
case CV_SymKind_GPROC32:
{
CV_SymProc32 *proc32 = (CV_SymProc32 *)sym_header_struct_base;
COFF_SectionHeader *section = (0 < proc32->sec && proc32->sec <= in->coff_sections.count) ? &in->coff_sections.v[proc32->sec-1] : 0;
if(section != 0)
{
base_voff = section->voff + proc32->off;
}
}break;
//- rjf: INLINESITE
case CV_SymKind_INLINESITE:
{
// rjf: unpack sym
CV_SymInlineSite *sym = (CV_SymInlineSite *)sym_header_struct_base;
String8 binary_annots = str8((U8 *)(sym+1), rec_range->hdr.size - sizeof(rec_range->hdr.kind) - sizeof(*sym));
// rjf: map inlinee -> parsed cv c13 inlinee line info
CV_C13InlineeLinesParsed *inlinee_lines_parsed = 0;
{
U64 hash = cv_hash_from_item_id(sym->inlinee);
U64 slot_idx = hash%pdb_unit_c13->inlinee_lines_parsed_slots_count;
for(CV_C13InlineeLinesParsedNode *n = pdb_unit_c13->inlinee_lines_parsed_slots[slot_idx]; n != 0; n = n->hash_next)
{
if(n->v.inlinee == sym->inlinee)
{
inlinee_lines_parsed = &n->v;
break;
}
}
}
// rjf: build line table, fill with parsed binary annotations
if(inlinee_lines_parsed != 0)
{
// rjf: grab checksums sub-section
CV_C13SubSectionNode *file_chksms = pdb_unit_c13->file_chksms_sub_section;
// rjf: gathered lines
U32 last_file_off = max_U32;
U32 curr_file_off = max_U32;
U64 line_count = 0;
CV_C13InlineSiteDecoder decoder = cv_c13_inline_site_decoder_init(inlinee_lines_parsed->file_off, inlinee_lines_parsed->first_source_ln, base_voff);
for(;;)
{
// rjf: step & update
CV_C13InlineSiteDecoderStep step = cv_c13_inline_site_decoder_step(&decoder, binary_annots);
if(step.flags & CV_C13InlineSiteDecoderStepFlag_EmitFile)
{
last_file_off = curr_file_off;
curr_file_off = step.file_off;
}
if(step.flags == 0 && line_count > 0)
{
last_file_off = curr_file_off;
curr_file_off = max_U32;
}
// rjf: file updated -> gather new file name
if(last_file_off != max_U32 && last_file_off != curr_file_off)
{
String8 seq_file_name = {0};
if(last_file_off + sizeof(CV_C13Checksum) <= file_chksms->size)
{
CV_C13Checksum *checksum = (CV_C13Checksum*)(pdb_unit_c13->data.str + file_chksms->off + last_file_off);
U32 name_off = checksum->name_off;
seq_file_name = pdb_strtbl_string_from_off(in->pdb_strtbl, name_off);
}
// rjf: file name -> normalized file path
String8 file_path = seq_file_name;
String8 file_path_normalized = lower_from_str8(scratch.arena, str8_skip_chop_whitespace(file_path));
{
PathStyle file_path_normalized_style = path_style_from_str8(file_path_normalized);
String8List file_path_normalized_parts = str8_split_path(scratch.arena, file_path_normalized);
if(file_path_normalized_style == PathStyle_Relative)
{
String8List obj_folder_path_parts = str8_split_path(scratch.arena, obj_folder_path);
str8_list_concat_in_place(&obj_folder_path_parts, &file_path_normalized_parts);
file_path_normalized_parts = obj_folder_path_parts;
file_path_normalized_style = path_style_from_str8(obj_folder_path);
}
str8_path_list_resolve_dots_in_place(&file_path_normalized_parts, file_path_normalized_style);
file_path_normalized = str8_path_list_join_by_style(scratch.arena, &file_path_normalized_parts, file_path_normalized_style);
}
// rjf: normalized file path -> source file node
U64 file_path_normalized_hash = rdi_hash(file_path_normalized.str, file_path_normalized.size);
U64 hit_path_slot = file_path_normalized_hash%hit_path_slots_count;
String8Node *hit_path_node = 0;
for(String8Node *n = hit_path_slots[hit_path_slot]; n != 0; n = n->next)
{
if(str8_match(n->string, file_path_normalized, 0))
{
hit_path_node = n;
break;
}
}
if(hit_path_node == 0)
{
hit_path_node = push_array(scratch.arena, String8Node, 1);
SLLStackPush(hit_path_slots[hit_path_slot], hit_path_node);
hit_path_node->string = file_path_normalized;
str8_list_push(scratch.arena, &src_file_paths, push_str8_copy(arena, file_path_normalized));
}
line_count = 0;
}
// rjf: count lines
if(step.flags & CV_C13InlineSiteDecoderStepFlag_EmitLine)
{
line_count += 1;
}
// rjf: no more flags -> done
if(step.flags == 0)
{
break;
}
}
}
}break;
}
}
}
}
out->src_file_paths = str8_array_from_list(arena, &src_file_paths);
scratch_end(scratch);
ProfEnd();
return out;
}
////////////////////////////////
//~ rjf: Unit Conversion Tasks
ASYNC_WORK_DEF(p2r_unit_convert_work)
{
ProfBeginFunction();
Arena *arena = p2r_state->work_thread_arenas[thread_idx];
Temp scratch = scratch_begin(&arena, 1);
P2R_UnitConvertIn *in = (P2R_UnitConvertIn *)input;
P2R_UnitConvertOut *out = push_array(arena, P2R_UnitConvertOut, 1);
////////////////////////////
//- rjf: pass 1: build per-unit info & per-unit line table
//
ProfScope("pass 1: build per-unit info & per-unit line table")
{
PDB_CompUnit *pdb_unit = in->comp_unit;
CV_SymParsed *pdb_unit_sym = in->comp_unit_syms;
CV_C13Parsed *pdb_unit_c13 = in->comp_unit_c13s;
//- rjf: produce unit name
String8 unit_name = pdb_unit->obj_name;
if(unit_name.size != 0)
{
String8 unit_name_past_last_slash = str8_skip_last_slash(unit_name);
if(unit_name_past_last_slash.size != 0)
{
unit_name = unit_name_past_last_slash;
}
}
//- rjf: produce obj name/path
String8 obj_name = pdb_unit->obj_name;
if(str8_match(obj_name, str8_lit("* Linker *"), 0) ||
str8_match(obj_name, str8_lit("Import:"), StringMatchFlag_RightSideSloppy))
{
MemoryZeroStruct(&obj_name);
}
String8 obj_folder_path = lower_from_str8(scratch.arena, str8_chop_last_slash(obj_name));
//- rjf: build this unit's line table, fill out primary line info (inline info added after)
RDIM_LineTable *line_table = 0;
for(CV_C13SubSectionNode *node = pdb_unit_c13->first_sub_section;
node != 0;
node = node->next)
{
if(node->kind == CV_C13SubSectionKind_Lines)
{
for(CV_C13LinesParsedNode *lines_n = node->lines_first;
lines_n != 0;
lines_n = lines_n->next)
{
CV_C13LinesParsed *lines = &lines_n->v;
// rjf: file name -> normalized file path
String8 file_path = lines->file_name;
String8 file_path_normalized = lower_from_str8(scratch.arena, str8_skip_chop_whitespace(file_path));
{
PathStyle file_path_normalized_style = path_style_from_str8(file_path_normalized);
String8List file_path_normalized_parts = str8_split_path(scratch.arena, file_path_normalized);
if(file_path_normalized_style == PathStyle_Relative)
{
String8List obj_folder_path_parts = str8_split_path(scratch.arena, obj_folder_path);
str8_list_concat_in_place(&obj_folder_path_parts, &file_path_normalized_parts);
file_path_normalized_parts = obj_folder_path_parts;
file_path_normalized_style = path_style_from_str8(obj_folder_path);
}
str8_path_list_resolve_dots_in_place(&file_path_normalized_parts, file_path_normalized_style);
file_path_normalized = str8_path_list_join_by_style(scratch.arena, &file_path_normalized_parts, file_path_normalized_style);
}
// rjf: normalized file path -> source file node
U64 file_path_normalized_hash = rdi_hash(file_path_normalized.str, file_path_normalized.size);
U64 src_file_slot = file_path_normalized_hash%in->src_file_map->slots_count;
P2R_SrcFileNode *src_file_node = 0;
if(lines->line_count != 0)
{
for(P2R_SrcFileNode *n = in->src_file_map->slots[src_file_slot]; n != 0; n = n->next)
{
if(str8_match(n->src_file->normal_full_path, file_path_normalized, 0))
{
src_file_node = n;
break;
}
}
}
// rjf: push sequence into both line table & source file's line map
if(src_file_node != 0)
{
if(line_table == 0)
{
line_table = rdim_line_table_chunk_list_push(arena, &out->line_tables, 256);
}
RDIM_LineSequence *seq = rdim_line_table_push_sequence(arena, &out->line_tables, line_table, src_file_node->src_file, lines->voffs, lines->line_nums, lines->col_nums, lines->line_count);
}
}
}
}
//- rjf: build unit
RDIM_Unit *dst_unit = rdim_unit_chunk_list_push(arena, &out->units, 1);
dst_unit->unit_name = unit_name;
dst_unit->compiler_name = pdb_unit_sym->info.compiler_name;
dst_unit->object_file = obj_name;
dst_unit->archive_file = pdb_unit->group_name;
dst_unit->language = p2r_rdi_language_from_cv_language(pdb_unit_sym->info.language);
dst_unit->line_table = line_table;
}
////////////////////////////
//- rjf: pass 2: build per-unit voff ranges from comp unit contributions table
//
PDB_CompUnitContribution *contrib_ptr = in->comp_unit_contributions->contributions;
PDB_CompUnitContribution *contrib_opl = contrib_ptr + in->comp_unit_contributions->count;
ProfScope("pass 2: build per-unit voff ranges from comp unit contributions table")
for(;contrib_ptr < contrib_opl; contrib_ptr += 1)
{
if(contrib_ptr->mod == in->comp_unit_idx)
{
RDIM_Unit *unit = &out->units.first->v[0];
RDIM_Rng1U64 range = {contrib_ptr->voff_first, contrib_ptr->voff_opl};
rdim_rng1u64_list_push(arena, &unit->voff_ranges, range);
}
}
////////////////////////////
//- rjf: pass 3: parse all inlinee line tables
//
ProfScope("pass 3: parse all inlinee line tables")
{
//- rjf: unpack unit
PDB_CompUnit *pdb_unit = in->comp_unit;
CV_SymParsed *pdb_unit_sym = in->comp_unit_syms;
CV_C13Parsed *pdb_unit_c13 = in->comp_unit_c13s;
CV_RecRange *rec_ranges_first = pdb_unit_sym->sym_ranges.ranges;
CV_RecRange *rec_ranges_opl = rec_ranges_first+pdb_unit_sym->sym_ranges.count;
//- rjf: produce obj name/path
String8 obj_name = pdb_unit->obj_name;
if(str8_match(obj_name, str8_lit("* Linker *"), 0) ||
str8_match(obj_name, str8_lit("Import:"), StringMatchFlag_RightSideSloppy))
{
MemoryZeroStruct(&obj_name);
}
String8 obj_folder_path = lower_from_str8(scratch.arena, str8_chop_last_slash(obj_name));
//- rjf: parse inlinee line tables
U64 base_voff = 0;
for(CV_RecRange *rec_range = rec_ranges_first;
rec_range < rec_ranges_opl;
rec_range += 1)
{
//- rjf: rec range -> symbol info range
U64 sym_off_first = rec_range->off + 2;
U64 sym_off_opl = rec_range->off + rec_range->hdr.size;
//- rjf: skip invalid ranges
if(sym_off_opl > pdb_unit_sym->data.size || sym_off_first > pdb_unit_sym->data.size || sym_off_first > sym_off_opl)
{
continue;
}
//- rjf: unpack symbol info
CV_SymKind kind = rec_range->hdr.kind;
U64 sym_header_struct_size = cv_header_struct_size_from_sym_kind(kind);
void *sym_header_struct_base = pdb_unit_sym->data.str + sym_off_first;
void *sym_data_opl = pdb_unit_sym->data.str + sym_off_opl;
//- rjf: skip bad sizes
if(sym_off_first + sym_header_struct_size > sym_off_opl)
{
continue;
}
//- rjf: process symbol
switch(kind)
{
default:{}break;
//- rjf: LPROC32/GPROC32 (gather base address)
case CV_SymKind_LPROC32:
case CV_SymKind_GPROC32:
{
CV_SymProc32 *proc32 = (CV_SymProc32 *)sym_header_struct_base;
COFF_SectionHeader *section = (0 < proc32->sec && proc32->sec <= in->coff_sections.count) ? &in->coff_sections.v[proc32->sec-1] : 0;
if(section != 0)
{
base_voff = section->voff + proc32->off;
}
}break;
//- rjf: INLINESITE
case CV_SymKind_INLINESITE:
{
// rjf: unpack sym
CV_SymInlineSite *sym = (CV_SymInlineSite *)sym_header_struct_base;
String8 binary_annots = str8((U8 *)(sym+1), rec_range->hdr.size - sizeof(rec_range->hdr.kind) - sizeof(*sym));
// rjf: map inlinee -> parsed cv c13 inlinee line info
CV_C13InlineeLinesParsed *inlinee_lines_parsed = 0;
{
U64 hash = cv_hash_from_item_id(sym->inlinee);
U64 slot_idx = hash%pdb_unit_c13->inlinee_lines_parsed_slots_count;
for(CV_C13InlineeLinesParsedNode *n = pdb_unit_c13->inlinee_lines_parsed_slots[slot_idx]; n != 0; n = n->hash_next)
{
if(n->v.inlinee == sym->inlinee)
{
inlinee_lines_parsed = &n->v;
break;
}
}
}
// rjf: build line table, fill with parsed binary annotations
if(inlinee_lines_parsed != 0)
{
// rjf: grab checksums sub-section
CV_C13SubSectionNode *file_chksms = pdb_unit_c13->file_chksms_sub_section;
// rjf: gathered lines
typedef struct LineChunk LineChunk;
struct LineChunk
{
LineChunk *next;
U64 cap;
U64 count;
U64 *voffs; // [line_count + 1] (sorted)
U32 *line_nums; // [line_count]
U16 *col_nums; // [2*line_count]
};
LineChunk *first_line_chunk = 0;
LineChunk *last_line_chunk = 0;
U64 total_line_chunk_line_count = 0;
U32 last_file_off = max_U32;
U32 curr_file_off = max_U32;
RDIM_LineTable* line_table = 0;
CV_C13InlineSiteDecoder decoder = cv_c13_inline_site_decoder_init(inlinee_lines_parsed->file_off, inlinee_lines_parsed->first_source_ln, base_voff);
for(;;)
{
// rjf: step & update
CV_C13InlineSiteDecoderStep step = cv_c13_inline_site_decoder_step(&decoder, binary_annots);
if(step.flags & CV_C13InlineSiteDecoderStepFlag_EmitFile)
{
last_file_off = curr_file_off;
curr_file_off = step.file_off;
}
if(step.flags == 0 && total_line_chunk_line_count > 0)
{
last_file_off = curr_file_off;
curr_file_off = max_U32;
}
// rjf: file updated -> push line chunks gathered for this file
if(last_file_off != max_U32 && last_file_off != curr_file_off)
{
String8 seq_file_name = {0};
if(last_file_off + sizeof(CV_C13Checksum) <= file_chksms->size)
{
CV_C13Checksum *checksum = (CV_C13Checksum*)(pdb_unit_c13->data.str + file_chksms->off + last_file_off);
U32 name_off = checksum->name_off;
seq_file_name = pdb_strtbl_string_from_off(in->pdb_strtbl, name_off);
}
// rjf: file name -> normalized file path
String8 file_path = seq_file_name;
String8 file_path_normalized = lower_from_str8(scratch.arena, str8_skip_chop_whitespace(file_path));
{
PathStyle file_path_normalized_style = path_style_from_str8(file_path_normalized);
String8List file_path_normalized_parts = str8_split_path(scratch.arena, file_path_normalized);
if(file_path_normalized_style == PathStyle_Relative)
{
String8List obj_folder_path_parts = str8_split_path(scratch.arena, obj_folder_path);
str8_list_concat_in_place(&obj_folder_path_parts, &file_path_normalized_parts);
file_path_normalized_parts = obj_folder_path_parts;
file_path_normalized_style = path_style_from_str8(obj_folder_path);
}
str8_path_list_resolve_dots_in_place(&file_path_normalized_parts, file_path_normalized_style);
file_path_normalized = str8_path_list_join_by_style(scratch.arena, &file_path_normalized_parts, file_path_normalized_style);
}
// rjf: normalized file path -> source file node
U64 file_path_normalized_hash = rdi_hash(file_path_normalized.str, file_path_normalized.size);
U64 src_file_slot = file_path_normalized_hash%in->src_file_map->slots_count;
P2R_SrcFileNode *src_file_node = 0;
for(P2R_SrcFileNode *n = in->src_file_map->slots[src_file_slot]; n != 0; n = n->next)
{
if(str8_match(n->src_file->normal_full_path, file_path_normalized, 0))
{
src_file_node = n;
break;
}
}
// rjf: gather all lines
RDI_U64 *voffs = 0;
RDI_U32 *line_nums = 0;
RDI_U64 line_count = 0;
if(src_file_node != 0)
{
voffs = push_array_no_zero(arena, RDI_U64, total_line_chunk_line_count+1);
line_nums = push_array_no_zero(arena, RDI_U32, total_line_chunk_line_count);
line_count = total_line_chunk_line_count;
U64 dst_idx = 0;
for(LineChunk *chunk = first_line_chunk; chunk != 0; chunk = chunk->next)
{
MemoryCopy(voffs+dst_idx, chunk->voffs, sizeof(U64)*(chunk->count+1));
MemoryCopy(line_nums+dst_idx, chunk->line_nums, sizeof(U32)*chunk->count);
dst_idx += chunk->count;
}
}
// rjf: push
if(line_count != 0)
{
if(line_table == 0)
{
line_table = rdim_line_table_chunk_list_push(arena, &out->line_tables, 256);
if(out->unit_first_inline_site_line_table == 0)
{
out->unit_first_inline_site_line_table = line_table;
}
}
rdim_line_table_push_sequence(arena, &out->line_tables, line_table, src_file_node->src_file, voffs, line_nums, 0, line_count);
}
// rjf: clear line chunks for subsequent sequences
first_line_chunk = last_line_chunk = 0;
total_line_chunk_line_count = 0;
}
// rjf: new line -> emit to chunk
if(step.flags & CV_C13InlineSiteDecoderStepFlag_EmitLine)
{
LineChunk *chunk = last_line_chunk;
if(chunk == 0 || chunk->count+1 >= chunk->cap)
{
chunk = push_array(scratch.arena, LineChunk, 1);
SLLQueuePush(first_line_chunk, last_line_chunk, chunk);
chunk->cap = 8;
chunk->voffs = push_array_no_zero(scratch.arena, U64, chunk->cap);
chunk->line_nums = push_array_no_zero(scratch.arena, U32, chunk->cap);
}
chunk->voffs[chunk->count] = step.line_voff;
chunk->voffs[chunk->count+1] = step.line_voff_end;
chunk->line_nums[chunk->count] = step.ln;
chunk->count += 1;
total_line_chunk_line_count += 1;
}
// rjf: no more flags -> done
if(step.flags == 0)
{
break;
}
}
}
}break;
}
}
}
scratch_end(scratch);
ProfEnd();
return out;
}
////////////////////////////////
//~ rjf: Unit Conversion Tasks
@@ -585,15 +1203,12 @@ ASYNC_WORK_DEF(p2r_units_convert_work)
ProfBeginFunction();
Arena *arena = p2r_state->work_thread_arenas[thread_idx];
Temp scratch = scratch_begin(&arena, 1);
P2R_UnitConvertIn *in = (P2R_UnitConvertIn *)input;
P2R_UnitConvertOut *out = push_array(arena, P2R_UnitConvertOut, 1);
P2R_UnitsConvertIn *in = (P2R_UnitsConvertIn *)input;
P2R_UnitsConvertOut *out = push_array(arena, P2R_UnitsConvertOut, 1);
ProfScope("build units, initial src file map, & collect unit source files")
if(in->comp_units != 0)
{
U64 units_chunk_cap = in->comp_units->count;
P2R_SrcFileMap src_file_map = {0};
src_file_map.slots_count = 65536;
src_file_map.slots = push_array(scratch.arena, P2R_SrcFileNode *, src_file_map.slots_count);
////////////////////////////
//- rjf: pass 1: build per-unit info & per-unit line tables
@@ -658,33 +1273,28 @@ ASYNC_WORK_DEF(p2r_units_convert_work)
// rjf: normalized file path -> source file node
U64 file_path_normalized_hash = rdi_hash(file_path_normalized.str, file_path_normalized.size);
U64 src_file_slot = file_path_normalized_hash%src_file_map.slots_count;
U64 src_file_slot = file_path_normalized_hash%in->src_file_map->slots_count;
P2R_SrcFileNode *src_file_node = 0;
for(P2R_SrcFileNode *n = src_file_map.slots[src_file_slot]; n != 0; n = n->next)
if(lines->line_count != 0)
{
if(str8_match(n->src_file->normal_full_path, file_path_normalized, 0))
for(P2R_SrcFileNode *n = in->src_file_map->slots[src_file_slot]; n != 0; n = n->next)
{
src_file_node = n;
break;
if(str8_match(n->src_file->normal_full_path, file_path_normalized, 0))
{
src_file_node = n;
break;
}
}
}
if(src_file_node == 0)
{
src_file_node = push_array(scratch.arena, P2R_SrcFileNode, 1);
SLLStackPush(src_file_map.slots[src_file_slot], src_file_node);
src_file_node->src_file = rdim_src_file_chunk_list_push(arena, &out->src_files, 4096);
src_file_node->src_file->normal_full_path = push_str8_copy(arena, file_path_normalized);
}
// rjf: push sequence into both line table & source file's line map
if(lines->line_count != 0)
if(src_file_node != 0)
{
if(line_table == 0)
{
line_table = rdim_line_table_chunk_list_push(arena, &out->line_tables, 256);
}
RDIM_LineSequence *seq = rdim_line_table_push_sequence(arena, &out->line_tables, line_table, src_file_node->src_file, lines->voffs, lines->line_nums, lines->col_nums, lines->line_count);
rdim_src_file_push_line_sequence(arena, &out->src_files, src_file_node->src_file, seq);
rdim_line_table_push_sequence(arena, &out->line_tables, line_table, src_file_node->src_file, lines->voffs, lines->line_nums, lines->col_nums, lines->line_count);
}
}
}
@@ -807,7 +1417,6 @@ ASYNC_WORK_DEF(p2r_units_convert_work)
}
// rjf: build line table, fill with parsed binary annotations
if(inlinee_lines_parsed != 0)
{
// rjf: grab checksums sub-section
@@ -834,8 +1443,8 @@ ASYNC_WORK_DEF(p2r_units_convert_work)
CV_C13InlineSiteDecoder decoder = cv_c13_inline_site_decoder_init(inlinee_lines_parsed->file_off, inlinee_lines_parsed->first_source_ln, base_voff);
for(;;)
{
// rjf: step & update
CV_C13InlineSiteDecoderStep step = cv_c13_inline_site_decoder_step(&decoder, binary_annots);
if(step.flags & CV_C13InlineSiteDecoderStepFlag_EmitFile)
{
last_file_off = curr_file_off;
@@ -846,10 +1455,11 @@ ASYNC_WORK_DEF(p2r_units_convert_work)
last_file_off = curr_file_off;
curr_file_off = max_U32;
}
if((last_file_off != max_U32 && last_file_off != curr_file_off))
// rjf: file updated -> push line chunks gathered for this file
if(last_file_off != max_U32 && last_file_off != curr_file_off)
{
String8 seq_file_name = {0};
if(last_file_off + sizeof(CV_C13Checksum) <= file_chksms->size)
{
CV_C13Checksum *checksum = (CV_C13Checksum*)(unit_c13->data.str + file_chksms->off + last_file_off);
@@ -876,9 +1486,9 @@ ASYNC_WORK_DEF(p2r_units_convert_work)
// rjf: normalized file path -> source file node
U64 file_path_normalized_hash = rdi_hash(file_path_normalized.str, file_path_normalized.size);
U64 src_file_slot = file_path_normalized_hash%src_file_map.slots_count;
U64 src_file_slot = file_path_normalized_hash%in->src_file_map->slots_count;
P2R_SrcFileNode *src_file_node = 0;
for(P2R_SrcFileNode *n = src_file_map.slots[src_file_slot]; n != 0; n = n->next)
for(P2R_SrcFileNode *n = in->src_file_map->slots[src_file_slot]; n != 0; n = n->next)
{
if(str8_match(n->src_file->normal_full_path, file_path_normalized, 0))
{
@@ -886,19 +1496,16 @@ ASYNC_WORK_DEF(p2r_units_convert_work)
break;
}
}
if(src_file_node == 0)
{
src_file_node = push_array(scratch.arena, P2R_SrcFileNode, 1);
SLLStackPush(src_file_map.slots[src_file_slot], src_file_node);
src_file_node->src_file = rdim_src_file_chunk_list_push(arena, &out->src_files, 4096);
src_file_node->src_file->normal_full_path = push_str8_copy(arena, file_path_normalized);
}
// rjf: gather all lines
RDI_U64 *voffs = push_array_no_zero(arena, RDI_U64, total_line_chunk_line_count+1);
RDI_U32 *line_nums = push_array_no_zero(arena, RDI_U32, total_line_chunk_line_count);
RDI_U64 line_count = total_line_chunk_line_count;
RDI_U64 *voffs = 0;
RDI_U32 *line_nums = 0;
RDI_U64 line_count = 0;
if(src_file_node != 0)
{
voffs = push_array_no_zero(arena, RDI_U64, total_line_chunk_line_count+1);
line_nums = push_array_no_zero(arena, RDI_U32, total_line_chunk_line_count);
line_count = total_line_chunk_line_count;
U64 dst_idx = 0;
for(LineChunk *chunk = first_line_chunk; chunk != 0; chunk = chunk->next)
{
@@ -919,8 +1526,7 @@ ASYNC_WORK_DEF(p2r_units_convert_work)
out->units_first_inline_site_line_tables[comp_unit_idx] = line_table;
}
}
RDIM_LineSequence *seq = rdim_line_table_push_sequence(arena, &out->line_tables, line_table, src_file_node->src_file, voffs, line_nums, 0, line_count);
rdim_src_file_push_line_sequence(arena, &out->src_files, src_file_node->src_file, seq);
rdim_line_table_push_sequence(arena, &out->line_tables, line_table, src_file_node->src_file, voffs, line_nums, 0, line_count);
}
// rjf: clear line chunks for subsequent sequences
@@ -928,6 +1534,7 @@ ASYNC_WORK_DEF(p2r_units_convert_work)
total_line_chunk_line_count = 0;
}
// rjf: new line -> emit to chunk
if(step.flags & CV_C13InlineSiteDecoderStepFlag_EmitLine)
{
LineChunk *chunk = last_line_chunk;
@@ -935,7 +1542,7 @@ ASYNC_WORK_DEF(p2r_units_convert_work)
{
chunk = push_array(scratch.arena, LineChunk, 1);
SLLQueuePush(first_line_chunk, last_line_chunk, chunk);
chunk->cap = 256;
chunk->cap = 8;
chunk->voffs = push_array_no_zero(scratch.arena, U64, chunk->cap);
chunk->line_nums = push_array_no_zero(scratch.arena, U32, chunk->cap);
}
@@ -946,6 +1553,7 @@ ASYNC_WORK_DEF(p2r_units_convert_work)
total_line_chunk_line_count += 1;
}
// rjf: no more flags -> done
if(step.flags == 0)
{
break;
@@ -962,6 +1570,29 @@ ASYNC_WORK_DEF(p2r_units_convert_work)
return out;
}
////////////////////////////////
//~ rjf: Source File Sequence Equipping Task
ASYNC_WORK_DEF(p2r_src_file_seq_equip_work)
{
ProfBeginFunction();
Arena *arena = p2r_state->work_thread_arenas[thread_idx];
P2R_SrcFileSeqEquipIn *in = (P2R_SrcFileSeqEquipIn *)input;
for(RDIM_LineTableChunkNode *line_table_chunk_n = in->line_tables.first; line_table_chunk_n != 0; line_table_chunk_n = line_table_chunk_n->next)
{
for EachIndex(chunk_line_table_idx, line_table_chunk_n->count)
{
RDIM_LineTable *line_table = &line_table_chunk_n->v[chunk_line_table_idx];
for(RDIM_LineSequenceNode *s = line_table->first_seq; s != 0; s = s->next)
{
rdim_src_file_push_line_sequence(arena, &in->src_files, s->v.src_file, &s->v);
}
}
}
ProfEnd();
return 0;
}
////////////////////////////////
//~ rjf: Link Name Map Building Tasks
@@ -3321,10 +3952,87 @@ p2r_convert(Arena *arena, P2R_User2Convert *in)
}
//////////////////////////////////////////////////////////////
//- rjf: kick off unit conversion & source file collection
//- rjf: gather all source file paths; build nodes
//
P2R_UnitConvertIn unit_convert_in = {strtbl, coff_sections, comp_units, comp_unit_contributions, sym_for_unit, c13_for_unit};
RDIM_SrcFileChunkList all_src_files__sequenceless = {0};
P2R_SrcFileMap src_file_map = {0};
ProfScope("gather all source file paths; build nodes")
{
U64 tasks_count = comp_unit_count;
P2R_GatherUnitSrcFilesIn *tasks_inputs = push_array(scratch.arena, P2R_GatherUnitSrcFilesIn, tasks_count);
P2R_GatherUnitSrcFilesOut *tasks_outputs = push_array(scratch.arena, P2R_GatherUnitSrcFilesOut, tasks_count);
ASYNC_Task **tasks = push_array(scratch.arena, ASYNC_Task *, tasks_count);
for EachIndex(idx, tasks_count)
{
tasks_inputs[idx].pdb_strtbl = strtbl;
tasks_inputs[idx].coff_sections = coff_sections;
tasks_inputs[idx].comp_unit = comp_units->units[idx];
tasks_inputs[idx].comp_unit_syms = sym_for_unit[idx];
tasks_inputs[idx].comp_unit_c13s = c13_for_unit[idx];
tasks[idx] = async_task_launch(scratch.arena, p2r_gather_unit_src_file_work, .input = &tasks_inputs[idx]);
}
U64 total_path_count = 0;
for EachIndex(idx, tasks_count)
{
tasks_outputs[idx] = *async_task_join_struct(tasks[idx], P2R_GatherUnitSrcFilesOut);
total_path_count += tasks_outputs[idx].src_file_paths.count;
}
src_file_map.slots_count = total_path_count + total_path_count/2 + 1;
src_file_map.slots = push_array(scratch.arena, P2R_SrcFileNode *, src_file_map.slots_count);
for EachIndex(idx, tasks_count)
{
for EachIndex(path_idx, tasks_outputs[idx].src_file_paths.count)
{
String8 file_path_normalized = tasks_outputs[idx].src_file_paths.v[path_idx];
U64 file_path_normalized_hash = rdi_hash(file_path_normalized.str, file_path_normalized.size);
U64 src_file_slot = file_path_normalized_hash%src_file_map.slots_count;
P2R_SrcFileNode *src_file_node = 0;
for(P2R_SrcFileNode *n = src_file_map.slots[src_file_slot]; n != 0; n = n->next)
{
if(str8_match(n->src_file->normal_full_path, file_path_normalized, 0))
{
src_file_node = n;
break;
}
}
if(src_file_node == 0)
{
src_file_node = push_array(scratch.arena, P2R_SrcFileNode, 1);
SLLStackPush(src_file_map.slots[src_file_slot], src_file_node);
src_file_node->src_file = rdim_src_file_chunk_list_push(arena, &all_src_files__sequenceless, total_path_count);
src_file_node->src_file->normal_full_path = push_str8_copy(arena, file_path_normalized);
}
}
}
}
//////////////////////////////////////////////////////////////
//- rjf: kick off unit conversion tasks
//
U64 unit_convert_tasks_count = comp_unit_count;
P2R_UnitConvertIn *unit_convert_tasks_ins = push_array(scratch.arena, P2R_UnitConvertIn, unit_convert_tasks_count);
ASYNC_Task **unit_convert_tasks = push_array(scratch.arena, ASYNC_Task *, unit_convert_tasks_count);
for EachIndex(idx, unit_convert_tasks_count)
{
P2R_UnitConvertIn *in = &unit_convert_tasks_ins[idx];
in->comp_unit_idx = idx;
in->pdb_strtbl = strtbl;
in->coff_sections = coff_sections;
in->comp_unit = comp_units->units[idx];
in->comp_unit_contributions = comp_unit_contributions;
in->comp_unit_syms = sym_for_unit[idx];
in->comp_unit_c13s = c13_for_unit[idx];
in->src_file_map = &src_file_map;
unit_convert_tasks[idx] = async_task_launch(scratch.arena, p2r_unit_convert_work, .input = in);
}
#if 0
//////////////////////////////////////////////////////////////
//- rjf: kick off unit conversion
//
P2R_UnitsConvertIn unit_convert_in = {strtbl, coff_sections, comp_units, comp_unit_contributions, sym_for_unit, c13_for_unit, &src_file_map};
ASYNC_Task *unit_convert_task = async_task_launch(scratch.arena, p2r_units_convert_work, .input = &unit_convert_in);
#endif
//////////////////////////////////////////////////////////////
//- rjf: join global sym stream parse
@@ -4019,21 +4727,49 @@ p2r_convert(Arena *arena, P2R_User2Convert *in)
link_name_map = &link_name_map__in_progress;
}
//////////////////////////////////////////////////////////////
//- rjf: join unit conversion tasks
//
RDIM_UnitChunkList all_units = {0};
RDIM_LineTableChunkList all_line_tables = {0};
RDIM_LineTable **units_first_inline_site_line_tables = push_array(arena, RDIM_LineTable *, unit_convert_tasks_count);
ProfScope("join unit conversion & src file tasks")
{
for EachIndex(idx, unit_convert_tasks_count)
{
P2R_UnitConvertOut *out = async_task_join_struct(unit_convert_tasks[idx], P2R_UnitConvertOut);
rdim_unit_chunk_list_concat_in_place(&all_units, &out->units);
rdim_line_table_chunk_list_concat_in_place(&all_line_tables, &out->line_tables);
units_first_inline_site_line_tables[idx] = out->unit_first_inline_site_line_table;
}
}
//////////////////////////////////////////////////////////////
//- rjf: kick off source file line sequence equipping task
//
RDIM_SrcFileChunkList all_src_files = {0};
{
P2R_SrcFileSeqEquipIn in = {all_src_files__sequenceless, all_line_tables};
ASYNC_Task *task = async_task_launch(scratch.arena, p2r_src_file_seq_equip_work, .input = &in);
async_task_join(task);
all_src_files = in.src_files;
}
#if 0
//////////////////////////////////////////////////////////////
//- rjf: join unit conversion & src file & line table tasks
//
RDIM_UnitChunkList all_units = {0};
RDIM_SrcFileChunkList all_src_files = {0};
RDIM_LineTableChunkList all_line_tables = {0};
RDIM_LineTable **units_first_inline_site_line_tables = 0;
ProfScope("join unit conversion & src file tasks")
{
P2R_UnitConvertOut *out = async_task_join_struct(unit_convert_task, P2R_UnitConvertOut);
P2R_UnitsConvertOut *out = async_task_join_struct(unit_convert_task, P2R_UnitsConvertOut);
all_units = out->units;
all_src_files = out->src_files;
all_line_tables = out->line_tables;
units_first_inline_site_line_tables = out->units_first_inline_site_line_tables;
}
#endif
//////////////////////////////////////////////////////////////
//- rjf: produce symbols from all streams
src/rdi_from_pdb/rdi_from_pdb.h
+68 -3
View File
@@ -169,10 +169,60 @@ struct P2R_SrcFileMap
U64 slots_count;
};
//- rjf: per-unit source files conversion tasks
typedef struct P2R_GatherUnitSrcFilesIn P2R_GatherUnitSrcFilesIn;
struct P2R_GatherUnitSrcFilesIn
{
PDB_Strtbl *pdb_strtbl;
COFF_SectionHeaderArray coff_sections;
PDB_CompUnit *comp_unit;
CV_SymParsed *comp_unit_syms;
CV_C13Parsed *comp_unit_c13s;
};
typedef struct P2R_GatherUnitSrcFilesOut P2R_GatherUnitSrcFilesOut;
struct P2R_GatherUnitSrcFilesOut
{
String8Array src_file_paths;
};
//- rjf: unit conversion tasks
typedef struct P2R_UnitConvertIn P2R_UnitConvertIn;
struct P2R_UnitConvertIn
{
U64 comp_unit_idx;
PDB_Strtbl *pdb_strtbl;
COFF_SectionHeaderArray coff_sections;
PDB_CompUnit *comp_unit;
PDB_CompUnitContributionArray *comp_unit_contributions;
CV_SymParsed *comp_unit_syms;
CV_C13Parsed *comp_unit_c13s;
P2R_SrcFileMap *src_file_map;
};
typedef struct P2R_UnitConvertOut P2R_UnitConvertOut;
struct P2R_UnitConvertOut
{
RDIM_UnitChunkList units;
RDIM_LineTableChunkList line_tables;
RDIM_LineTable *unit_first_inline_site_line_table;
};
//- rjf: src file sequence equipping task
typedef struct P2R_SrcFileSeqEquipIn P2R_SrcFileSeqEquipIn;
struct P2R_SrcFileSeqEquipIn
{
RDIM_SrcFileChunkList src_files;
RDIM_LineTableChunkList line_tables;
};
//- rjf: units conversion tasks
typedef struct P2R_UnitsConvertIn P2R_UnitsConvertIn;
struct P2R_UnitsConvertIn
{
PDB_Strtbl *pdb_strtbl;
COFF_SectionHeaderArray coff_sections;
@@ -180,13 +230,13 @@ struct P2R_UnitConvertIn
PDB_CompUnitContributionArray *comp_unit_contributions;
CV_SymParsed **comp_unit_syms;
CV_C13Parsed **comp_unit_c13s;
P2R_SrcFileMap *src_file_map;
};
typedef struct P2R_UnitConvertOut P2R_UnitConvertOut;
struct P2R_UnitConvertOut
typedef struct P2R_UnitsConvertOut P2R_UnitsConvertOut;
struct P2R_UnitsConvertOut
{
RDIM_UnitChunkList units;
RDIM_SrcFileChunkList src_files;
RDIM_LineTableChunkList line_tables;
RDIM_LineTable **units_first_inline_site_line_tables;
};
@@ -334,11 +384,26 @@ ASYNC_WORK_DEF(p2r_c13_stream_parse_work);
ASYNC_WORK_DEF(p2r_comp_unit_parse_work);
ASYNC_WORK_DEF(p2r_comp_unit_contributions_parse_work);
////////////////////////////////
//~ rjf: Unit Source File Gathering Tasks
ASYNC_WORK_DEF(p2r_gather_unit_src_file_work);
////////////////////////////////
//~ rjf: Unit Conversion Tasks
ASYNC_WORK_DEF(p2r_unit_convert_work);
////////////////////////////////
//~ rjf: Units Conversion Tasks
ASYNC_WORK_DEF(p2r_units_convert_work);
////////////////////////////////
//~ rjf: Source File Sequence Equipping Task
ASYNC_WORK_DEF(p2r_src_file_seq_equip_work);
////////////////////////////////
//~ rjf: Link Name Map Building Tasks