diff --git a/src/lib_rdi_make/rdi_make.c b/src/lib_rdi_make/rdi_make.c index 335344e5..68264120 100644 --- a/src/lib_rdi_make/rdi_make.c +++ b/src/lib_rdi_make/rdi_make.c @@ -646,6 +646,8 @@ rdim_src_file_chunk_list_concat_in_place(RDIM_SrcFileChunkList *dst, RDIM_SrcFil dst->last = to_push->last; dst->chunk_count += to_push->chunk_count; dst->total_count += to_push->total_count; + dst->source_line_map_count += to_push->source_line_map_count; + dst->total_line_count += to_push->total_line_count; } else if(dst->first == 0) { @@ -714,7 +716,9 @@ rdim_line_table_chunk_list_concat_in_place(RDIM_LineTableChunkList *dst, RDIM_Li dst->last = to_push->last; dst->chunk_count += to_push->chunk_count; dst->total_count += to_push->total_count; + dst->total_seq_count += to_push->total_seq_count; dst->total_line_count += to_push->total_line_count; + dst->total_col_count += to_push->total_col_count; } else if(dst->first == 0) { diff --git a/src/rdi_from_pdb/rdi_from_pdb.c b/src/rdi_from_pdb/rdi_from_pdb.c index 41ba890e..2c3b9047 100644 --- a/src/rdi_from_pdb/rdi_from_pdb.c +++ b/src/rdi_from_pdb/rdi_from_pdb.c @@ -577,6 +577,624 @@ ASYNC_WORK_DEF(p2r_comp_unit_contributions_parse_work) return out; } +//////////////////////////////// +//~ rjf: Unit Source File Gathering Tasks + +ASYNC_WORK_DEF(p2r_gather_unit_src_file_work) +{ + ProfBeginFunction(); + Arena *arena = p2r_state->work_thread_arenas[thread_idx]; + Temp scratch = scratch_begin(&arena, 1); + P2R_GatherUnitSrcFilesIn *in = (P2R_GatherUnitSrcFilesIn *)input; + P2R_GatherUnitSrcFilesOut *out = push_array(arena, P2R_GatherUnitSrcFilesOut, 1); + PDB_CompUnit *pdb_unit = in->comp_unit; + CV_SymParsed *pdb_unit_sym = in->comp_unit_syms; + CV_C13Parsed *pdb_unit_c13 = in->comp_unit_c13s; + CV_RecRange *rec_ranges_first = pdb_unit_sym->sym_ranges.ranges; + CV_RecRange *rec_ranges_opl = rec_ranges_first+pdb_unit_sym->sym_ranges.count; + String8List src_file_paths = {0}; + { + //- rjf: build local hash table to dedup files within this unit + U64 hit_path_slots_count = 4096; + String8Node **hit_path_slots = push_array(scratch.arena, String8Node *, hit_path_slots_count); + + //- rjf: produce obj name/path + String8 obj_name = pdb_unit->obj_name; + if(str8_match(obj_name, str8_lit("* Linker *"), 0) || + str8_match(obj_name, str8_lit("Import:"), StringMatchFlag_RightSideSloppy)) + { + MemoryZeroStruct(&obj_name); + } + String8 obj_folder_path = lower_from_str8(scratch.arena, str8_chop_last_slash(obj_name)); + + //- rjf: find all files in this unit's (non-inline) line info + ProfScope("find all files in this unit's (non-inline) line info") + for(CV_C13SubSectionNode *node = pdb_unit_c13->first_sub_section; + node != 0; + node = node->next) + { + if(node->kind == CV_C13SubSectionKind_Lines) + { + for(CV_C13LinesParsedNode *lines_n = node->lines_first; + lines_n != 0; + lines_n = lines_n->next) + { + // rjf: file name -> normalized file path + String8 file_path = lines_n->v.file_name; + String8 file_path_normalized = lower_from_str8(scratch.arena, str8_skip_chop_whitespace(file_path)); + { + PathStyle file_path_normalized_style = path_style_from_str8(file_path_normalized); + String8List file_path_normalized_parts = str8_split_path(scratch.arena, file_path_normalized); + if(file_path_normalized_style == PathStyle_Relative) + { + String8List obj_folder_path_parts = str8_split_path(scratch.arena, obj_folder_path); + str8_list_concat_in_place(&obj_folder_path_parts, &file_path_normalized_parts); + file_path_normalized_parts = obj_folder_path_parts; + file_path_normalized_style = path_style_from_str8(obj_folder_path); + } + str8_path_list_resolve_dots_in_place(&file_path_normalized_parts, file_path_normalized_style); + file_path_normalized = str8_path_list_join_by_style(scratch.arena, &file_path_normalized_parts, file_path_normalized_style); + } + + // rjf: normalized file path -> source file node + U64 file_path_normalized_hash = rdi_hash(file_path_normalized.str, file_path_normalized.size); + U64 hit_path_slot = file_path_normalized_hash%hit_path_slots_count; + String8Node *hit_path_node = 0; + for(String8Node *n = hit_path_slots[hit_path_slot]; n != 0; n = n->next) + { + if(str8_match(n->string, file_path_normalized, 0)) + { + hit_path_node = n; + break; + } + } + if(hit_path_node == 0) + { + hit_path_node = push_array(scratch.arena, String8Node, 1); + SLLStackPush(hit_path_slots[hit_path_slot], hit_path_node); + hit_path_node->string = file_path_normalized; + str8_list_push(scratch.arena, &src_file_paths, push_str8_copy(arena, file_path_normalized)); + } + } + } + } + + //- rjf: find all files in unit's inline line info + ProfScope("find all files in unit's inline line info") + { + U64 base_voff = 0; + for(CV_RecRange *rec_range = rec_ranges_first; + rec_range < rec_ranges_opl; + rec_range += 1) + { + //- rjf: rec range -> symbol info range + U64 sym_off_first = rec_range->off + 2; + U64 sym_off_opl = rec_range->off + rec_range->hdr.size; + + //- rjf: skip invalid ranges + if(sym_off_opl > pdb_unit_sym->data.size || sym_off_first > pdb_unit_sym->data.size || sym_off_first > sym_off_opl) + { + continue; + } + + //- rjf: unpack symbol info + CV_SymKind kind = rec_range->hdr.kind; + U64 sym_header_struct_size = cv_header_struct_size_from_sym_kind(kind); + void *sym_header_struct_base = pdb_unit_sym->data.str + sym_off_first; + void *sym_data_opl = pdb_unit_sym->data.str + sym_off_opl; + + //- rjf: skip bad sizes + if(sym_off_first + sym_header_struct_size > sym_off_opl) + { + continue; + } + + //- rjf: process symbol + switch(kind) + { + default:{}break; + + //- rjf: LPROC32/GPROC32 (gather base address) + case CV_SymKind_LPROC32: + case CV_SymKind_GPROC32: + { + CV_SymProc32 *proc32 = (CV_SymProc32 *)sym_header_struct_base; + COFF_SectionHeader *section = (0 < proc32->sec && proc32->sec <= in->coff_sections.count) ? &in->coff_sections.v[proc32->sec-1] : 0; + if(section != 0) + { + base_voff = section->voff + proc32->off; + } + }break; + + //- rjf: INLINESITE + case CV_SymKind_INLINESITE: + { + // rjf: unpack sym + CV_SymInlineSite *sym = (CV_SymInlineSite *)sym_header_struct_base; + String8 binary_annots = str8((U8 *)(sym+1), rec_range->hdr.size - sizeof(rec_range->hdr.kind) - sizeof(*sym)); + + // rjf: map inlinee -> parsed cv c13 inlinee line info + CV_C13InlineeLinesParsed *inlinee_lines_parsed = 0; + { + U64 hash = cv_hash_from_item_id(sym->inlinee); + U64 slot_idx = hash%pdb_unit_c13->inlinee_lines_parsed_slots_count; + for(CV_C13InlineeLinesParsedNode *n = pdb_unit_c13->inlinee_lines_parsed_slots[slot_idx]; n != 0; n = n->hash_next) + { + if(n->v.inlinee == sym->inlinee) + { + inlinee_lines_parsed = &n->v; + break; + } + } + } + + // rjf: build line table, fill with parsed binary annotations + if(inlinee_lines_parsed != 0) + { + // rjf: grab checksums sub-section + CV_C13SubSectionNode *file_chksms = pdb_unit_c13->file_chksms_sub_section; + + // rjf: gathered lines + U32 last_file_off = max_U32; + U32 curr_file_off = max_U32; + U64 line_count = 0; + CV_C13InlineSiteDecoder decoder = cv_c13_inline_site_decoder_init(inlinee_lines_parsed->file_off, inlinee_lines_parsed->first_source_ln, base_voff); + for(;;) + { + // rjf: step & update + CV_C13InlineSiteDecoderStep step = cv_c13_inline_site_decoder_step(&decoder, binary_annots); + if(step.flags & CV_C13InlineSiteDecoderStepFlag_EmitFile) + { + last_file_off = curr_file_off; + curr_file_off = step.file_off; + } + if(step.flags == 0 && line_count > 0) + { + last_file_off = curr_file_off; + curr_file_off = max_U32; + } + + // rjf: file updated -> gather new file name + if(last_file_off != max_U32 && last_file_off != curr_file_off) + { + String8 seq_file_name = {0}; + if(last_file_off + sizeof(CV_C13Checksum) <= file_chksms->size) + { + CV_C13Checksum *checksum = (CV_C13Checksum*)(pdb_unit_c13->data.str + file_chksms->off + last_file_off); + U32 name_off = checksum->name_off; + seq_file_name = pdb_strtbl_string_from_off(in->pdb_strtbl, name_off); + } + + // rjf: file name -> normalized file path + String8 file_path = seq_file_name; + String8 file_path_normalized = lower_from_str8(scratch.arena, str8_skip_chop_whitespace(file_path)); + { + PathStyle file_path_normalized_style = path_style_from_str8(file_path_normalized); + String8List file_path_normalized_parts = str8_split_path(scratch.arena, file_path_normalized); + if(file_path_normalized_style == PathStyle_Relative) + { + String8List obj_folder_path_parts = str8_split_path(scratch.arena, obj_folder_path); + str8_list_concat_in_place(&obj_folder_path_parts, &file_path_normalized_parts); + file_path_normalized_parts = obj_folder_path_parts; + file_path_normalized_style = path_style_from_str8(obj_folder_path); + } + str8_path_list_resolve_dots_in_place(&file_path_normalized_parts, file_path_normalized_style); + file_path_normalized = str8_path_list_join_by_style(scratch.arena, &file_path_normalized_parts, file_path_normalized_style); + } + + // rjf: normalized file path -> source file node + U64 file_path_normalized_hash = rdi_hash(file_path_normalized.str, file_path_normalized.size); + U64 hit_path_slot = file_path_normalized_hash%hit_path_slots_count; + String8Node *hit_path_node = 0; + for(String8Node *n = hit_path_slots[hit_path_slot]; n != 0; n = n->next) + { + if(str8_match(n->string, file_path_normalized, 0)) + { + hit_path_node = n; + break; + } + } + if(hit_path_node == 0) + { + hit_path_node = push_array(scratch.arena, String8Node, 1); + SLLStackPush(hit_path_slots[hit_path_slot], hit_path_node); + hit_path_node->string = file_path_normalized; + str8_list_push(scratch.arena, &src_file_paths, push_str8_copy(arena, file_path_normalized)); + } + line_count = 0; + } + + // rjf: count lines + if(step.flags & CV_C13InlineSiteDecoderStepFlag_EmitLine) + { + line_count += 1; + } + + // rjf: no more flags -> done + if(step.flags == 0) + { + break; + } + } + } + }break; + } + } + } + } + out->src_file_paths = str8_array_from_list(arena, &src_file_paths); + scratch_end(scratch); + ProfEnd(); + return out; +} + +//////////////////////////////// +//~ rjf: Unit Conversion Tasks + +ASYNC_WORK_DEF(p2r_unit_convert_work) +{ + ProfBeginFunction(); + Arena *arena = p2r_state->work_thread_arenas[thread_idx]; + Temp scratch = scratch_begin(&arena, 1); + P2R_UnitConvertIn *in = (P2R_UnitConvertIn *)input; + P2R_UnitConvertOut *out = push_array(arena, P2R_UnitConvertOut, 1); + + //////////////////////////// + //- rjf: pass 1: build per-unit info & per-unit line table + // + ProfScope("pass 1: build per-unit info & per-unit line table") + { + PDB_CompUnit *pdb_unit = in->comp_unit; + CV_SymParsed *pdb_unit_sym = in->comp_unit_syms; + CV_C13Parsed *pdb_unit_c13 = in->comp_unit_c13s; + + //- rjf: produce unit name + String8 unit_name = pdb_unit->obj_name; + if(unit_name.size != 0) + { + String8 unit_name_past_last_slash = str8_skip_last_slash(unit_name); + if(unit_name_past_last_slash.size != 0) + { + unit_name = unit_name_past_last_slash; + } + } + + //- rjf: produce obj name/path + String8 obj_name = pdb_unit->obj_name; + if(str8_match(obj_name, str8_lit("* Linker *"), 0) || + str8_match(obj_name, str8_lit("Import:"), StringMatchFlag_RightSideSloppy)) + { + MemoryZeroStruct(&obj_name); + } + String8 obj_folder_path = lower_from_str8(scratch.arena, str8_chop_last_slash(obj_name)); + + //- rjf: build this unit's line table, fill out primary line info (inline info added after) + RDIM_LineTable *line_table = 0; + for(CV_C13SubSectionNode *node = pdb_unit_c13->first_sub_section; + node != 0; + node = node->next) + { + if(node->kind == CV_C13SubSectionKind_Lines) + { + for(CV_C13LinesParsedNode *lines_n = node->lines_first; + lines_n != 0; + lines_n = lines_n->next) + { + CV_C13LinesParsed *lines = &lines_n->v; + + // rjf: file name -> normalized file path + String8 file_path = lines->file_name; + String8 file_path_normalized = lower_from_str8(scratch.arena, str8_skip_chop_whitespace(file_path)); + { + PathStyle file_path_normalized_style = path_style_from_str8(file_path_normalized); + String8List file_path_normalized_parts = str8_split_path(scratch.arena, file_path_normalized); + if(file_path_normalized_style == PathStyle_Relative) + { + String8List obj_folder_path_parts = str8_split_path(scratch.arena, obj_folder_path); + str8_list_concat_in_place(&obj_folder_path_parts, &file_path_normalized_parts); + file_path_normalized_parts = obj_folder_path_parts; + file_path_normalized_style = path_style_from_str8(obj_folder_path); + } + str8_path_list_resolve_dots_in_place(&file_path_normalized_parts, file_path_normalized_style); + file_path_normalized = str8_path_list_join_by_style(scratch.arena, &file_path_normalized_parts, file_path_normalized_style); + } + + // rjf: normalized file path -> source file node + U64 file_path_normalized_hash = rdi_hash(file_path_normalized.str, file_path_normalized.size); + U64 src_file_slot = file_path_normalized_hash%in->src_file_map->slots_count; + P2R_SrcFileNode *src_file_node = 0; + if(lines->line_count != 0) + { + for(P2R_SrcFileNode *n = in->src_file_map->slots[src_file_slot]; n != 0; n = n->next) + { + if(str8_match(n->src_file->normal_full_path, file_path_normalized, 0)) + { + src_file_node = n; + break; + } + } + } + + // rjf: push sequence into both line table & source file's line map + if(src_file_node != 0) + { + if(line_table == 0) + { + line_table = rdim_line_table_chunk_list_push(arena, &out->line_tables, 256); + } + RDIM_LineSequence *seq = rdim_line_table_push_sequence(arena, &out->line_tables, line_table, src_file_node->src_file, lines->voffs, lines->line_nums, lines->col_nums, lines->line_count); + } + } + } + } + + //- rjf: build unit + RDIM_Unit *dst_unit = rdim_unit_chunk_list_push(arena, &out->units, 1); + dst_unit->unit_name = unit_name; + dst_unit->compiler_name = pdb_unit_sym->info.compiler_name; + dst_unit->object_file = obj_name; + dst_unit->archive_file = pdb_unit->group_name; + dst_unit->language = p2r_rdi_language_from_cv_language(pdb_unit_sym->info.language); + dst_unit->line_table = line_table; + } + + //////////////////////////// + //- rjf: pass 2: build per-unit voff ranges from comp unit contributions table + // + PDB_CompUnitContribution *contrib_ptr = in->comp_unit_contributions->contributions; + PDB_CompUnitContribution *contrib_opl = contrib_ptr + in->comp_unit_contributions->count; + ProfScope("pass 2: build per-unit voff ranges from comp unit contributions table") + for(;contrib_ptr < contrib_opl; contrib_ptr += 1) + { + if(contrib_ptr->mod == in->comp_unit_idx) + { + RDIM_Unit *unit = &out->units.first->v[0]; + RDIM_Rng1U64 range = {contrib_ptr->voff_first, contrib_ptr->voff_opl}; + rdim_rng1u64_list_push(arena, &unit->voff_ranges, range); + } + } + + //////////////////////////// + //- rjf: pass 3: parse all inlinee line tables + // + ProfScope("pass 3: parse all inlinee line tables") + { + //- rjf: unpack unit + PDB_CompUnit *pdb_unit = in->comp_unit; + CV_SymParsed *pdb_unit_sym = in->comp_unit_syms; + CV_C13Parsed *pdb_unit_c13 = in->comp_unit_c13s; + CV_RecRange *rec_ranges_first = pdb_unit_sym->sym_ranges.ranges; + CV_RecRange *rec_ranges_opl = rec_ranges_first+pdb_unit_sym->sym_ranges.count; + + //- rjf: produce obj name/path + String8 obj_name = pdb_unit->obj_name; + if(str8_match(obj_name, str8_lit("* Linker *"), 0) || + str8_match(obj_name, str8_lit("Import:"), StringMatchFlag_RightSideSloppy)) + { + MemoryZeroStruct(&obj_name); + } + String8 obj_folder_path = lower_from_str8(scratch.arena, str8_chop_last_slash(obj_name)); + + //- rjf: parse inlinee line tables + U64 base_voff = 0; + for(CV_RecRange *rec_range = rec_ranges_first; + rec_range < rec_ranges_opl; + rec_range += 1) + { + //- rjf: rec range -> symbol info range + U64 sym_off_first = rec_range->off + 2; + U64 sym_off_opl = rec_range->off + rec_range->hdr.size; + + //- rjf: skip invalid ranges + if(sym_off_opl > pdb_unit_sym->data.size || sym_off_first > pdb_unit_sym->data.size || sym_off_first > sym_off_opl) + { + continue; + } + + //- rjf: unpack symbol info + CV_SymKind kind = rec_range->hdr.kind; + U64 sym_header_struct_size = cv_header_struct_size_from_sym_kind(kind); + void *sym_header_struct_base = pdb_unit_sym->data.str + sym_off_first; + void *sym_data_opl = pdb_unit_sym->data.str + sym_off_opl; + + //- rjf: skip bad sizes + if(sym_off_first + sym_header_struct_size > sym_off_opl) + { + continue; + } + + //- rjf: process symbol + switch(kind) + { + default:{}break; + + //- rjf: LPROC32/GPROC32 (gather base address) + case CV_SymKind_LPROC32: + case CV_SymKind_GPROC32: + { + CV_SymProc32 *proc32 = (CV_SymProc32 *)sym_header_struct_base; + COFF_SectionHeader *section = (0 < proc32->sec && proc32->sec <= in->coff_sections.count) ? &in->coff_sections.v[proc32->sec-1] : 0; + if(section != 0) + { + base_voff = section->voff + proc32->off; + } + }break; + + //- rjf: INLINESITE + case CV_SymKind_INLINESITE: + { + // rjf: unpack sym + CV_SymInlineSite *sym = (CV_SymInlineSite *)sym_header_struct_base; + String8 binary_annots = str8((U8 *)(sym+1), rec_range->hdr.size - sizeof(rec_range->hdr.kind) - sizeof(*sym)); + + // rjf: map inlinee -> parsed cv c13 inlinee line info + CV_C13InlineeLinesParsed *inlinee_lines_parsed = 0; + { + U64 hash = cv_hash_from_item_id(sym->inlinee); + U64 slot_idx = hash%pdb_unit_c13->inlinee_lines_parsed_slots_count; + for(CV_C13InlineeLinesParsedNode *n = pdb_unit_c13->inlinee_lines_parsed_slots[slot_idx]; n != 0; n = n->hash_next) + { + if(n->v.inlinee == sym->inlinee) + { + inlinee_lines_parsed = &n->v; + break; + } + } + } + + // rjf: build line table, fill with parsed binary annotations + if(inlinee_lines_parsed != 0) + { + // rjf: grab checksums sub-section + CV_C13SubSectionNode *file_chksms = pdb_unit_c13->file_chksms_sub_section; + + // rjf: gathered lines + typedef struct LineChunk LineChunk; + struct LineChunk + { + LineChunk *next; + U64 cap; + U64 count; + U64 *voffs; // [line_count + 1] (sorted) + U32 *line_nums; // [line_count] + U16 *col_nums; // [2*line_count] + }; + LineChunk *first_line_chunk = 0; + LineChunk *last_line_chunk = 0; + U64 total_line_chunk_line_count = 0; + U32 last_file_off = max_U32; + U32 curr_file_off = max_U32; + RDIM_LineTable* line_table = 0; + + CV_C13InlineSiteDecoder decoder = cv_c13_inline_site_decoder_init(inlinee_lines_parsed->file_off, inlinee_lines_parsed->first_source_ln, base_voff); + for(;;) + { + // rjf: step & update + CV_C13InlineSiteDecoderStep step = cv_c13_inline_site_decoder_step(&decoder, binary_annots); + if(step.flags & CV_C13InlineSiteDecoderStepFlag_EmitFile) + { + last_file_off = curr_file_off; + curr_file_off = step.file_off; + } + if(step.flags == 0 && total_line_chunk_line_count > 0) + { + last_file_off = curr_file_off; + curr_file_off = max_U32; + } + + // rjf: file updated -> push line chunks gathered for this file + if(last_file_off != max_U32 && last_file_off != curr_file_off) + { + String8 seq_file_name = {0}; + if(last_file_off + sizeof(CV_C13Checksum) <= file_chksms->size) + { + CV_C13Checksum *checksum = (CV_C13Checksum*)(pdb_unit_c13->data.str + file_chksms->off + last_file_off); + U32 name_off = checksum->name_off; + seq_file_name = pdb_strtbl_string_from_off(in->pdb_strtbl, name_off); + } + + // rjf: file name -> normalized file path + String8 file_path = seq_file_name; + String8 file_path_normalized = lower_from_str8(scratch.arena, str8_skip_chop_whitespace(file_path)); + { + PathStyle file_path_normalized_style = path_style_from_str8(file_path_normalized); + String8List file_path_normalized_parts = str8_split_path(scratch.arena, file_path_normalized); + if(file_path_normalized_style == PathStyle_Relative) + { + String8List obj_folder_path_parts = str8_split_path(scratch.arena, obj_folder_path); + str8_list_concat_in_place(&obj_folder_path_parts, &file_path_normalized_parts); + file_path_normalized_parts = obj_folder_path_parts; + file_path_normalized_style = path_style_from_str8(obj_folder_path); + } + str8_path_list_resolve_dots_in_place(&file_path_normalized_parts, file_path_normalized_style); + file_path_normalized = str8_path_list_join_by_style(scratch.arena, &file_path_normalized_parts, file_path_normalized_style); + } + + // rjf: normalized file path -> source file node + U64 file_path_normalized_hash = rdi_hash(file_path_normalized.str, file_path_normalized.size); + U64 src_file_slot = file_path_normalized_hash%in->src_file_map->slots_count; + P2R_SrcFileNode *src_file_node = 0; + for(P2R_SrcFileNode *n = in->src_file_map->slots[src_file_slot]; n != 0; n = n->next) + { + if(str8_match(n->src_file->normal_full_path, file_path_normalized, 0)) + { + src_file_node = n; + break; + } + } + + // rjf: gather all lines + RDI_U64 *voffs = 0; + RDI_U32 *line_nums = 0; + RDI_U64 line_count = 0; + if(src_file_node != 0) + { + voffs = push_array_no_zero(arena, RDI_U64, total_line_chunk_line_count+1); + line_nums = push_array_no_zero(arena, RDI_U32, total_line_chunk_line_count); + line_count = total_line_chunk_line_count; + U64 dst_idx = 0; + for(LineChunk *chunk = first_line_chunk; chunk != 0; chunk = chunk->next) + { + MemoryCopy(voffs+dst_idx, chunk->voffs, sizeof(U64)*(chunk->count+1)); + MemoryCopy(line_nums+dst_idx, chunk->line_nums, sizeof(U32)*chunk->count); + dst_idx += chunk->count; + } + } + + // rjf: push + if(line_count != 0) + { + if(line_table == 0) + { + line_table = rdim_line_table_chunk_list_push(arena, &out->line_tables, 256); + if(out->unit_first_inline_site_line_table == 0) + { + out->unit_first_inline_site_line_table = line_table; + } + } + rdim_line_table_push_sequence(arena, &out->line_tables, line_table, src_file_node->src_file, voffs, line_nums, 0, line_count); + } + + // rjf: clear line chunks for subsequent sequences + first_line_chunk = last_line_chunk = 0; + total_line_chunk_line_count = 0; + } + + // rjf: new line -> emit to chunk + if(step.flags & CV_C13InlineSiteDecoderStepFlag_EmitLine) + { + LineChunk *chunk = last_line_chunk; + if(chunk == 0 || chunk->count+1 >= chunk->cap) + { + chunk = push_array(scratch.arena, LineChunk, 1); + SLLQueuePush(first_line_chunk, last_line_chunk, chunk); + chunk->cap = 8; + chunk->voffs = push_array_no_zero(scratch.arena, U64, chunk->cap); + chunk->line_nums = push_array_no_zero(scratch.arena, U32, chunk->cap); + } + chunk->voffs[chunk->count] = step.line_voff; + chunk->voffs[chunk->count+1] = step.line_voff_end; + chunk->line_nums[chunk->count] = step.ln; + chunk->count += 1; + total_line_chunk_line_count += 1; + } + + // rjf: no more flags -> done + if(step.flags == 0) + { + break; + } + } + } + }break; + } + } + } + scratch_end(scratch); + ProfEnd(); + return out; +} + //////////////////////////////// //~ rjf: Unit Conversion Tasks @@ -585,15 +1203,12 @@ ASYNC_WORK_DEF(p2r_units_convert_work) ProfBeginFunction(); Arena *arena = p2r_state->work_thread_arenas[thread_idx]; Temp scratch = scratch_begin(&arena, 1); - P2R_UnitConvertIn *in = (P2R_UnitConvertIn *)input; - P2R_UnitConvertOut *out = push_array(arena, P2R_UnitConvertOut, 1); + P2R_UnitsConvertIn *in = (P2R_UnitsConvertIn *)input; + P2R_UnitsConvertOut *out = push_array(arena, P2R_UnitsConvertOut, 1); ProfScope("build units, initial src file map, & collect unit source files") if(in->comp_units != 0) { U64 units_chunk_cap = in->comp_units->count; - P2R_SrcFileMap src_file_map = {0}; - src_file_map.slots_count = 65536; - src_file_map.slots = push_array(scratch.arena, P2R_SrcFileNode *, src_file_map.slots_count); //////////////////////////// //- rjf: pass 1: build per-unit info & per-unit line tables @@ -658,33 +1273,28 @@ ASYNC_WORK_DEF(p2r_units_convert_work) // rjf: normalized file path -> source file node U64 file_path_normalized_hash = rdi_hash(file_path_normalized.str, file_path_normalized.size); - U64 src_file_slot = file_path_normalized_hash%src_file_map.slots_count; + U64 src_file_slot = file_path_normalized_hash%in->src_file_map->slots_count; P2R_SrcFileNode *src_file_node = 0; - for(P2R_SrcFileNode *n = src_file_map.slots[src_file_slot]; n != 0; n = n->next) + if(lines->line_count != 0) { - if(str8_match(n->src_file->normal_full_path, file_path_normalized, 0)) + for(P2R_SrcFileNode *n = in->src_file_map->slots[src_file_slot]; n != 0; n = n->next) { - src_file_node = n; - break; + if(str8_match(n->src_file->normal_full_path, file_path_normalized, 0)) + { + src_file_node = n; + break; + } } } - if(src_file_node == 0) - { - src_file_node = push_array(scratch.arena, P2R_SrcFileNode, 1); - SLLStackPush(src_file_map.slots[src_file_slot], src_file_node); - src_file_node->src_file = rdim_src_file_chunk_list_push(arena, &out->src_files, 4096); - src_file_node->src_file->normal_full_path = push_str8_copy(arena, file_path_normalized); - } // rjf: push sequence into both line table & source file's line map - if(lines->line_count != 0) + if(src_file_node != 0) { if(line_table == 0) { line_table = rdim_line_table_chunk_list_push(arena, &out->line_tables, 256); } - RDIM_LineSequence *seq = rdim_line_table_push_sequence(arena, &out->line_tables, line_table, src_file_node->src_file, lines->voffs, lines->line_nums, lines->col_nums, lines->line_count); - rdim_src_file_push_line_sequence(arena, &out->src_files, src_file_node->src_file, seq); + rdim_line_table_push_sequence(arena, &out->line_tables, line_table, src_file_node->src_file, lines->voffs, lines->line_nums, lines->col_nums, lines->line_count); } } } @@ -807,7 +1417,6 @@ ASYNC_WORK_DEF(p2r_units_convert_work) } // rjf: build line table, fill with parsed binary annotations - if(inlinee_lines_parsed != 0) { // rjf: grab checksums sub-section @@ -834,8 +1443,8 @@ ASYNC_WORK_DEF(p2r_units_convert_work) CV_C13InlineSiteDecoder decoder = cv_c13_inline_site_decoder_init(inlinee_lines_parsed->file_off, inlinee_lines_parsed->first_source_ln, base_voff); for(;;) { + // rjf: step & update CV_C13InlineSiteDecoderStep step = cv_c13_inline_site_decoder_step(&decoder, binary_annots); - if(step.flags & CV_C13InlineSiteDecoderStepFlag_EmitFile) { last_file_off = curr_file_off; @@ -846,10 +1455,11 @@ ASYNC_WORK_DEF(p2r_units_convert_work) last_file_off = curr_file_off; curr_file_off = max_U32; } - if((last_file_off != max_U32 && last_file_off != curr_file_off)) + + // rjf: file updated -> push line chunks gathered for this file + if(last_file_off != max_U32 && last_file_off != curr_file_off) { String8 seq_file_name = {0}; - if(last_file_off + sizeof(CV_C13Checksum) <= file_chksms->size) { CV_C13Checksum *checksum = (CV_C13Checksum*)(unit_c13->data.str + file_chksms->off + last_file_off); @@ -876,9 +1486,9 @@ ASYNC_WORK_DEF(p2r_units_convert_work) // rjf: normalized file path -> source file node U64 file_path_normalized_hash = rdi_hash(file_path_normalized.str, file_path_normalized.size); - U64 src_file_slot = file_path_normalized_hash%src_file_map.slots_count; + U64 src_file_slot = file_path_normalized_hash%in->src_file_map->slots_count; P2R_SrcFileNode *src_file_node = 0; - for(P2R_SrcFileNode *n = src_file_map.slots[src_file_slot]; n != 0; n = n->next) + for(P2R_SrcFileNode *n = in->src_file_map->slots[src_file_slot]; n != 0; n = n->next) { if(str8_match(n->src_file->normal_full_path, file_path_normalized, 0)) { @@ -886,19 +1496,16 @@ ASYNC_WORK_DEF(p2r_units_convert_work) break; } } - if(src_file_node == 0) - { - src_file_node = push_array(scratch.arena, P2R_SrcFileNode, 1); - SLLStackPush(src_file_map.slots[src_file_slot], src_file_node); - src_file_node->src_file = rdim_src_file_chunk_list_push(arena, &out->src_files, 4096); - src_file_node->src_file->normal_full_path = push_str8_copy(arena, file_path_normalized); - } // rjf: gather all lines - RDI_U64 *voffs = push_array_no_zero(arena, RDI_U64, total_line_chunk_line_count+1); - RDI_U32 *line_nums = push_array_no_zero(arena, RDI_U32, total_line_chunk_line_count); - RDI_U64 line_count = total_line_chunk_line_count; + RDI_U64 *voffs = 0; + RDI_U32 *line_nums = 0; + RDI_U64 line_count = 0; + if(src_file_node != 0) { + voffs = push_array_no_zero(arena, RDI_U64, total_line_chunk_line_count+1); + line_nums = push_array_no_zero(arena, RDI_U32, total_line_chunk_line_count); + line_count = total_line_chunk_line_count; U64 dst_idx = 0; for(LineChunk *chunk = first_line_chunk; chunk != 0; chunk = chunk->next) { @@ -919,8 +1526,7 @@ ASYNC_WORK_DEF(p2r_units_convert_work) out->units_first_inline_site_line_tables[comp_unit_idx] = line_table; } } - RDIM_LineSequence *seq = rdim_line_table_push_sequence(arena, &out->line_tables, line_table, src_file_node->src_file, voffs, line_nums, 0, line_count); - rdim_src_file_push_line_sequence(arena, &out->src_files, src_file_node->src_file, seq); + rdim_line_table_push_sequence(arena, &out->line_tables, line_table, src_file_node->src_file, voffs, line_nums, 0, line_count); } // rjf: clear line chunks for subsequent sequences @@ -928,6 +1534,7 @@ ASYNC_WORK_DEF(p2r_units_convert_work) total_line_chunk_line_count = 0; } + // rjf: new line -> emit to chunk if(step.flags & CV_C13InlineSiteDecoderStepFlag_EmitLine) { LineChunk *chunk = last_line_chunk; @@ -935,7 +1542,7 @@ ASYNC_WORK_DEF(p2r_units_convert_work) { chunk = push_array(scratch.arena, LineChunk, 1); SLLQueuePush(first_line_chunk, last_line_chunk, chunk); - chunk->cap = 256; + chunk->cap = 8; chunk->voffs = push_array_no_zero(scratch.arena, U64, chunk->cap); chunk->line_nums = push_array_no_zero(scratch.arena, U32, chunk->cap); } @@ -946,6 +1553,7 @@ ASYNC_WORK_DEF(p2r_units_convert_work) total_line_chunk_line_count += 1; } + // rjf: no more flags -> done if(step.flags == 0) { break; @@ -962,6 +1570,29 @@ ASYNC_WORK_DEF(p2r_units_convert_work) return out; } +//////////////////////////////// +//~ rjf: Source File Sequence Equipping Task + +ASYNC_WORK_DEF(p2r_src_file_seq_equip_work) +{ + ProfBeginFunction(); + Arena *arena = p2r_state->work_thread_arenas[thread_idx]; + P2R_SrcFileSeqEquipIn *in = (P2R_SrcFileSeqEquipIn *)input; + for(RDIM_LineTableChunkNode *line_table_chunk_n = in->line_tables.first; line_table_chunk_n != 0; line_table_chunk_n = line_table_chunk_n->next) + { + for EachIndex(chunk_line_table_idx, line_table_chunk_n->count) + { + RDIM_LineTable *line_table = &line_table_chunk_n->v[chunk_line_table_idx]; + for(RDIM_LineSequenceNode *s = line_table->first_seq; s != 0; s = s->next) + { + rdim_src_file_push_line_sequence(arena, &in->src_files, s->v.src_file, &s->v); + } + } + } + ProfEnd(); + return 0; +} + //////////////////////////////// //~ rjf: Link Name Map Building Tasks @@ -3321,10 +3952,87 @@ p2r_convert(Arena *arena, P2R_User2Convert *in) } ////////////////////////////////////////////////////////////// - //- rjf: kick off unit conversion & source file collection + //- rjf: gather all source file paths; build nodes // - P2R_UnitConvertIn unit_convert_in = {strtbl, coff_sections, comp_units, comp_unit_contributions, sym_for_unit, c13_for_unit}; + RDIM_SrcFileChunkList all_src_files__sequenceless = {0}; + P2R_SrcFileMap src_file_map = {0}; + ProfScope("gather all source file paths; build nodes") + { + U64 tasks_count = comp_unit_count; + P2R_GatherUnitSrcFilesIn *tasks_inputs = push_array(scratch.arena, P2R_GatherUnitSrcFilesIn, tasks_count); + P2R_GatherUnitSrcFilesOut *tasks_outputs = push_array(scratch.arena, P2R_GatherUnitSrcFilesOut, tasks_count); + ASYNC_Task **tasks = push_array(scratch.arena, ASYNC_Task *, tasks_count); + for EachIndex(idx, tasks_count) + { + tasks_inputs[idx].pdb_strtbl = strtbl; + tasks_inputs[idx].coff_sections = coff_sections; + tasks_inputs[idx].comp_unit = comp_units->units[idx]; + tasks_inputs[idx].comp_unit_syms = sym_for_unit[idx]; + tasks_inputs[idx].comp_unit_c13s = c13_for_unit[idx]; + tasks[idx] = async_task_launch(scratch.arena, p2r_gather_unit_src_file_work, .input = &tasks_inputs[idx]); + } + U64 total_path_count = 0; + for EachIndex(idx, tasks_count) + { + tasks_outputs[idx] = *async_task_join_struct(tasks[idx], P2R_GatherUnitSrcFilesOut); + total_path_count += tasks_outputs[idx].src_file_paths.count; + } + src_file_map.slots_count = total_path_count + total_path_count/2 + 1; + src_file_map.slots = push_array(scratch.arena, P2R_SrcFileNode *, src_file_map.slots_count); + for EachIndex(idx, tasks_count) + { + for EachIndex(path_idx, tasks_outputs[idx].src_file_paths.count) + { + String8 file_path_normalized = tasks_outputs[idx].src_file_paths.v[path_idx]; + U64 file_path_normalized_hash = rdi_hash(file_path_normalized.str, file_path_normalized.size); + U64 src_file_slot = file_path_normalized_hash%src_file_map.slots_count; + P2R_SrcFileNode *src_file_node = 0; + for(P2R_SrcFileNode *n = src_file_map.slots[src_file_slot]; n != 0; n = n->next) + { + if(str8_match(n->src_file->normal_full_path, file_path_normalized, 0)) + { + src_file_node = n; + break; + } + } + if(src_file_node == 0) + { + src_file_node = push_array(scratch.arena, P2R_SrcFileNode, 1); + SLLStackPush(src_file_map.slots[src_file_slot], src_file_node); + src_file_node->src_file = rdim_src_file_chunk_list_push(arena, &all_src_files__sequenceless, total_path_count); + src_file_node->src_file->normal_full_path = push_str8_copy(arena, file_path_normalized); + } + } + } + } + + ////////////////////////////////////////////////////////////// + //- rjf: kick off unit conversion tasks + // + U64 unit_convert_tasks_count = comp_unit_count; + P2R_UnitConvertIn *unit_convert_tasks_ins = push_array(scratch.arena, P2R_UnitConvertIn, unit_convert_tasks_count); + ASYNC_Task **unit_convert_tasks = push_array(scratch.arena, ASYNC_Task *, unit_convert_tasks_count); + for EachIndex(idx, unit_convert_tasks_count) + { + P2R_UnitConvertIn *in = &unit_convert_tasks_ins[idx]; + in->comp_unit_idx = idx; + in->pdb_strtbl = strtbl; + in->coff_sections = coff_sections; + in->comp_unit = comp_units->units[idx]; + in->comp_unit_contributions = comp_unit_contributions; + in->comp_unit_syms = sym_for_unit[idx]; + in->comp_unit_c13s = c13_for_unit[idx]; + in->src_file_map = &src_file_map; + unit_convert_tasks[idx] = async_task_launch(scratch.arena, p2r_unit_convert_work, .input = in); + } + +#if 0 + ////////////////////////////////////////////////////////////// + //- rjf: kick off unit conversion + // + P2R_UnitsConvertIn unit_convert_in = {strtbl, coff_sections, comp_units, comp_unit_contributions, sym_for_unit, c13_for_unit, &src_file_map}; ASYNC_Task *unit_convert_task = async_task_launch(scratch.arena, p2r_units_convert_work, .input = &unit_convert_in); +#endif ////////////////////////////////////////////////////////////// //- rjf: join global sym stream parse @@ -4019,21 +4727,49 @@ p2r_convert(Arena *arena, P2R_User2Convert *in) link_name_map = &link_name_map__in_progress; } + ////////////////////////////////////////////////////////////// + //- rjf: join unit conversion tasks + // + RDIM_UnitChunkList all_units = {0}; + RDIM_LineTableChunkList all_line_tables = {0}; + RDIM_LineTable **units_first_inline_site_line_tables = push_array(arena, RDIM_LineTable *, unit_convert_tasks_count); + ProfScope("join unit conversion & src file tasks") + { + for EachIndex(idx, unit_convert_tasks_count) + { + P2R_UnitConvertOut *out = async_task_join_struct(unit_convert_tasks[idx], P2R_UnitConvertOut); + rdim_unit_chunk_list_concat_in_place(&all_units, &out->units); + rdim_line_table_chunk_list_concat_in_place(&all_line_tables, &out->line_tables); + units_first_inline_site_line_tables[idx] = out->unit_first_inline_site_line_table; + } + } + + ////////////////////////////////////////////////////////////// + //- rjf: kick off source file line sequence equipping task + // + RDIM_SrcFileChunkList all_src_files = {0}; + { + P2R_SrcFileSeqEquipIn in = {all_src_files__sequenceless, all_line_tables}; + ASYNC_Task *task = async_task_launch(scratch.arena, p2r_src_file_seq_equip_work, .input = &in); + async_task_join(task); + all_src_files = in.src_files; + } + +#if 0 ////////////////////////////////////////////////////////////// //- rjf: join unit conversion & src file & line table tasks // RDIM_UnitChunkList all_units = {0}; - RDIM_SrcFileChunkList all_src_files = {0}; RDIM_LineTableChunkList all_line_tables = {0}; RDIM_LineTable **units_first_inline_site_line_tables = 0; ProfScope("join unit conversion & src file tasks") { - P2R_UnitConvertOut *out = async_task_join_struct(unit_convert_task, P2R_UnitConvertOut); + P2R_UnitsConvertOut *out = async_task_join_struct(unit_convert_task, P2R_UnitsConvertOut); all_units = out->units; - all_src_files = out->src_files; all_line_tables = out->line_tables; units_first_inline_site_line_tables = out->units_first_inline_site_line_tables; } +#endif ////////////////////////////////////////////////////////////// //- rjf: produce symbols from all streams diff --git a/src/rdi_from_pdb/rdi_from_pdb.h b/src/rdi_from_pdb/rdi_from_pdb.h index 1994f91f..0b05c99e 100644 --- a/src/rdi_from_pdb/rdi_from_pdb.h +++ b/src/rdi_from_pdb/rdi_from_pdb.h @@ -169,10 +169,60 @@ struct P2R_SrcFileMap U64 slots_count; }; +//- rjf: per-unit source files conversion tasks + +typedef struct P2R_GatherUnitSrcFilesIn P2R_GatherUnitSrcFilesIn; +struct P2R_GatherUnitSrcFilesIn +{ + PDB_Strtbl *pdb_strtbl; + COFF_SectionHeaderArray coff_sections; + PDB_CompUnit *comp_unit; + CV_SymParsed *comp_unit_syms; + CV_C13Parsed *comp_unit_c13s; +}; + +typedef struct P2R_GatherUnitSrcFilesOut P2R_GatherUnitSrcFilesOut; +struct P2R_GatherUnitSrcFilesOut +{ + String8Array src_file_paths; +}; + //- rjf: unit conversion tasks typedef struct P2R_UnitConvertIn P2R_UnitConvertIn; struct P2R_UnitConvertIn +{ + U64 comp_unit_idx; + PDB_Strtbl *pdb_strtbl; + COFF_SectionHeaderArray coff_sections; + PDB_CompUnit *comp_unit; + PDB_CompUnitContributionArray *comp_unit_contributions; + CV_SymParsed *comp_unit_syms; + CV_C13Parsed *comp_unit_c13s; + P2R_SrcFileMap *src_file_map; +}; + +typedef struct P2R_UnitConvertOut P2R_UnitConvertOut; +struct P2R_UnitConvertOut +{ + RDIM_UnitChunkList units; + RDIM_LineTableChunkList line_tables; + RDIM_LineTable *unit_first_inline_site_line_table; +}; + +//- rjf: src file sequence equipping task + +typedef struct P2R_SrcFileSeqEquipIn P2R_SrcFileSeqEquipIn; +struct P2R_SrcFileSeqEquipIn +{ + RDIM_SrcFileChunkList src_files; + RDIM_LineTableChunkList line_tables; +}; + +//- rjf: units conversion tasks + +typedef struct P2R_UnitsConvertIn P2R_UnitsConvertIn; +struct P2R_UnitsConvertIn { PDB_Strtbl *pdb_strtbl; COFF_SectionHeaderArray coff_sections; @@ -180,13 +230,13 @@ struct P2R_UnitConvertIn PDB_CompUnitContributionArray *comp_unit_contributions; CV_SymParsed **comp_unit_syms; CV_C13Parsed **comp_unit_c13s; + P2R_SrcFileMap *src_file_map; }; -typedef struct P2R_UnitConvertOut P2R_UnitConvertOut; -struct P2R_UnitConvertOut +typedef struct P2R_UnitsConvertOut P2R_UnitsConvertOut; +struct P2R_UnitsConvertOut { RDIM_UnitChunkList units; - RDIM_SrcFileChunkList src_files; RDIM_LineTableChunkList line_tables; RDIM_LineTable **units_first_inline_site_line_tables; }; @@ -334,11 +384,26 @@ ASYNC_WORK_DEF(p2r_c13_stream_parse_work); ASYNC_WORK_DEF(p2r_comp_unit_parse_work); ASYNC_WORK_DEF(p2r_comp_unit_contributions_parse_work); +//////////////////////////////// +//~ rjf: Unit Source File Gathering Tasks + +ASYNC_WORK_DEF(p2r_gather_unit_src_file_work); + //////////////////////////////// //~ rjf: Unit Conversion Tasks +ASYNC_WORK_DEF(p2r_unit_convert_work); + +//////////////////////////////// +//~ rjf: Units Conversion Tasks + ASYNC_WORK_DEF(p2r_units_convert_work); +//////////////////////////////// +//~ rjf: Source File Sequence Equipping Task + +ASYNC_WORK_DEF(p2r_src_file_seq_equip_work); + //////////////////////////////// //~ rjf: Link Name Map Building Tasks