From b5f0862a915e2711705b025f1579984d6bc7a87b Mon Sep 17 00:00:00 2001 From: Nikita Smith Date: Tue, 31 Mar 2026 17:53:23 -0700 Subject: [PATCH] fixup file checksums in the MSF streams --- src/linker/codeview_ext/codeview.c | 441 ++++++----------------------- src/linker/codeview_ext/codeview.h | 62 +--- src/linker/lnk.c | 25 +- src/linker/lnk_debug_helper.c | 57 ++-- src/linker/lnk_debug_helper.h | 2 +- src/linker/lnk_debug_info.c | 305 ++++++++++++-------- src/linker/lnk_debug_info.h | 14 +- src/linker/lnk_obj.c | 18 +- src/linker/pdb_ext/msf_builder.c | 40 ++- src/linker/pdb_ext/msf_builder.h | 2 +- 10 files changed, 360 insertions(+), 606 deletions(-) diff --git a/src/linker/codeview_ext/codeview.c b/src/linker/codeview_ext/codeview.c index 9b2a8d93..66435866 100644 --- a/src/linker/codeview_ext/codeview.c +++ b/src/linker/codeview_ext/codeview.c @@ -202,6 +202,7 @@ cv_write_symbol_buf(String8Node *buf, U64 *buf_pos, CV_Symbol *symbol, U64 align CV_SymSize record_size16 = (CV_SymSize)record_size; U64 write_size = 0; + write_size += str8_buffer_write(buf, buf_pos, str8((U8 *)&(CV_SymbolHeader){ .size = record_size16, .kind = symbol->kind }, sizeof(CV_SymbolHeader))); write_size += str8_buffer_write(buf, buf_pos, symbol->data); write_size += str8_buffer_write_zeroes(buf, buf_pos, AlignPadPow2(symbol->data.size, align)); @@ -383,6 +384,43 @@ cv_make_envblock(Arena *arena, String8List string_list) return result; } +internal String8 +cv_make_proc32(Arena *arena, CV_SymProc32 proc, String8 name) +{ + U64 buf_size = sizeof(proc) + name.size + 1; + U8 *buf = push_array(arena, U8, buf_size); + CV_SymProc32 *proc_dst = (CV_SymProc32 *)buf; + MemoryCopy(proc_dst, &proc, sizeof(proc)); + MemoryCopy(proc_dst + 1, name.str, name.size); + MemorySet((U8 *)(proc_dst + 1) + name.size , 0, 1); + String8 result = str8(buf, buf_size); + return result; +} + +internal String8 +cv_make_end(Arena *arena) +{ (void)arena; + return str8_zero(); +} + +internal String8 +cv_make_inline_site(Arena *arena, CV_SymInlineSite inline_site, String8 annots) +{ + U64 buf_size = sizeof(inline_site) + annots.size; + U8 *buf = push_array(arena, U8, buf_size); + CV_SymInlineSite *inline_dst = (CV_SymInlineSite *)buf; + MemoryCopy(inline_dst, &inline_site, sizeof(inline_site)); + MemoryCopy(inline_dst + 1, annots.str, annots.size); + String8 result = str8(buf, buf_size); + return result; +} + +internal String8 +cv_make_inline_site_end(Arena *arena) +{ (void)arena; + return str8_zero(); +} + internal CV_Symbol cv_make_proc_ref(Arena *arena, CV_ModIndex imod, U32 stream_offset, String8 name, B32 is_local) { @@ -428,27 +466,6 @@ cv_make_pub32(Arena *arena, CV_Pub32Flags flags, U32 off, U16 isect, String8 nam return symbol; } -internal CV_SymbolList -cv_make_proc_refs(Arena *arena, CV_ModIndex imod, CV_SymbolList symbol_list) -{ - CV_SymbolList proc_ref_list = {0}; - for (CV_SymbolNode *symbol_node = symbol_list.first; symbol_node != 0; symbol_node = symbol_node->next) { - CV_Symbol *symbol = &symbol_node->data; - if (symbol->kind == CV_SymKind_GPROC32) { - String8 name = cv_name_from_symbol(symbol->kind, symbol->data); - CV_Symbol ref = cv_make_proc_ref(arena, imod, safe_cast_u32(symbol->offset), name, /* is_local: */ 0); - CV_SymbolNode *proc_ref_node = cv_symbol_list_push(arena, &proc_ref_list); - proc_ref_node->data = ref; - } else if (symbol->kind == CV_SymKind_LPROC32) { - String8 name = cv_name_from_symbol(symbol->kind, symbol->data); - CV_Symbol ref = cv_make_proc_ref(arena, imod, safe_cast_u32(symbol->offset), name, /* is_local */ 1); - CV_SymbolNode *proc_ref_node = cv_symbol_list_push(arena, &proc_ref_list); - proc_ref_node->data = ref; - } - } - return proc_ref_list; -} - internal B32 cv_is_lproc(CV_Symbol symbol) { @@ -1198,61 +1215,6 @@ cv_debug_t_is_type_server_ref(CV_DebugT *debug_t) // $$Symbols -internal void -cv_parse_symbol_sub_section_capped(Arena *arena, CV_SymbolList *list, U64 offset_base, String8 data, U64 align, U64 cap) -{ - U64 count = 0; - for (U64 cursor = 0, opl = data.size; cursor < opl && count < cap; count += 1) { - // read symbol header - CV_SymbolHeader header; - cursor += str8_deserial_read_struct(data, cursor, &header); - - // size from header has to be larger than 2 bytes - if (header.size < sizeof(header.kind)) { - Assert(!"TODO: error handle invalid symbol data"); - break; - } - - // is there enough bytes in the range? - U64 symbol_opl = cursor + (header.size - sizeof(header.kind)); - if (symbol_opl > opl) { - Assert(!"TODO: error handle corrupted symbol data"); - break; - } - - // get symbol data - Rng1U64 symbol_data_range = r1u64(cursor, symbol_opl); - String8 symbol_data = str8_substr(data, symbol_data_range); - - // init symbol - CV_SymbolNode *node = cv_symbol_list_push(arena, list); - node->data.offset = offset_base + cursor; - node->data.kind = header.kind; - node->data.data = symbol_data; - - // advance cursor - cursor = symbol_opl; - cursor = AlignPow2(cursor, align); - } -} - -internal void -cv_parse_symbol_sub_section(Arena *arena, CV_SymbolList *list, U64 offset_base, String8 data, U64 align) -{ - cv_parse_symbol_sub_section_capped(arena, list, offset_base, data, align, max_U64); -} - -internal CV_SymbolList -cv_symbol_list_from_data_list(Arena *arena, String8List data_list, U64 align) -{ - CV_SymbolList symbol_list = {0}; - U64 cursor = 0; - for (String8Node *sect = data_list.first; sect != 0; cursor += sect->string.size, sect = sect->next) { - cv_parse_symbol_sub_section(arena, &symbol_list, cursor, sect->string, align); - } - return symbol_list; -} - internal void cv_symbol_list_push_node(CV_SymbolList *list, CV_SymbolNode *node) { @@ -1263,309 +1225,77 @@ cv_symbol_list_push_node(CV_SymbolList *list, CV_SymbolNode *node) } internal CV_SymbolNode * -cv_symbol_list_push(Arena *arena, CV_SymbolList *list) +cv_symbol_list_push(Arena *arena, CV_SymbolList *list, CV_Symbol v) { CV_SymbolNode *node = push_array(arena, CV_SymbolNode, 1); + node->data = v; cv_symbol_list_push_node(list, node); return node; } -internal CV_SymbolNode * -cv_symbol_list_push_data(Arena *arena, CV_SymbolList *list, CV_SymKind kind, String8 data) -{ - CV_SymbolNode *node = cv_symbol_list_push(arena, list); - node->data.kind = kind; - node->data.data = data; - return node; -} - -internal CV_SymbolNode * -cv_symbol_list_push_many(Arena *arena, CV_SymbolList *list, U64 count) -{ - CV_SymbolNode *node_arr = push_array_no_zero(arena, CV_SymbolNode, 1); - for (U64 node_idx = 0; node_idx < count; node_idx += 1) { - cv_symbol_list_push_node(list, &node_arr[node_idx]); - } - return node_arr; -} - -internal void -cv_symbol_list_remove_node(CV_SymbolList *list, CV_SymbolNode *node) -{ - Assert(list->count > 0); - list->count -= 1; - DLLRemove(list->first, list->last, node); -} - -internal void -cv_symbol_list_concat_in_place(CV_SymbolList *list, CV_SymbolList *to_concat) -{ - SLLConcatInPlace(list, to_concat); -} - -internal void -cv_symbol_list_concat_in_place_arr(CV_SymbolList *list, U64 count, CV_SymbolList *to_concat) -{ - SLLConcatInPlaceArray(list, to_concat, count); -} - internal U64 -cv_symbol_list_arr_get_count(U64 count, CV_SymbolList *list_arr) +cv_patch_symbol_tree_offsets(String8List raw_symbols, U64 base_offset, U64 align) { - U64 result = 0; - for (U64 idx = 0; idx < count; idx += 1) { - result += list_arr[idx].count; - } - return result; -} - -internal String8List -cv_write_symbol_list(Arena *arena, CV_SymbolList symbol_list, U64 align) -{ - String8List data_list = {0}; - for (CV_SymbolNode *node = symbol_list.first; node != 0; node = node->next) { - String8 data = cv_data_from_symbol(arena, &node->data, align); - str8_list_push(arena, &data_list, data); - } - return data_list; -} - -internal -THREAD_POOL_TASK_FUNC(cv_symbol_list_syncer) -{ - ProfBeginFunction(); - - CV_SymbolListSyncer *task = raw_task; - - // context shortcuts - Rng1U64 list_range = task->list_range_arr[task_id]; - U64 symbol_base = task->symbol_base_arr[task_id]; - - for (U64 list_idx = list_range.min, symbol_idx = symbol_base; list_idx < list_range.max; list_idx += 1) { - // pick up assigned list - CV_SymbolList list = task->list_arr[list_idx]; - - // fill out assigned range in the symbol array - for (CV_SymbolNode *node = list.first; node != 0; node = node->next, symbol_idx += 1) { - task->symbol_arr[symbol_idx] = node; - } - } - - ProfEnd(); -} - -internal CV_SymbolPtrArray -cv_symbol_ptr_array_from_list(Arena *arena, TP_Context *tp, U64 count, CV_SymbolList *list_arr) -{ - ProfBeginFunction(); - Temp scratch = scratch_begin(&arena, 1); - - U64 total_count = cv_symbol_list_arr_get_count(count, list_arr); - - CV_SymbolListSyncer task = {0}; - task.list_arr = list_arr; - task.symbol_arr = push_array_no_zero(arena, CV_SymbolNode *, total_count); - task.symbol_base_arr = push_array_no_zero(scratch.arena, U64, tp->worker_count); - task.list_range_arr = tp_divide_work(scratch.arena, count, tp->worker_count); - - for (U64 thread_idx = 0, symbol_base = 0; thread_idx < tp->worker_count; thread_idx += 1) { - task.symbol_base_arr[thread_idx] = symbol_base; - Rng1U64 range = task.list_range_arr[thread_idx]; - for (U64 list_idx = range.min; list_idx < range.max; list_idx += 1) { - symbol_base += list_arr[list_idx].count; - } - } - - tp_for_parallel(tp, 0, tp->worker_count, cv_symbol_list_syncer, &task); - - CV_SymbolPtrArray result = {0}; - result.count = total_count; - result.v = task.symbol_arr; - - scratch_end(scratch); - ProfEnd(); - return result; -} - -internal CV_Scope * -cv_scope_list_push(Arena *arena, CV_ScopeList *list) -{ - CV_Scope *node = push_array(arena, CV_Scope, 1); - SLLQueuePush(list->first, list->last, node); - return node; -} - -internal CV_SymbolList -cv_global_scope_symbols_from_list(Arena *arena, CV_SymbolList list) -{ - CV_SymbolList gsym_list = {0}; - S64 scope_depth = 0; - for (CV_SymbolNode *symbol_n = list.first; symbol_n != 0; symbol_n = symbol_n->next) { - CV_Symbol symbol = symbol_n->data; - if (cv_is_global_symbol(symbol.kind) && scope_depth == 0) { - cv_symbol_list_push_data(arena, &gsym_list, symbol.kind, symbol.data); - } else if (cv_is_scope_symbol(symbol.kind)) { - scope_depth += 1; - } else if (cv_is_end_symbol(symbol.kind)) { - scope_depth -= 1; - if (scope_depth < 0) { - break; - } - } - } - return gsym_list; -} - -internal CV_ScopeList -cv_symbol_tree_from_symbol_list(Arena *arena, CV_SymbolList list) -{ - Temp scratch = scratch_begin(&arena, 1); - - CV_ScopeList root = {0}; - - // setup root frame - CV_ScopeFrame *stack = push_array(scratch.arena, CV_ScopeFrame, 1); - stack->list = &root; - - for (CV_SymbolNode *symbol_node = list.first; symbol_node != 0; symbol_node = symbol_node->next) { - // store symbol in current scope - CV_Scope *scope = cv_scope_list_push(arena, stack->list); - scope->symbol = symbol_node->data; - - // does this symbol define a new scope? - if (cv_is_scope_symbol(symbol_node->data.kind)) { - CV_ScopeFrame *frame = push_array(scratch.arena, CV_ScopeFrame, 1); - frame->list = push_array(arena, CV_ScopeList, 1); - SLLStackPush(stack, frame); - } - // does this symbol end current scope? - else if (cv_is_end_symbol(symbol_node->data.kind)) { - CV_ScopeFrame *prev_stack_frame = stack->next; - if (prev_stack_frame) { - // set children in parent scope - CV_Scope *parent_scope = prev_stack_frame->list->last; - parent_scope->children = stack->list; - } - - // pop frame - SLLStackPop(stack); - } - } - - scratch_end(scratch); - return root; -} - -internal U64 -cv_patch_symbol_tree_offsets(CV_SymbolList list, U64 base_offset, U64 align) -{ - struct Stack { - struct Stack *next; - CV_Symbol *symbol; - U64 offset; - }; Temp scratch = scratch_begin(0, 0); - struct Stack *stack = 0; - struct Stack *free_list = 0; - U32 cursor = safe_cast_u32(base_offset); - for EachNode(symbol_n, CV_SymbolNode, list.first) { - CV_Symbol symbol = symbol_n->data; - if (cv_is_scope_symbol(symbol.kind)) { + + struct Stack { struct Stack *next; String8Node symbol_buf; U64 symbol_pos; U64 offset; }; + struct Stack *stack = 0, *free_list = 0; + + String8Node buf = *raw_symbols.first; + U64 buf_pos = 0; + U64 symbol_offset = base_offset; + U64 depth = 0; + + for (;;) { + CV_SymbolHeader symbol_header; + if (str8_buffer_read(&buf, &buf_pos, sizeof(symbol_header), &symbol_header) != sizeof(symbol_header)) { break; } + + if (cv_is_scope_symbol(symbol_header.kind)) { // NOTE: We don't patch 'next' offset in PROC symbols because // it's not used by visual studio and MSVC leaves the offsets // zeroed. LLD is on the same page. - Assert(symbol.data.size >= sizeof(U32)*2); + Assert(symbol_header.size >= sizeof(CV_SymKind) + sizeof(U32)*2); // patch parent symbol offset if (stack) { - memory_write32(symbol.data.str, stack->offset); + String8Node temp_buf = buf; + U64 temp_pos = buf_pos; + str8_buffer_write_u32(&temp_buf, &temp_pos, stack->offset); } // reuse/alloc frame - struct Stack *frame; - if (free_list) { - frame = free_list; - SLLStackPop(free_list); - } else { - frame = push_array_no_zero(scratch.arena, struct Stack, 1); - } + struct Stack *frame = free_list; + if (frame) { SLLStackPop(free_list); } + else { frame = push_array_no_zero(scratch.arena, struct Stack, 1); } // push frame to the stack - frame->symbol = &symbol_n->data; - frame->offset = cursor; + frame->symbol_buf = buf; + frame->symbol_pos = buf_pos; + frame->offset = safe_cast_u32(symbol_offset); SLLStackPush(stack, frame); - } else if (cv_is_end_symbol(symbol.kind)) { + + depth += 1; + } else if (cv_is_end_symbol(symbol_header.kind)) { // patch symbol end - U32 *end_off_ptr = (U32 *)stack->symbol->data.str + /* skip parent off */ 1; - memory_write32(end_off_ptr, cursor); + String8Node temp_buf = stack->symbol_buf; + U64 temp_pos = stack->symbol_pos; + str8_buffer_skip(&temp_buf, &temp_pos, sizeof(U32)); // skip parent offset + str8_buffer_write_u32(&temp_buf, &temp_pos, symbol_offset); // recycle frame struct Stack *free_frame = stack; SLLStackPop(stack); SLLStackPush(free_list, free_frame); + + if (depth == 0) { Assert(0 && "malformed symbol stream"); continue; } + depth -= 1; } - // advance cursor - cursor += cv_size_from_symbol(&symbol, align); - } + // advance symbol offset + symbol_offset += sizeof(CV_SymSize) + symbol_header.size; + symbol_offset = AlignPow2(symbol_offset, align); - scratch_end(scratch); - U64 serial_size = cursor - base_offset; - return serial_size; -} - -internal U64 -cv_patch_symbol_tree_offsets_new(String8List raw_symbols, U64 base_offset, U64 align) -{ - Temp scratch = scratch_begin(0, 0); - struct Stack { struct Stack *next; CV_Symbol symbol; U64 offset; }; - struct Stack *stack = 0, *free_list = 0; - U64 symbol_offset = safe_cast_u32(base_offset); - for EachNode(n, String8Node, raw_symbols.first) { - for (U64 cursor = 0, depth = 0; cursor + sizeof(CV_SymbolHeader) <= n->string.size; ) { - CV_Symbol symbol = {0}; - TryReadBreak(cv_read_symbol(n->string, cursor, align, &symbol), cursor); - - if (cv_is_scope_symbol(symbol.kind)) { - // NOTE: We don't patch 'next' offset in PROC symbols because - // it's not used by visual studio and MSVC leaves the offsets - // zeroed. LLD is on the same page. - Assert(symbol.data.size >= sizeof(U32)*2); - - // patch parent symbol offset - if (stack) { - memory_write32(symbol.data.str, stack->offset); - } - - // reuse/alloc frame - struct Stack *frame = free_list; - if (frame) { SLLStackPop(free_list); } - else { frame = push_array_no_zero(scratch.arena, struct Stack, 1); } - - // push frame to the stack - frame->symbol = symbol; - frame->offset = symbol_offset; - SLLStackPush(stack, frame); - - depth += 1; - } else if (cv_is_end_symbol(symbol.kind)) { - // patch symbol end - U32 *end_off_ptr = (U32 *)stack->symbol.data.str + /* skip parent off */ 1; - memory_write32(end_off_ptr, symbol_offset); - - // recycle frame - struct Stack *free_frame = stack; - SLLStackPop(stack); - SLLStackPush(free_list, free_frame); - - if (depth == 0) { Assert(0 && "malformed symbol stream"); goto next_block; } - depth -= 1; - } - - // advance tree offset - symbol_offset += cv_size_from_symbol(&symbol, align); - } - next_block:; + str8_buffer_skip(&buf, &buf_pos, symbol_header.size - sizeof(CV_SymKind)); } scratch_end(scratch); @@ -1618,13 +1348,12 @@ cv_c13_parse_checksum_data_list(Arena *arena, String8List checksum_data_list) internal void cv_c13_patch_string_offsets_in_checksum_list(CV_ChecksumList checksum_list, String8 string_data, U64 string_data_base_offset, CV_StringHashTable string_ht) { - for (CV_ChecksumNode *node = checksum_list.first; node != 0; node = node->next) { - CV_Checksum *checksum = &node->data; - CV_C13Checksum *header = checksum->header; - String8 name = str8_cstring_capped(string_data.str + header->name_off, string_data.str + string_data.size); - CV_StringBucket *bucket = cv_string_hash_table_lookup(string_ht, name); - - U64 name_off64 = string_data_base_offset + bucket->u.offset; + for EachNode(node, CV_ChecksumNode, checksum_list.first) { + CV_Checksum *checksum = &node->data; + CV_C13Checksum *header = checksum->header; + String8 name = str8_cstring_capped(string_data.str + header->name_off, string_data.str + string_data.size); + CV_StringBucket *bucket = cv_string_hash_table_lookup(string_ht, name); + U64 name_off64 = string_data_base_offset + bucket->u.offset; header->name_off = safe_cast_u32(name_off64); } } diff --git a/src/linker/codeview_ext/codeview.h b/src/linker/codeview_ext/codeview.h index 4cc68079..3b477148 100644 --- a/src/linker/codeview_ext/codeview.h +++ b/src/linker/codeview_ext/codeview.h @@ -42,12 +42,6 @@ typedef struct CV_SymbolNode CV_Symbol data; } CV_SymbolNode; -typedef struct CV_SymbolPtrNode -{ - struct CV_SymbolPtrNode *next; - CV_Symbol *data; -} CV_SymbolPtrNode; - typedef struct CV_SymbolList { U64 count; @@ -62,36 +56,6 @@ typedef struct CV_SymbolListArray CV_SymbolList *v; } CV_SymbolListArray; -typedef struct CV_SymbolPtrArray -{ - U64 count; - CV_SymbolNode **v; -} CV_SymbolPtrArray; - -typedef struct CV_Scope -{ - struct CV_ScopeList *children; - struct CV_Scope *next; - struct CV_Scope *prev; - CV_Symbol symbol; -} CV_Scope; - -typedef struct CV_ScopeList -{ - CV_Scope *first; - CV_Scope *last; -} CV_ScopeList; - -typedef struct CV_ScopeFrame -{ - struct CV_ScopeFrame *next; - CV_ScopeList *list; - CV_Scope *curr; - U64 symbol_off; - U32 *parent_off_ptr; - U32 *end_off_ptr; -} CV_ScopeFrame; - //- $$FileChksms typedef struct CV_Checksum @@ -340,14 +304,6 @@ typedef struct CV_StringHashTableResult //////////////////////////////// //~ Task Contexts -typedef struct -{ - CV_SymbolList *list_arr; - Rng1U64 *list_range_arr; - U64 *symbol_base_arr; - CV_SymbolNode **symbol_arr; -} CV_SymbolListSyncer; - typedef struct { CV_DebugS *arr; @@ -401,6 +357,7 @@ internal String8 cv_make_symbol(Arena *arena, CV_SymKind kind, String8 dat internal String8 cv_make_obj_name(Arena *arena, String8 obj_path, U32 sig); internal String8 cv_make_comp3(Arena *arena, CV_Compile3Flags flags, CV_Language lang, CV_Arch arch, U16 ver_fe_major, U16 ver_fe_minor, U16 ver_fe_build, U16 ver_feqfe, U16 ver_major, U16 ver_minor, U16 ver_build, U16 ver_qfe, String8 version_string); internal String8 cv_make_envblock(Arena *arena, String8List string_list); +internal String8 cv_make_end(Arena *arena); internal CV_Symbol cv_make_proc_ref(Arena *arena, CV_ModIndex imod, U32 stream_offset, String8 name, B32 is_local); internal CV_Symbol cv_make_pub32(Arena *arena, CV_Pub32Flags flags, U32 off, U16 isect, String8 name); @@ -448,21 +405,10 @@ internal B32 cv_debug_t_is_type_server_ref(CV_DebugT *debug_t); //~ Sub Section helpers // $$Symbols -internal void cv_parse_symbol_sub_section_capped(Arena *arena, CV_SymbolList *list, U64 offset_base, String8 data, U64 align, U64 cap); -internal void cv_parse_symbol_sub_section(Arena *arena, CV_SymbolList *list, U64 offset_base, String8 data, U64 align); +internal void cv_symbol_list_push_node(CV_SymbolList *list, CV_SymbolNode *node); +internal CV_SymbolNode * cv_symbol_list_push(Arena *arena, CV_SymbolList *list, CV_Symbol v); -internal void cv_symbol_list_push_node(CV_SymbolList *list, CV_SymbolNode *node); -internal CV_SymbolNode * cv_symbol_list_push(Arena *arena, CV_SymbolList *list); -internal CV_SymbolNode * cv_symbol_list_push_data(Arena *arena, CV_SymbolList *list, CV_SymKind kind, String8 data); -internal CV_SymbolNode * cv_symbol_list_push_many(Arena *arena, CV_SymbolList *list, U64 count); -internal void cv_symbol_list_remove_node(CV_SymbolList *list, CV_SymbolNode *node); -internal void cv_symbol_list_concat_in_place(CV_SymbolList *list, CV_SymbolList *to_concat); -internal void cv_symbol_list_concat_in_place_arr(CV_SymbolList *list, U64 count, CV_SymbolList *to_concat); -internal U64 cv_symbol_list_arr_get_count(U64 count, CV_SymbolList *list_arr); - -internal String8List cv_write_symbol_list(Arena *arena, CV_SymbolList symbol_list, U64 align); -internal CV_SymbolList cv_global_scope_symbols_from_list(Arena *arena, CV_SymbolList list); -internal CV_SymbolPtrArray cv_symbol_ptr_array_from_list(Arena *arena, TP_Context *tp, U64 count, CV_SymbolList *symbol_list_arr); +internal U64 cv_patch_symbol_tree_offsets(String8List raw_symbols, U64 base_offset, U64 align); // $$FileChksms #define CV_MAP_STRING_TO_OFFSET_FUNC(name) U64 name(void *ud, String8 string) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index 14920524..68c061bd 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -842,16 +842,11 @@ lnk_make_linker_coff_obj(Arena *arena, String8 debug_symbols = {0}; { - CV_SymbolList symbol_list = { .signature = CV_Signature_C13 }; - - // S_OBJ - String8 obj_data = cv_make_obj_name(scratch.arena, obj_name, 0); - cv_symbol_list_push_data(scratch.arena, &symbol_list, CV_SymKind_OBJNAME, obj_data); - - // S_COMPILE3 - String8 comp3_data = lnk_make_linker_compile3(scratch.arena, machine); - cv_symbol_list_push_data(scratch.arena, &symbol_list, CV_SymKind_COMPILE3, comp3_data); + String8List symbols = {0}; + str8_list_push(scratch.arena, &symbols, cv_make_symbol(scratch.arena, CV_SymKind_OBJNAME, cv_make_obj_name(scratch.arena, obj_name, 0))); + str8_list_push(scratch.arena, &symbols, cv_make_symbol(scratch.arena, CV_SymKind_COMPILE3, lnk_make_linker_compile3(scratch.arena, machine))); + // S_ENVBLOCK String8List env_list = {0}; str8_list_push(scratch.arena, &env_list, str8_lit("cwd")); @@ -864,12 +859,12 @@ lnk_make_linker_coff_obj(Arena *arena, str8_list_push(scratch.arena, &env_list, cmd_line); str8_list_push(scratch.arena, &env_list, str8_lit("")); str8_list_push(scratch.arena, &env_list, str8_lit("")); - cv_symbol_list_push_data(scratch.arena, &symbol_list, CV_SymKind_ENVBLOCK, cv_make_envblock(scratch.arena, env_list)); + str8_list_push(scratch.arena, &symbols, cv_make_symbol(scratch.arena, CV_SymKind_ENVBLOCK, cv_make_envblock(scratch.arena, env_list))); // TODO: emit S_SECTION and S_COFFGROUP // TODO: emit S_TRAMPOLINE - debug_symbols = lnk_make_debug_s(scratch.arena, symbol_list); + debug_symbols = lnk_make_debug_s(scratch.arena, symbols); } String8 obj; @@ -1863,8 +1858,8 @@ lnk_link_image(TP_Context *tp, TP_Arena *arena, LNK_Config *config, LNK_Inputer for (String8Node *dll_name_n = delayed_dll_names.first; dll_name_n != 0; dll_name_n = dll_name_n->next) { PE_MakeImportList *imports = hash_table_search_path_raw(delayed_imports_ht, dll_name_n->string); - String8 import_debug_symbols = lnk_make_dll_import_debug_symbols(scratch.arena, config->machine, dll_name_n->string); - String8 import_obj = pe_make_import_dll_obj_delayed(arena->v[0], time_stamp, config->machine, dll_name_n->string, config->delay_load_helper_name, import_debug_symbols, *imports, emit_biat, emit_uiat); + String8 import_debug_symbols = lnk_make_dll_import_debug_symbols(scratch.arena, config->machine, dll_name_n->string); + String8 import_obj = pe_make_import_dll_obj_delayed(arena->v[0], time_stamp, config->machine, dll_name_n->string, config->delay_load_helper_name, import_debug_symbols, *imports, emit_biat, emit_uiat); lnk_inputer_push_obj(inputer, 0, dll_name_n->string, import_obj); } @@ -1885,8 +1880,8 @@ lnk_link_image(TP_Context *tp, TP_Arena *arena, LNK_Config *config, LNK_Inputer for (String8Node *dll_name_n = static_dll_names.first; dll_name_n != 0; dll_name_n = dll_name_n->next) { PE_MakeImportList *imports = hash_table_search_path_raw(static_imports_ht, dll_name_n->string); - String8 import_debug_symbols = lnk_make_dll_import_debug_symbols(scratch.arena, config->machine, dll_name_n->string); - String8 import_obj = pe_make_import_dll_obj_static(arena->v[0], time_stamp, config->machine, dll_name_n->string, import_debug_symbols, *imports); + String8 import_debug_symbols = lnk_make_dll_import_debug_symbols(scratch.arena, config->machine, dll_name_n->string); + String8 import_obj = pe_make_import_dll_obj_static(arena->v[0], time_stamp, config->machine, dll_name_n->string, import_debug_symbols, *imports); lnk_inputer_push_obj(inputer, 0, dll_name_n->string, import_obj); } diff --git a/src/linker/lnk_debug_helper.c b/src/linker/lnk_debug_helper.c index 7cbfe291..fbf9bb36 100644 --- a/src/linker/lnk_debug_helper.c +++ b/src/linker/lnk_debug_helper.c @@ -4,31 +4,29 @@ internal String8 lnk_make_linker_compile3(Arena *arena, COFF_MachineType machine) { - String8 comp3_data = cv_make_comp3(arena, - 0, - CV_Language_LINK, - cv_arch_from_coff_machine(machine), - /* ver_fe_major */ 0, - /* ver_fe_minor */ 0, - /* ver_fe_build */ 0, - /* ver_feqfe */ 0, - /* ver_major */ 14, - /* ver_minor */ 36, - /* ver_build */ 32537, - /* ver_qfe */ 0, - str8_lit(BUILD_TITLE)); - return comp3_data; + return cv_make_comp3(arena, + 0, + CV_Language_LINK, + cv_arch_from_coff_machine(machine), + /* ver_fe_major */ 0, + /* ver_fe_minor */ 0, + /* ver_fe_build */ 0, + /* ver_feqfe */ 0, + /* ver_major */ 14, + /* ver_minor */ 36, + /* ver_build */ 32537, + /* ver_qfe */ 0, + str8_lit(BUILD_TITLE)); } internal String8 -lnk_make_debug_s(Arena *arena, CV_SymbolList symbol_list) +lnk_make_debug_s(Arena *arena, String8List symbols) { Temp scratch = scratch_begin(&arena, 1); - CV_DebugS debug_s = {0}; - String8List *symbol_list_ptr = cv_sub_section_ptr_from_debug_s(&debug_s, CV_C13SubSectionKind_Symbols); - *symbol_list_ptr = cv_write_symbol_list(scratch.arena, symbol_list, CV_SymbolAlign); + cv_patch_symbol_tree_offsets(symbols, sizeof(CV_Signature), CV_SymbolAlign); + CV_DebugS debug_s = { .data_list[CV_C13SubSectionIdxKind_Symbols] = symbols }; String8List debug_s_data_list = cv_data_from_debug_s_c13(scratch.arena, &debug_s, 1); String8 debug_s_data = str8_list_join(arena, &debug_s_data_list, 0); @@ -40,10 +38,11 @@ internal String8 lnk_make_linker_debug_symbols(Arena *arena, COFF_MachineType machine) { Temp scratch = scratch_begin(&arena, 1); - CV_SymbolList symbol_list = { .signature = CV_Signature_C13 }; - String8 comp3_data = lnk_make_linker_compile3(scratch.arena, machine); - cv_symbol_list_push_data(scratch.arena, &symbol_list, CV_SymKind_COMPILE3, comp3_data); - String8 debug_symbols = lnk_make_debug_s(arena, symbol_list); + + String8List symbols = {0}; + str8_list_push(scratch.arena, &symbols, cv_make_symbol(scratch.arena, CV_SymKind_COMPILE3, lnk_make_linker_compile3(scratch.arena, machine))); + String8 debug_symbols = lnk_make_debug_s(arena, symbols); + scratch_end(scratch); return debug_symbols; } @@ -53,20 +52,14 @@ lnk_make_dll_import_debug_symbols(Arena *arena, COFF_MachineType machine, String { Temp scratch = scratch_begin(&arena,1); - CV_SymbolList symbol_list = { .signature = CV_Signature_C13 }; - - // S_OBJ - String8 obj_data = cv_make_obj_name(scratch.arena, dll_name, 0); - cv_symbol_list_push_data(scratch.arena, &symbol_list, CV_SymKind_OBJNAME, obj_data); - - // S_COMPILE3 - String8 comp3_data = lnk_make_linker_compile3(scratch.arena, machine); - cv_symbol_list_push_data(scratch.arena, &symbol_list, CV_SymKind_COMPILE3, comp3_data); + String8List symbols = {0}; + str8_list_push(scratch.arena, &symbols, cv_make_symbol(scratch.arena, CV_SymKind_OBJNAME, cv_make_obj_name(scratch.arena, dll_name, 0))); + str8_list_push(scratch.arena, &symbols, cv_make_symbol(scratch.arena, CV_SymKind_COMPILE3, lnk_make_linker_compile3(scratch.arena, machine))); // TODO: add thunks // serialize symbols - String8 debug_symbols = lnk_make_debug_s(arena, symbol_list); + String8 debug_symbols = lnk_make_debug_s(arena, symbols); scratch_end(scratch); return debug_symbols; diff --git a/src/linker/lnk_debug_helper.h b/src/linker/lnk_debug_helper.h index dd1df4b3..14904168 100644 --- a/src/linker/lnk_debug_helper.h +++ b/src/linker/lnk_debug_helper.h @@ -4,5 +4,5 @@ #pragma once internal String8 lnk_make_linker_compile3(Arena *arena, COFF_MachineType machine); -internal String8 lnk_make_debug_s(Arena *arena, CV_SymbolList symbol_list); +internal String8 lnk_make_debug_s(Arena *arena, String8List symbols); diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index ae9505a5..9d4696c1 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -15,6 +15,9 @@ lnk_get_huge_arena(void) internal void lnk_discard_cv_debug_info(LNK_CodeViewInput *input, U64 obj_idx) { + // discard types + MemoryZeroStruct(&input->debug_t_arr[obj_idx]); + // discard symbols String8List *symbols_ptr = cv_sub_section_ptr_from_debug_s(&input->debug_s_arr[obj_idx], CV_C13SubSectionKind_Symbols); MemoryZeroStruct(symbols_ptr); @@ -276,6 +279,8 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_IO_Flags io_fla ProfScope("Set up /Zi") { input.obj_to_ts = push_array(tp_arena->v[0], U64, input.count); + MemorySet(input.obj_to_ts, 0xff, input.count * sizeof(input.obj_to_ts[0])); + LNK_TypeServerList ts_list = {0}; HashTable *ts_ht = hash_table_init(scratch.arena, 256); @@ -294,6 +299,11 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_IO_Flags io_fla CV_TypeServerInfo ts_info = cv_type_server_info_from_leaf(leaf); String8 ts_path = lnk_find_first_file(scratch.arena, lib_dir_list, ts_info.name); + if (ts_path.size == 0) { + lnk_discard_cv_debug_info(&input, obj_idx); + continue; + } + // insert new type server LNK_TypeServer *ts = hash_table_search_path_raw(ts_ht, ts_path); if (ts == 0) { @@ -313,6 +323,7 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_IO_Flags io_fla obj_arr[obj_idx], "type server signature conflicts with type server from '%S'", obj_arr[ts->obj_indices.first->data]->path); + lnk_discard_cv_debug_info(&input, obj_idx); continue; } @@ -342,6 +353,7 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_IO_Flags io_fla MemoryCopyTyped(input.debug_s_arr, prev.debug_s_arr, prev.count); MemoryCopyTyped(input.debug_t_arr, prev.debug_t_arr, prev.count); MemoryCopyTyped(input.obj_to_ts, prev.obj_to_ts, prev.count); + MemorySet(input.obj_to_ts + input.obj_count, 0xff, ts_arr.count * sizeof(input.obj_to_ts[0])); input.ts_obj_range = r1u64(prev.count, input.count); @@ -522,10 +534,10 @@ lnk_leaf_ref_from_ti(LNK_CodeViewInput *input, U32 obj_idx, CV_TypeIndexSource s // ti range: external type server U64 ts_idx = input->obj_to_ts[obj_idx]; - if (ts_idx) { - U64 ts_obj_idx = input->ts_obj_range.min + ts_idx; - CV_DebugT *ts_debug_t = input->debug_t_arr + ts_obj_idx; - return (LNK_LeafRef){ ts_obj_idx, cv_leaf_idx_from_ti(ts_debug_t, source, ti) }; + if (ts_idx != max_U64) { + U64 ts_debug_t_idx = input->ts_obj_range.min + ts_idx; + CV_DebugT *ts_debug_t = input->debug_t_arr + ts_debug_t_idx; + return (LNK_LeafRef){ ts_debug_t_idx, cv_leaf_idx_from_ti(ts_debug_t, source, ti) }; } // ti range: internal type server @@ -553,14 +565,10 @@ internal B32 lnk_match_leaf_ref(LNK_CodeViewInput *input, LNK_LeafRef a, LNK_LeafRef b) { B32 is_match = 0; - CV_DebugT *a_debug_t = &input->debug_t_arr[a.obj_idx]; - CV_DebugT *b_debug_t = &input->debug_t_arr[b.obj_idx]; - U64 a_hash = input->debug_h_arr[a.obj_idx].v[a.leaf_idx]; - U64 b_hash = input->debug_h_arr[b.obj_idx].v[b.leaf_idx]; + U64 a_hash = input->debug_h_arr[a.obj_idx].v[a.leaf_idx]; + U64 b_hash = input->debug_h_arr[b.obj_idx].v[b.leaf_idx]; if (a_hash == b_hash) { - CV_Leaf a_leaf = cv_debug_t_get_leaf(a_debug_t, a.leaf_idx); - CV_Leaf b_leaf = cv_debug_t_get_leaf(b_debug_t, b.leaf_idx); - Assert(a_leaf.kind == b_leaf.kind); + Assert(cv_debug_t_get_leaf(&input->debug_t_arr[a.obj_idx], a.leaf_idx).kind == cv_debug_t_get_leaf(&input->debug_t_arr[b.obj_idx], b.leaf_idx).kind); is_match = 1; } return is_match; @@ -633,7 +641,7 @@ lnk_hash_cv_leaf(LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInf U32 leaf_idx = curr_ti - debug_t->ti_ranges[curr_ti_source].min; U8 *leaf_header = debug_t->data.str + debug_t->offsets[leaf_idx]; memory_write16(leaf_header + OffsetOf(CV_LeafHeader, kind), CV_LeafKind_NOTYPE); - memory_write16(leaf_header + OffsetOf(CV_LeafHeader, size), 0); + memory_write16(leaf_header + OffsetOf(CV_LeafHeader, size), sizeof(CV_LeafKind)); // log error Temp scratch = scratch_begin(0,0); @@ -1130,21 +1138,29 @@ lnk_fixup_cv_type_indices(LNK_MergeTypes *ctx, U32 obj_idx, String8 data, CV_Typ // skip basic types if (ti < ctx->input->min_type_indices[n->source]) { continue; } - U64 ts_idx = ctx->input->obj_to_ts[obj_idx]; - CV_DebugT *debug_t = ts_idx ? &ctx->input->debug_t_arr[ctx->input->ts_obj_range.min + ts_idx] : &ctx->input->debug_t_arr[obj_idx]; + CV_DebugT *debug_t; + if (ctx->input->obj_to_ts[obj_idx] != max_U64) { + U64 ts_idx = ctx->input->obj_to_ts[obj_idx]; + U64 debug_t_idx = ctx->input->ts_obj_range.min + ts_idx; + debug_t = ctx->input->debug_t_arr + debug_t_idx; + } else { + debug_t = ctx->input->debug_t_arr + obj_idx; + } CV_TypeIndex final_ti = 0; if (contains_1u64(debug_t->ti_ranges[n->source], ti)) { LNK_LeafRef leaf_ref = lnk_leaf_ref_from_ti(ctx->input, obj_idx, n->source, ti); LNK_LeafHashTable *leaf_ht = &ctx->leaf_ht_arr[n->source]; LNK_LeafRef *final_leaf = lnk_leaf_hash_table_search(leaf_ht, ctx->input, leaf_ref); - U64 final_hash = u64_hash_from_str8(str8_struct(final_leaf)); - final_ti = lnk_assigned_type_ht_search(ctx->assigned_type_caps [n->source], - ctx->assigned_type_hts [n->source], - ctx->min_type_indices [n->source], - ctx->unique_leaf_refs_arr[n->source], - final_leaf, - final_hash); + if (final_leaf) { + U64 final_hash = u64_hash_from_str8(str8_struct(final_leaf)); + final_ti = lnk_assigned_type_ht_search(ctx->assigned_type_caps [n->source], + ctx->assigned_type_hts [n->source], + ctx->min_type_indices [n->source], + ctx->unique_leaf_refs_arr[n->source], + final_leaf, + final_hash); + } } else { Assert(0 && "invalid type index"); } @@ -1323,7 +1339,7 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input) tp_for_parallel(tp, 0, task.indices.count, lnk_hash_debug_t_task, &task); task.indices = input->type_server_indices; - ProfScope("Type Servers [Count: %.*s]", str8_varg(str8_from_count(scratch.arena, task.indices.count))); + ProfScope("Type Servers [Count: %.*s]", str8_varg(str8_from_count(scratch.arena, task.indices.count))) tp_for_parallel(tp, 0, task.indices.count, lnk_hash_debug_t_deep_task, &task); ProfEnd(); @@ -1755,9 +1771,6 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) void *ptr = cv_ptr_from_symbol(symbol); void_list_push(scratch.arena, &global_symbols, ptr); } - } else if (symbol.kind == 0x1176) { - CV_SymKind *kind_ptr = cv_kind_ptr_from_symbol(symbol); - *kind_ptr = CV_SymKind_SKIP; } if (cv_is_scope_symbol(symbol.kind)) { @@ -1840,7 +1853,7 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) CV_DebugS debug_s = task->cv->debug_s_arr[obj_idx]; String8List symbols = cv_sub_section_from_debug_s(debug_s, CV_C13SubSectionKind_Symbols); for EachNode(n, String8Node, symbols.first) { - for (U64 cursor = 0, symbol_cursor = 0; cursor + sizeof(CV_SymbolHeader) <= n->string.size; ) { + for (U64 cursor = 0; cursor + sizeof(CV_SymbolHeader) <= n->string.size; ) { CV_Symbol symbol = {0}; TryReadBreak(cv_read_symbol(n->string, cursor, CV_SymbolAlign, &symbol), cursor); @@ -1857,35 +1870,26 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) barrier_wait(tp->barrier); if (task_id == 0) { - // take into account arena header - for EachIndex(i, tp->worker_count) { task->proc_ref_sizes[i] += ARENA_HEADER_SIZE; } - U64 total_proc_ref_count = sum_array_u64(tp->worker_count, task->proc_ref_counts); - // prealloc output buffer for the proc refs - task->proc_refs.size = sum_array_u64(tp->worker_count, task->proc_ref_sizes); - task->proc_refs.str = push_array(gsi->arena, U8, task->proc_refs.size); task->proc_ref_hashes = push_array(scratch.arena, U64, total_proc_ref_count); - task->proc_ref_offs = offsets_from_counts_array_u64(scratch.arena, task->proc_ref_sizes, tp->worker_count); task->proc_ref_indices = offsets_from_counts_array_u64(scratch.arena, task->proc_ref_counts, tp->worker_count); - - task->proc_ref_count = sum_array_u64(tp->worker_count, task->proc_ref_counts); - task->proc_ref_nodes = push_array(gsi->arena, CV_SymbolNode, task->proc_ref_count); + task->proc_ref_arenas = alloc_arena_many(gsi->arena, tp->worker_count, task->proc_ref_sizes); + task->proc_ref_count = sum_array_u64(tp->worker_count, task->proc_ref_counts); + task->proc_ref_nodes = push_array(gsi->arena, CV_SymbolNode, task->proc_ref_count); } barrier_wait(tp->barrier); - U64 proc_refs_size = task->proc_ref_sizes[task_id]; - U64 proc_refs_off = task->proc_ref_offs[task_id]; U64 proc_ref_idx = task->proc_ref_indices[task_id]; - U8 *proc_refs = task->proc_refs.str + proc_refs_off; - Arena *proc_ref_arena = arena_alloc_(&(ArenaParams){ .flags = ArenaFlag_NoChain, .optional_backing_buffer = proc_refs, .reserve_size = proc_refs_size, .commit_size = proc_refs_size }); + Arena *proc_ref_arena = task->proc_ref_arenas[task_id]; for EachIndex(i, obj_indices.count) { - U64 obj_idx = obj_indices.v[i]; - CV_DebugS debug_s = task->cv->debug_s_arr[obj_idx]; - String8List symbols = cv_sub_section_from_debug_s(debug_s, CV_C13SubSectionKind_Symbols); - CV_ModIndex imod = task->mod_arr[obj_idx]->imod; + U64 obj_idx = obj_indices.v[i]; + CV_DebugS debug_s = task->cv->debug_s_arr[obj_idx]; + String8List symbols = cv_sub_section_from_debug_s(debug_s, CV_C13SubSectionKind_Symbols); + CV_ModIndex imod = task->mod_arr[obj_idx]->imod; + U64 symbol_cursor = sizeof(CV_Signature); for EachNode(n, String8Node, symbols.first) { - for (U64 cursor = 0, symbol_cursor = 0; cursor + sizeof(CV_SymbolHeader) <= n->string.size; ) { + for (U64 cursor = 0; cursor + sizeof(CV_SymbolHeader) <= n->string.size; ) { CV_Symbol symbol = {0}; TryReadBreak(cv_read_symbol(n->string, cursor, CV_SymbolAlign, &symbol), cursor); if (symbol.kind == CV_SymKind_SKIP) { continue; } @@ -1944,25 +1948,16 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) barrier_wait(tp->barrier); if (task_id == 0) { - for EachIndex(i, tp->worker_count) { task->public_symbol_sizes[i] += ARENA_HEADER_SIZE; } - U64 public_symbol_buffer_size = sum_array_u64(tp->worker_count, task->public_symbol_sizes); - task->public_symbol_offs = offsets_from_counts_array_u64(scratch.arena, task->public_symbol_sizes, tp->worker_count); - task->public_symbol_buffer = push_array(task->pdb->psi->arena, U8, public_symbol_buffer_size); - U64 public_symbol_total_count = sum_array_u64(tp->worker_count, task->public_symbol_node_counts); - task->public_symbol_node_offsets = offsets_from_counts_array_u64(scratch.arena, task->public_symbol_node_counts, tp->worker_count); - task->public_symbol_nodes = push_array(psi->gsi->arena, CV_SymbolNode, public_symbol_total_count); - task->public_symbols = push_array(scratch.arena, CV_SymbolList, tp->worker_count); - task->public_symbol_hashes = push_array(scratch.arena, U32 *, tp->worker_count); + task->public_symbol_arenas = alloc_arena_many(psi->gsi->arena, tp->worker_count, task->public_symbol_sizes); + task->public_symbol_node_arenas = alloc_arena_array(psi->gsi->arena, tp->worker_count, task->public_symbol_node_counts, CV_SymbolNode); + task->public_symbols = push_array(scratch.arena, CV_SymbolList, tp->worker_count); + task->public_symbol_hashes = push_array(scratch.arena, U32 *, tp->worker_count); } barrier_wait(tp->barrier); // make CV public symbols - U64 public_symbol_buffer_chunk_size = task->public_symbol_sizes[task_id]; - U8 *public_symbol_buffer_chunk = task->public_symbol_buffer + task->public_symbol_offs[task_id]; - Arena *public_symbol_arena = arena_alloc_(&(ArenaParams){ .flags = ArenaFlag_NoChain, .optional_backing_buffer = public_symbol_buffer_chunk, .reserve_size = public_symbol_buffer_chunk_size, .commit_size = public_symbol_buffer_chunk_size }); for EachNode(chunk, LNK_SymbolHashTrieChunk, task->symtab->chunks[task_id].first) { - U64 node_idx = task->public_symbol_node_offsets[task_id]; for EachIndex(i, chunk->count) { LNK_Symbol *symbol = chunk->v[i].symbol; LNK_ObjSymbolRef symbol_ref = lnk_ref_from_symbol(symbol); @@ -1975,11 +1970,8 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) CV_Pub32Flags flags = COFF_SymbolType_IsFunc(symbol_parsed.type) ? CV_Pub32Flag_Function : 0; ISectOff sc = lnk_sc_from_symbol(symbol); - CV_Symbol pub_symbol = cv_make_pub32(public_symbol_arena, flags, safe_cast_u32(sc.off), safe_cast_u16(sc.isect), symbol->name); - - task->public_symbol_nodes[node_idx].data = pub_symbol; - cv_symbol_list_push_node(&task->public_symbols[task_id], &task->public_symbol_nodes[node_idx]); - node_idx += 1; + CV_Symbol pub_symbol = cv_make_pub32(task->public_symbol_arenas[task_id], flags, safe_cast_u32(sc.off), safe_cast_u16(sc.isect), symbol->name); + cv_symbol_list_push(task->public_symbol_node_arenas[task_id], &task->public_symbols[task_id], pub_symbol); } } barrier_wait(tp->barrier); @@ -1998,11 +1990,11 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) // insert public symbols into PSI if (task_id == 0) { for EachIndex(i, tp->worker_count) { - U64 curr_idx = 0; - for (CV_SymbolNode *curr = task->public_symbols[i].first, *next = 0; curr != 0; curr = next) { + U64 k = 0; + for (CV_SymbolNode *curr = task->public_symbols[i].first, *next = 0; curr != 0; curr = next, k += 1) { next = curr->next; - gsi_push_(psi->gsi, task->public_symbol_hashes[i][curr_idx], curr); - curr_idx += 1; + curr->next = 0; + gsi_push_(psi->gsi, task->public_symbol_hashes[i][k], curr); } } } @@ -2023,15 +2015,26 @@ lnk_write_debug_s_to_pdb_module(PDB_DbiModule *mod, CV_DebugS debug_s, String8No mod->globrefs_size = 0; // signature - mod_cursor += str8_buffer_write_u32(buf, buf_pos, CV_Signature_C13); + U64 sig_size = str8_buffer_write_u32(buf, buf_pos, CV_Signature_C13); + mod->sym_data_size += sig_size; + mod_cursor += sig_size; // write symbols String8List symbols = cv_sub_section_from_debug_s(debug_s, CV_C13SubSectionKind_Symbols); + U64 scope_depth = 0; for EachNode(n, String8Node, symbols.first) { for (U64 cursor = 0; cursor + sizeof(CV_SymbolHeader) <= n->string.size; ) { CV_Symbol symbol = {0}; TryReadBreak(cv_read_symbol(n->string, cursor, CV_SymbolAlign, &symbol), cursor); if (symbol.kind == CV_SymKind_SKIP) { continue; } + + if (cv_is_global_symbol(symbol.kind)) { continue; } + else if (cv_is_typedef(symbol.kind) && scope_depth == 0) { continue; } + else if (symbol.kind == 0x1176) { continue; } + + if (cv_is_scope_symbol(symbol.kind)) { scope_depth += 1; } + else if (cv_is_end_symbol(symbol.kind)) { scope_depth -= 1; } + U64 symbol_size = cv_write_symbol_buf(buf, buf_pos, &symbol, PDB_SYMBOL_ALIGN); mod_cursor += symbol_size; mod->sym_data_size += symbol_size; @@ -2039,19 +2042,19 @@ lnk_write_debug_s_to_pdb_module(PDB_DbiModule *mod, CV_DebugS debug_s, String8No } // write file checksums, inlinee lines etc. - CV_C13SubSectionIdxKind mod_c13_layout[] = { - CV_C13SubSectionIdxKind_FileChksms, - CV_C13SubSectionIdxKind_FrameData, - CV_C13SubSectionIdxKind_InlineeLines, - CV_C13SubSectionIdxKind_CrossScopeImports, - CV_C13SubSectionIdxKind_CrossScopeExports, - CV_C13SubSectionIdxKind_IlLines, - CV_C13SubSectionIdxKind_FuncMDTokenMap, - CV_C13SubSectionIdxKind_TypeMDTokenMap, - CV_C13SubSectionIdxKind_MergedAssemblyInput, - CV_C13SubSectionIdxKind_CoffSymbolRVA, - CV_C13SubSectionIdxKind_XfgHashType, - CV_C13SubSectionIdxKind_XfgHashVirtual, + CV_C13SubSectionKind mod_c13_layout[] = { + CV_C13SubSectionKind_FileChksms, + CV_C13SubSectionKind_FrameData, + CV_C13SubSectionKind_InlineeLines, + CV_C13SubSectionKind_CrossScopeImports, + CV_C13SubSectionKind_CrossScopeExports, + CV_C13SubSectionKind_IlLines, + CV_C13SubSectionKind_FuncMDTokenMap, + CV_C13SubSectionKind_TypeMDTokenMap, + CV_C13SubSectionKind_MergedAssemblyInput, + CV_C13SubSectionKind_CoffSymbolRVA, + CV_C13SubSectionKind_XfgHashType, + CV_C13SubSectionKind_XfgHashVirtual, }; for EachElement(i, mod_c13_layout) { String8List data = cv_sub_section_from_debug_s(debug_s, mod_c13_layout[i]); @@ -2059,7 +2062,7 @@ lnk_write_debug_s_to_pdb_module(PDB_DbiModule *mod, CV_DebugS debug_s, String8No U64 ss_size = 0; ss_size += str8_buffer_write(buf, buf_pos, str8_struct((&(CV_C13SubSectionHeader ){ .kind = mod_c13_layout[i], .size = safe_cast_u32(data.total_size) }))); ss_size += str8_buffer_write_string_list(buf, buf_pos, data); - ss_size += str8_buffer_write_zeroes(buf, buf_pos, AlignPadPow2(mod_cursor, CV_C13SubSectionAlign)); + ss_size += str8_buffer_write_zeroes(buf, buf_pos, AlignPadPow2(mod_cursor + ss_size, CV_C13SubSectionAlign)); mod_cursor += ss_size; mod->c13_data_size += ss_size; } @@ -2071,18 +2074,16 @@ lnk_write_debug_s_to_pdb_module(PDB_DbiModule *mod, CV_DebugS debug_s, String8No U64 ss_size = 0; ss_size += str8_buffer_write(buf, buf_pos, str8_struct((&(CV_C13SubSectionHeader){ .kind = CV_C13SubSectionKind_Lines, .size = safe_cast_u32(n->string.size) }))); ss_size += str8_buffer_write(buf, buf_pos, n->string); - ss_size += str8_buffer_write_zeroes(buf, buf_pos, AlignPadPow2(mod_cursor, CV_C13SubSectionAlign)); + ss_size += str8_buffer_write_zeroes(buf, buf_pos, AlignPadPow2(mod_cursor + ss_size, CV_C13SubSectionAlign)); mod_cursor += ss_size; mod->c13_data_size += ss_size; } // write global refs String8List globrefs = cv_sub_section_from_debug_s(debug_s, CV_C13SubSectionKind_GlobalRefs); - U64 globrefs_size = 0; - globrefs_size += str8_buffer_write_u32(buf, buf_pos, safe_cast_u32(globrefs.total_size)); - globrefs_size += str8_buffer_write_string_list(buf, buf_pos, globrefs); - mod_cursor += globrefs_size; - mod->globrefs_size += globrefs_size; + mod->globrefs_size += str8_buffer_write_u32(buf, buf_pos, safe_cast_u32(globrefs.total_size)); + mod->globrefs_size += str8_buffer_write_string_list(buf, buf_pos, globrefs); + mod_cursor += mod->globrefs_size; return mod_cursor; } @@ -2118,41 +2119,114 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) for EachIndex(i, obj_indices.count) { Temp temp = temp_begin(scratch.arena); - U64 obj_idx = obj_indices.v[i]; - PDB_DbiModule *mod = task->mod_arr[obj_idx]; - CV_DebugS debug_s = task->cv->debug_s_arr[obj_idx]; + U64 obj_idx = obj_indices.v[i]; + PDB_DbiModule *mod = task->mod_arr[obj_idx]; + CV_DebugS debug_s = task->cv->debug_s_arr[obj_idx]; + String8List mod_data = msf_data_from_sn(temp.arena, task->pdb->msf, mod->sn); - String8List mod_pages = msf_raw_pages_from_sn(temp.arena, task->pdb->msf, mod->sn); - if (mod_pages.node_count == 0) { continue; } + if (mod_data.node_count) { + String8Node buf = *mod_data.first; + U64 pos = 0; + lnk_write_debug_s_to_pdb_module(mod, debug_s, &buf, &pos); - String8Node buf = *mod_pages.first; - U64 pos = 0; - lnk_write_debug_s_to_pdb_module(mod, debug_s, &buf, &pos); - - // sub range symbol data pages and patch symbol tree offsets - String8List mod_symbols = str8_list_substr(temp.arena, mod_pages, r1u64(sizeof(CV_Signature), sizeof(CV_Signature) + mod->sym_data_size)); - cv_patch_symbol_tree_offsets_new(mod_symbols, sizeof(CV_Signature), PDB_SYMBOL_ALIGN); + // sub range symbol data pages and patch symbol tree offsets + Rng1U64 sym_data_range = r1u64(sizeof(CV_Signature), mod->sym_data_size); + String8List mod_symbols = str8_list_substr(temp.arena, mod_data, sym_data_range); + Assert(mod_symbols.total_size == dim_1u64(sym_data_range)); + cv_patch_symbol_tree_offsets(mod_symbols, sizeof(CV_Signature), PDB_SYMBOL_ALIGN); + } temp_end(temp); } barrier_wait(tp->barrier); - for EachIndex(i, obj_indices.count) { - U64 obj_idx = obj_indices.v[i]; - CV_DebugS debug_s = task->cv->debug_s_arr[obj_idx]; - String8 string_table = cv_string_table_from_debug_s(debug_s); - String8List file_chksms_raw = cv_sub_section_from_debug_s(debug_s, CV_C13SubSectionKind_FileChksms); - CV_ChecksumList file_chksms = cv_c13_parse_checksum_data_list(scratch.arena, file_chksms_raw); - String8List src_file_names = cv_c13_collect_source_file_names(arena, file_chksms, string_table); + { + // count strings in string tables + { + if (task_id == 0) { + task->string_counts = push_array(scratch.arena, U64, tp->worker_count); + } + barrier_wait(tp->barrier); - // collect source files module refs - str8_list_concat_in_place(&task->mod_arr[obj_idx]->source_file_list, &src_file_names); + for EachIndex(i, obj_indices.count) { + U64 obj_idx = obj_indices.v[i]; + CV_DebugS debug_s = task->cv->debug_s_arr[obj_idx]; + String8 string_table = cv_string_table_from_debug_s(debug_s); + U64 string_count = 0; + for (U64 cursor = 0; cursor < string_table.size; cursor += 1) { + if (string_table.str[cursor] == '\0') { + task->string_counts[task_id] += 1; + } + } + } + barrier_wait(tp->barrier); + } - // patch path offsets in file checksum headers (in obj the offsets point to a string table in .debug$S, - // and because linker merges string tables the old offsets need to be fixed up) - cv_c13_patch_string_offsets_in_checksum_list(file_chksms, string_table, task->pdb->info->strtab.size, task->string_ht); + if (task_id == 0) { + task->string_arenas = alloc_arena_array(task->pdb->dbi->arena, tp->worker_count, task->string_counts, String8Node); + } + barrier_wait(tp->barrier); + + for EachIndex(i, obj_indices.count) { + Temp temp = temp_begin(scratch.arena); + + U64 obj_idx = obj_indices.v[i]; + PDB_DbiModule *mod = task->mod_arr[obj_idx]; + String8List mod_data = msf_data_from_sn(temp.arena, task->pdb->msf, mod->sn); + Rng1U64 c13_data_range = r1u64(mod->sym_data_size + mod->c11_data_size, mod->sym_data_size + mod->c11_data_size + mod->c13_data_size); + String8List c13_data = str8_list_substr(temp.arena, mod_data, c13_data_range); + + CV_DebugS debug_s = task->cv->debug_s_arr[obj_idx]; + String8 string_table = cv_string_table_from_debug_s(debug_s); + + // checksum is always at the head of C13 data + String8List file_chksms_raw = {0}; + { + String8Node buf = *c13_data.first; + U64 buf_pos = 0; + + CV_C13SubSectionHeader header = {0}; + str8_buffer_read(&buf, &buf_pos, sizeof(header), &header); + + if (header.kind == CV_C13SubSectionKind_FileChksms) { + Rng1U64 file_chksms_range = r1u64(sizeof(CV_C13SubSectionHeader), sizeof(CV_C13SubSectionHeader) + header.size); + file_chksms_raw = str8_list_substr(temp.arena, c13_data, file_chksms_range); + } + } + + // fixup file name offsets in checksum headers + if (file_chksms_raw.total_size) { + String8Node buf = *file_chksms_raw.first; + U64 buf_pos = 0; + U64 cursor = 0; + for (;;) { + CV_C13Checksum header = {0}; + if (str8_buffer_peek(&buf, &buf_pos, sizeof(header), &header) != sizeof(header)) { break; } + + String8 name = str8_cstring_capped(string_table.str + header.name_off, string_table.str + string_table.size); + CV_StringBucket *bucket = cv_string_hash_table_lookup(task->string_ht, name); + U64 name_off = task->pdb->info->strtab.size + bucket->u.offset; + + // update name offset + { + String8Node buf_copy = buf; + U64 buf_pos_copy = buf_pos; + str8_buffer_skip(&buf_copy, &buf_pos_copy, OffsetOf(CV_C13Checksum, name_off)); + str8_buffer_write_u32(&buf_copy, &buf_pos_copy, safe_cast_u32(name_off)); + } + + str8_buffer_skip(&buf, &buf_pos, AlignPow2(sizeof(header) + header.len, CV_FileCheckSumsAlign)); + } + } + + // collect mod source files + String8List source_file_list = str8_split_by_string_chars(task->string_arenas[task_id], string_table, str8_lit("\0"), 0); + str8_list_concat_in_place(&mod->source_file_list, &source_file_list); + + temp_end(temp); + } + barrier_wait(tp->barrier); } - barrier_wait(tp->barrier); scratch_end(scratch); } @@ -2275,7 +2349,6 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config ProfBegin("Merge String Tables"); task.string_ht = cv_dedup_string_tables(tp_arena, tp, cv->obj_count, cv->debug_s_arr); cv_string_hash_table_assign_buffer_offsets(tp, task.string_ht); - pdb_strtab_add_cv_string_hash_table(&task.pdb->info->strtab, task.string_ht); ProfEnd(); ProfScope ("Alloc Modules") for EachIndex(obj_idx, cv->obj_count) { task.mod_arr[obj_idx] = dbi_push_module(task.pdb->dbi, cv->obj_arr[obj_idx]->path, lnk_obj_get_lib_path(cv->obj_arr[obj_idx])); } @@ -2290,6 +2363,10 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config gsi_build(tp, pdb->gsi, pdb->msf, dbi->globals_sn, dbi->symbols_sn); } ProfScope("Write Modules") tp_for_parallel(tp, 0, tp->worker_count, lnk_write_pdb_modules, &task); + + ProfBegin("Add string tables"); + pdb_strtab_add_cv_string_hash_table(&task.pdb->info->strtab, task.string_ht); + ProfEnd(); ProfBegin("Build Section Contrib Map"); { diff --git a/src/linker/lnk_debug_info.h b/src/linker/lnk_debug_info.h index 31b42819..1591d0a8 100644 --- a/src/linker/lnk_debug_info.h +++ b/src/linker/lnk_debug_info.h @@ -130,7 +130,7 @@ typedef struct typedef struct { - String8 image_data; + String8 image_data; LNK_SymbolTable *symtab; LNK_CodeViewInput *cv; @@ -155,25 +155,23 @@ typedef struct U64 *proc_ref_counts; // [worker_count] U64 *proc_ref_sizes; // [worker_count] U64 *proc_ref_hashes; // [total_proc_ref_count] - U64 *proc_ref_offs; // [worker_count] U64 *proc_ref_indices; // [worker_count] String8 proc_refs; U64 proc_ref_count; CV_SymbolNode *proc_ref_nodes; // [proc_ref_count] - Arena *proc_ref_arena; + Arena **proc_ref_arenas; U64 *public_symbol_sizes; // [worker_count] - U64 *public_symbol_offs; // [worker_count] - U8 *public_symbol_buffer; + Arena **public_symbol_arenas; + Arena **public_symbol_node_arenas; U64 *public_symbol_node_counts; // [worker_count] - U64 *public_symbol_node_offsets; // [worker_count] - CV_SymbolNode *public_symbol_nodes; // [public_symbol_total_count] CV_SymbolList *public_symbols; // [worker_count] U32 **public_symbol_hashes; // [worker_count][public_symbol.count] // process C13 data - String8List *source_file_names_list_arr; CV_StringHashTable string_ht; + U64 *string_counts; + Arena **string_arenas; // push DBI SC Map PE_BinInfo pe; diff --git a/src/linker/lnk_obj.c b/src/linker/lnk_obj.c index b128b177..90bf0464 100644 --- a/src/linker/lnk_obj.c +++ b/src/linker/lnk_obj.c @@ -302,14 +302,16 @@ THREAD_POOL_TASK_FUNC(lnk_obj_initer) String8 debug_s_data = str8_substr(input->data, rng_1u64(sect_header->foff, sect_header->foff+sect_header->fsize)); CV_DebugS debug_s = cv_debug_s_from_data(temp.arena, debug_s_data); for EachNode(symbols_n, String8Node, debug_s.data_list[CV_C13SubSectionIdxKind_Symbols].first) { - CV_SymbolList symbol_list = {0}; - cv_parse_symbol_sub_section_capped(scratch.arena, &symbol_list, 0, symbols_n->string, CV_SymbolAlign, 2); - if (symbol_list.first->data.kind == CV_SymKind_COMPILE3) { - comp_symbol = symbol_list.first->data; - goto found_comp_symbol; - } else if (symbol_list.last->data.kind == CV_SymKind_COMPILE3) { - comp_symbol = symbol_list.last->data; - goto found_comp_symbol; + for (U64 cursor = 0, count = 0; cursor < symbols_n->string.size && count < 2; count += 1) { + CV_SymbolHeader symbol_header; + TryReadBreak(str8_deserial_read_struct(symbols_n->string, cursor, &symbol_header), cursor); + if (symbol_header.kind == CV_SymKind_COMPILE3) { + String8 raw_symbol = str8_substr(symbols_n->string, r1u64(cursor, cursor + symbol_header.size + sizeof(CV_SymSize))); + comp_symbol = cv_symbol_from_ptr(raw_symbol.str); + goto found_comp_symbol; + } + cursor += symbol_header.size + sizeof(CV_SymSize); + cursor = AlignPow2(cursor, CV_SymbolAlign); } } temp_end(temp); diff --git a/src/linker/pdb_ext/msf_builder.c b/src/linker/pdb_ext/msf_builder.c index 6d82d45d..fbbe28e0 100644 --- a/src/linker/pdb_ext/msf_builder.c +++ b/src/linker/pdb_ext/msf_builder.c @@ -830,8 +830,8 @@ msf_stream_resize_ex(MSF_Context *msf, MSF_Stream *stream, MSF_UInt size) } // update stream - stream->size = Min(stream->size, stream->page_list.count * msf->page_size); - stream->pos = Min(stream->pos, stream->size); + stream->size = size; + stream->pos = Min(stream->pos, stream->size); stream->pos_page = 0; return 1; @@ -1212,26 +1212,40 @@ msf_stream_write_string_parallel(TP_Context *tp, MSF_Context *msf, MSF_StreamNum } internal String8List -msf_raw_pages_from_sn(Arena *arena, MSF_Context *msf, MSF_StreamNumber sn) +msf_data_from_sn(Arena *arena, MSF_Context *msf, MSF_StreamNumber sn) { String8List result = {0}; MSF_Stream *stream = msf_find_stream(msf, sn); String8 acc = {0}; - for EachNode(n, MSF_PageNode, stream->page_list.first) { - String8 p = msf_data_from_pn(msf->page_data_list, msf->page_size, n->pn); - if (acc.str + acc.size == p.str) { - acc.size += p.size; + + MSF_PageNode *n; + for (n = stream->page_list.first; n != stream->page_list.last; n = n->next) { + String8 page = msf_data_from_pn(msf->page_data_list, msf->page_size, n->pn); + + if (acc.str + acc.size != page.str) { + if (acc.size > 0) { + str8_list_push(arena, &result, acc); + } + acc = page; } else { - str8_list_push(arena, &result, acc); - acc = p; + acc.size += page.size; } } - if (result.node_count && result.last->string.str + result.last->string.size == acc.str) { - result.last->string.size += acc.size; - } else { - str8_list_push(arena, &result, acc); + if (n) { + String8 page = msf_data_from_pn(msf->page_data_list, msf->page_size, n->pn); + if (acc.str + acc.size != page.str) { + acc = page; + } else { + acc.size += page.size; + } + + acc.size = Min(stream->size - result.total_size, acc.size); + + if (acc.size > 0) { + str8_list_push(arena, &result, acc); + } } return result; diff --git a/src/linker/pdb_ext/msf_builder.h b/src/linker/pdb_ext/msf_builder.h index ebbdc1f1..7364cf7f 100644 --- a/src/linker/pdb_ext/msf_builder.h +++ b/src/linker/pdb_ext/msf_builder.h @@ -188,7 +188,7 @@ internal B32 msf_stream_write_parallel(TP_Context *tp, MSF_Context *msf, MSF_Str #define msf_stream_write_array(m, s, v, c) msf_stream_write(m, s, (void*)(v), sizeof(*(v)) * (c)) #define msf_stream_write_struct(m, s, v ) msf_stream_write_array(m, s, v, 1) -internal String8List msf_raw_pages_from_sn(Arena *arena, MSF_Context *msf, MSF_StreamNumber sn); +internal String8List msf_data_from_sn(Arena *arena, MSF_Context *msf, MSF_StreamNumber sn); internal MSF_UInt msf_count_pages(MSF_UInt page_size, U64 data_size); internal MSF_PageNumber msf_get_page_count_cap(MSF_PageDataList page_data_list, MSF_UInt page_size);