From d2db4542cb735f50ba615dd3aa94220fe88a99eb Mon Sep 17 00:00:00 2001 From: Nikita Smith Date: Thu, 17 Oct 2024 13:56:33 -0700 Subject: [PATCH] check type server signature against signature in obj --- src/linker/codeview_ext/codeview.h | 19 +++- src/linker/lnk_debug_info.c | 151 ++++++++++++++++++++++------- src/linker/lnk_debug_info.h | 16 ++- src/linker/lnk_error.h | 1 + src/linker/pdb_ext/pdb_builder.c | 87 +++++++++-------- src/linker/pdb_ext/pdb_builder.h | 10 ++ 6 files changed, 199 insertions(+), 85 deletions(-) diff --git a/src/linker/codeview_ext/codeview.h b/src/linker/codeview_ext/codeview.h index 1dfa42be..83c44adb 100644 --- a/src/linker/codeview_ext/codeview.h +++ b/src/linker/codeview_ext/codeview.h @@ -207,11 +207,24 @@ typedef struct CV_UDTInfo typedef struct CV_TypeServerInfo { - String8 name; - COFF_Guid sig; - U32 age; + String8 name; + OS_Guid sig; + U32 age; } CV_TypeServerInfo; +typedef struct CV_TypeServerInfoNode +{ + struct CV_TypeServerInfoNode *next; + CV_TypeServerInfo data; +} CV_TypeServerInfoNode; + +typedef struct CV_TypeServerInfoList +{ + CV_TypeServerInfoNode *first; + CV_TypeServerInfoNode *last; + U64 count; +} CV_TypeServerInfoList; + typedef struct CV_PrecompInfo { CV_TypeIndex start_index; diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 134ab865..36c5c953 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -274,29 +274,52 @@ lnk_do_debug_info_discard(CV_DebugS *debug_s_arr, CV_SymbolListArray *parsed_sym MemoryZeroStruct(inlineelines_ptr); } +internal +THREAD_POOL_TASK_FUNC(lnk_msf_parsed_from_data_task) +{ + ProfBeginFunction(); + LNK_MsfParsedFromDataTask *task = raw_task; + // TODO: pick Info, TPI and IPI to flattten to make sure we don't waste compute on throw-away streams + task->msf_parse_arr[task_id] = msf_parsed_from_data(arena, task->data_arr.v[task_id]); + ProfEnd(); +} + +internal MSF_Parsed ** +lnk_msf_parsed_from_data_parallel(TP_Arena *arena, TP_Context *tp, String8Array data_arr) +{ + ProfBeginFunction(); + LNK_MsfParsedFromDataTask task = {0}; + task.data_arr = data_arr; + task.msf_parse_arr = push_array_no_zero(arena->v[0], MSF_Parsed *, data_arr.count); + tp_for_parallel(tp, arena, data_arr.count, lnk_msf_parsed_from_data_task, &task); + ProfEnd(); + return task.msf_parse_arr; +} + internal THREAD_POOL_TASK_FUNC(lnk_get_external_leaves_task) { ProfBeginFunction(); - LNK_GetExternalLeavesTask *task = raw_task; - - U64 ts_idx = task_id; + U64 ts_idx = task_id; + LNK_GetExternalLeavesTask *task = raw_task; + MSF_Parsed *msf_parse = task->msf_parse_arr[ts_idx]; task->external_ti_ranges[ts_idx] = push_array_no_zero(arena, Rng1U64, CV_TypeIndexSource_COUNT); task->external_leaves[ts_idx] = push_array_no_zero(arena, CV_DebugT, CV_TypeIndexSource_COUNT); task->is_corrupted[ts_idx] = 1; - // TODO: pick TPI and IPI to flattten to make sure we don't waste compute on throw-away streams - MSF_Parsed *msf_parse = msf_parsed_from_data(arena, task->msf_data_arr[ts_idx]); - if (msf_parse) { - PDB_TypeServerParse tpi_parse, ipi_parse; - PDB_OpenTypeServerError tpi_error = pdb_type_server_parse_from_data(msf_parse->streams[PDB_FixedStream_Tpi], &tpi_parse); - PDB_OpenTypeServerError ipi_error = pdb_type_server_parse_from_data(msf_parse->streams[PDB_FixedStream_Ipi], &ipi_parse); + PDB_OpenTypeServerError tpi_error = PDB_OpenTypeServerError_UNKNOWN; + PDB_OpenTypeServerError ipi_error = PDB_OpenTypeServerError_UNKNOWN; - if (tpi_error == PDB_OpenTypeServerError_OK && - ipi_error == PDB_OpenTypeServerError_OK) { + PDB_TypeServerParse tpi_parse, ipi_parse; + if (PDB_FixedStream_Tpi < msf_parse->stream_count && PDB_FixedStream_Ipi < msf_parse->stream_count) { + tpi_error = pdb_type_server_parse_from_data(msf_parse->streams[PDB_FixedStream_Tpi], &tpi_parse); + ipi_error = pdb_type_server_parse_from_data(msf_parse->streams[PDB_FixedStream_Ipi], &ipi_parse); + } + + if (tpi_error == PDB_OpenTypeServerError_OK && ipi_error == PDB_OpenTypeServerError_OK) { task->is_corrupted[ts_idx] = 0; task->external_ti_ranges[ts_idx][CV_TypeIndexSource_NULL] = rng_1u64(0,0); @@ -308,15 +331,12 @@ THREAD_POOL_TASK_FUNC(lnk_get_external_leaves_task) task->external_leaves[ts_idx][CV_TypeIndexSource_IPI] = cv_debug_t_from_data(arena, ipi_parse.leaf_data, PDB_LEAF_ALIGN); } else { if (tpi_error != PDB_OpenTypeServerError_OK) { - lnk_error(LNK_Error_UnableToOpenTypeServer, "failed to open TPI in %S, reson %S", task->path_arr[ts_idx], pdb_string_from_open_type_server_error(tpi_error)); + lnk_error(LNK_Error_UnableToOpenTypeServer, "failed to open TPI in %S, reson %S", task->ts_info_arr[ts_idx].name, pdb_string_from_open_type_server_error(tpi_error)); } if (ipi_error != PDB_OpenTypeServerError_OK) { - lnk_error(LNK_Error_UnableToOpenTypeServer, "failed to open IPI in %S, reason %S", task->path_arr[ts_idx], pdb_string_from_open_type_server_error(ipi_error)); + lnk_error(LNK_Error_UnableToOpenTypeServer, "failed to open IPI in %S, reason %S", task->ts_info_arr[ts_idx].name, pdb_string_from_open_type_server_error(ipi_error)); } } - } else { - MemoryZeroTyped(task->external_ti_ranges[ts_idx], CV_TypeIndexSource_COUNT); - MemoryZeroTyped(task->external_leaves[ts_idx], CV_TypeIndexSource_COUNT); } ProfEnd(); @@ -537,18 +557,20 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, String8List lib_dir CV_DebugT *merged_debug_t_p_arr = lnk_merge_debug_t_and_debug_p(tp_arena->v[0], internal_count, internal_debug_t_arr, internal_debug_p_arr); ProfBegin("Analyze & Read External Type Server Files"); - String8Array type_server_path_arr; + String8Array ts_path_arr; Rng1U64 **external_ti_ranges; CV_DebugT **external_leaves; U64 *obj_to_ts_idx_arr = push_array_no_zero(tp_arena->v[0], U64, external_count); U64List *ts_to_obj_arr = push_array(tp_arena->v[0], U64List, external_count); { - HashTable *type_server_path_ht = hash_table_init(scratch.arena, 256); - HashTable *ignored_path_ht = hash_table_init(scratch.arena, 256); - String8List type_server_path_list; MemoryZeroStruct(&type_server_path_list); + HashTable *type_server_path_ht = hash_table_init(scratch.arena, 256); + HashTable *ignored_path_ht = hash_table_init(scratch.arena, 256); + CV_TypeServerInfoList ts_info_list = {0}; // push null - str8_list_pushf(scratch.arena, &type_server_path_list, ""); + CV_TypeServerInfoNode *null_ts_info = push_array(scratch.arena, CV_TypeServerInfoNode, 1); + SLLQueuePush(ts_info_list.first, ts_info_list.last, null_ts_info); + ++ts_info_list.count; for (U64 obj_idx = 0; obj_idx < external_count; ++obj_idx) { // first leaf always type server @@ -620,9 +642,20 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, String8List lib_dir present->obj->path); } } else { - U64 ts_idx = type_server_path_list.node_count; + U64 ts_idx = ts_info_list.count; + + // when we search matches on disk we store path on scratch, + // make path copy in case we need it for error reporting path = push_str8_copy(tp_arena->v[0], path); - str8_list_push(scratch.arena, &type_server_path_list, path); + + // fill out type server info we read from obj + CV_TypeServerInfoNode *ts_info_node = push_array(scratch.arena, CV_TypeServerInfoNode, 1); + ts_info_node->data = ts; + ts_info_node->data.name = path; + + // push to type server info list + SLLQueuePush(ts_info_list.first, ts_info_list.last, ts_info_node); + ts_info_list.count += 1; // wire obj to type server obj_to_ts_idx_arr[obj_idx] = ts_idx; @@ -642,25 +675,69 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, String8List lib_dir } } - // read type servers from disk in parallel - type_server_path_arr = str8_array_from_list(tp_arena->v[0], &type_server_path_list); + // type server info list -> array + ts_path_arr.count = ts_info_list.count; + ts_path_arr.v = push_array(tp_arena->v[0], String8, ts_info_list.count); + CV_TypeServerInfo *ts_info_arr = push_array(scratch.arena, CV_TypeServerInfo, ts_info_list.count); + { + U64 idx = 0; + for (CV_TypeServerInfoNode *n = ts_info_list.first; n != 0; n = n->next, ++idx) { + ts_path_arr.v[idx] = n->data.name; + ts_info_arr[idx] = n->data; + } + } + // read type servers from disk in parallel { ProfBegin("Read External Type Servers"); - String8Array msf_data_arr = os_data_from_file_path_parallel(tp, scratch.arena, type_server_path_arr); + String8Array msf_data_arr = os_data_from_file_path_parallel(tp, scratch.arena, ts_path_arr); ProfEnd(); - ProfBeginDynamic("Open External Type Servers [Count %llu]", type_server_path_arr.count); - LNK_GetExternalLeavesTask task; - task.path_arr = type_server_path_arr.v; - task.msf_data_arr = msf_data_arr.v; - task.external_ti_ranges = push_array_no_zero(tp_arena->v[0], Rng1U64 *, msf_data_arr.count); - task.external_leaves = push_array_no_zero(tp_arena->v[0], CV_DebugT *, msf_data_arr.count); - task.is_corrupted = push_array_no_zero(scratch.arena, B8, msf_data_arr.count); + MSF_Parsed **msf_parse_arr = lnk_msf_parsed_from_data_parallel(tp_arena, tp, msf_data_arr); + + ProfBegin("Error check type servers"); + for (U64 ts_idx = 0; ts_idx < msf_data_arr.count; ++ts_idx) { + MSF_Parsed *msf_parse = msf_parse_arr[ts_idx]; + + B32 do_debug_info_discard = 0; + + if (!msf_parse) { + do_debug_info_discard = 1; + } else { + PDB_InfoParse info_parse = {0}; + pdb_info_parse_from_data(msf_parse->streams[PDB_FixedStream_Info], &info_parse); + if (!MemoryMatchStruct(&info_parse.guid, &ts_info_arr[ts_idx].sig)) { + Temp scratch = scratch_begin(0,0); + String8 expected_sig_str = os_string_from_guid(scratch.arena, ts_info_arr[ts_idx].sig); + String8 on_disk_sig_str = os_string_from_guid(scratch.arena, info_parse.guid); + lnk_error(LNK_Warning_MismatchedTypeServerSignature, "%S: signature mismatch in type server read from disk, expected %S, got %S", + ts_info_arr[ts_idx].name, expected_sig_str, on_disk_sig_str); + scratch_end(scratch); + + do_debug_info_discard = 1; + } + } + + if (do_debug_info_discard) { + U64List obj_idx_list = ts_to_obj_arr[ts_idx]; + for (U64Node *obj_idx_n = obj_idx_list.first; obj_idx_n != 0; obj_idx_n = obj_idx_n->next) { + lnk_do_debug_info_discard(external_debug_s_arr, external_parsed_symbols, obj_idx_n->data); + } + } + } + ProfEnd(); + + ProfBeginDynamic("Open External Type Servers [Count %llu]", ts_path_arr.count); + LNK_GetExternalLeavesTask task = {0}; + task.ts_info_arr = ts_info_arr; + task.msf_parse_arr = msf_parse_arr; + task.external_ti_ranges = push_array_no_zero(tp_arena->v[0], Rng1U64 *, msf_data_arr.count); + task.external_leaves = push_array_no_zero(tp_arena->v[0], CV_DebugT *, msf_data_arr.count); + task.is_corrupted = push_array_no_zero(scratch.arena, B8, msf_data_arr.count); tp_for_parallel(tp, tp_arena, msf_data_arr.count, lnk_get_external_leaves_task, &task); ProfEnd(); - String8List unopen_type_server_list; MemoryZeroStruct(&unopen_type_server_list); + String8List unopen_type_server_list = {0}; // discard debug info that depends on the missing type server for (U64 ts_idx = 1; ts_idx < msf_data_arr.count; ++ts_idx) { @@ -676,7 +753,7 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, String8List lib_dir for (U64 ts_idx = 1; ts_idx < msf_data_arr.count; ++ts_idx) { if (task.is_corrupted[ts_idx]) { U64List obj_idx_list = ts_to_obj_arr[ts_idx]; - str8_list_pushf(scratch.arena, &unopen_type_server_list, "\t%S\n", type_server_path_arr.v[ts_idx]); + str8_list_pushf(scratch.arena, &unopen_type_server_list, "\t%S\n", ts_path_arr.v[ts_idx]); str8_list_pushf(scratch.arena, &unopen_type_server_list, "\t\tDependent obj(s):\n"); for (U64Node *obj_idx_node = obj_idx_list.first; obj_idx_node != 0; obj_idx_node = obj_idx_node->next) { String8 obj_path = external_obj_arr[obj_idx_node->data].path; @@ -704,8 +781,8 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, String8List lib_dir cv.count = obj_count; cv.internal_count = internal_count; cv.external_count = external_count; - cv.type_server_count = type_server_path_arr.count; - cv.type_server_path_arr = type_server_path_arr.v; + cv.type_server_count = ts_path_arr.count; + cv.type_server_path_arr = ts_path_arr.v; cv.ts_to_obj_arr = ts_to_obj_arr; cv.obj_arr = sorted_obj_arr; cv.pch_arr = pch_arr; diff --git a/src/linker/lnk_debug_info.h b/src/linker/lnk_debug_info.h index c5c2614f..979dcef0 100644 --- a/src/linker/lnk_debug_info.h +++ b/src/linker/lnk_debug_info.h @@ -142,11 +142,17 @@ typedef struct typedef struct { - String8 *path_arr; - String8 *msf_data_arr; - Rng1U64 **external_ti_ranges; - CV_DebugT **external_leaves; - B8 *is_corrupted; + String8Array data_arr; + MSF_Parsed **msf_parse_arr; +} LNK_MsfParsedFromDataTask; + +typedef struct +{ + CV_TypeServerInfo *ts_info_arr; + MSF_Parsed **msf_parse_arr; + Rng1U64 **external_ti_ranges; + CV_DebugT **external_leaves; + B8 *is_corrupted; } LNK_GetExternalLeavesTask; //////////////////////////////// diff --git a/src/linker/lnk_error.h b/src/linker/lnk_error.h index 1bf30beb..9e75a312 100644 --- a/src/linker/lnk_error.h +++ b/src/linker/lnk_error.h @@ -60,6 +60,7 @@ typedef enum LNK_Warning_InvalidNatvisFileExt, LNK_Warning_LargePages, LNK_Warning_LargePagesNotEnabled, + LNK_Warning_MismatchedTypeServerSignature, LNK_Warning_MissingExternalTypeServer, LNK_Warning_MultipleDebugTAndDebugP, LNK_Warning_MultipleExternalTypeServers, diff --git a/src/linker/pdb_ext/pdb_builder.c b/src/linker/pdb_ext/pdb_builder.c index 77e77376..02e09ef6 100644 --- a/src/linker/pdb_ext/pdb_builder.c +++ b/src/linker/pdb_ext/pdb_builder.c @@ -1662,23 +1662,11 @@ pdb_info_alloc(U32 age, COFF_TimeStamp time_stamp, OS_Guid guid) return info; } -internal PDB_InfoContext * -pdb_info_open(MSF_Context *msf, MSF_StreamNumber sn) +internal void +pdb_info_parse_from_data(String8 data, PDB_InfoParse *parse_out) { - ProfBeginFunction(); - Temp scratch = scratch_begin(0,0); - - COFF_TimeStamp time_stamp = 0; - U32 age = 0; - OS_Guid guid = {0}; - PDB_FeatureFlags flags = 0; - PDB_HashTable named_stream_ht = {0}; - - U64 info_size = msf_stream_get_size(msf, sn); - String8 info_data = msf_stream_read_block(scratch.arena, msf, sn, info_size); - PDB_InfoVersion version = 0; - str8_deserial_read_struct(info_data, 0, &version); + str8_deserial_read_struct(data, 0, &version); switch (version) { case PDB_InfoVersion_VC70: { @@ -1686,23 +1674,55 @@ pdb_info_open(MSF_Context *msf, MSF_StreamNumber sn) // read header PDB_InfoHeaderV70 header; - cursor += str8_deserial_read_struct(info_data, cursor, &header); + cursor += str8_deserial_read_struct(data, cursor, &header); - time_stamp = header.time_stamp; - age = header.age; - guid = header.guid; + parse_out->version = version; + parse_out->time_stamp = header.time_stamp; + parse_out->age = header.age; + parse_out->guid = header.guid; + parse_out->extra_info = str8_skip(data, cursor); + } break; + case PDB_InfoVersion_VC2: + case PDB_InfoVersion_VC4: + case PDB_InfoVersion_VC41: + case PDB_InfoVersion_VC50: + case PDB_InfoVersion_VC98: + case PDB_InfoVersion_VC70_DEP: + case PDB_InfoVersion_VC80: + case PDB_InfoVersion_VC110: + case PDB_InfoVersion_VC140: { + NotImplemented; + } break; + default: Assert(!"invalid info stream version"); break; + } +} +internal PDB_InfoContext * +pdb_info_open(MSF_Context *msf, MSF_StreamNumber sn) +{ + ProfBeginFunction(); + Temp scratch = scratch_begin(0,0); + + U64 info_size = msf_stream_get_size(msf, sn); + String8 info_data = msf_stream_read_block(scratch.arena, msf, sn, info_size); + + PDB_InfoParse parse = {0}; + pdb_info_parse_from_data(info_data, &parse); + + PDB_FeatureFlags flags = 0; + PDB_HashTable named_stream_ht = {0}; + if (parse.version == PDB_InfoVersion_VC70) { // open named stream hash table - String8 named_stream_ht_data = str8_skip(info_data, cursor); + U64 cursor = 0; U64 named_stream_ht_size = 0; - PDB_HashTableParseError named_stream_ht_error = pdb_named_stream_ht_from_data(&named_stream_ht, named_stream_ht_data, &named_stream_ht_size); + PDB_HashTableParseError named_stream_ht_error = pdb_named_stream_ht_from_data(&named_stream_ht, parse.extra_info, &named_stream_ht_size); if (named_stream_ht_error == PDB_HashTableParseError_OK) { cursor += named_stream_ht_size; - + // read PDB features while (cursor < info_data.size) { PDB_FeatureSig sig = 0; - cursor += str8_deserial_read_struct(info_data, cursor, &sig); + cursor += str8_deserial_read_struct(parse.extra_info, cursor, &sig); switch (sig) { case PDB_FeatureSig_NULL: break; case PDB_FeatureSig_VC140: { @@ -1720,21 +1740,8 @@ pdb_info_open(MSF_Context *msf, MSF_StreamNumber sn) } else { Assert(!"unable to open named stream hash table"); } - } break; - case PDB_InfoVersion_VC2: - case PDB_InfoVersion_VC4: - case PDB_InfoVersion_VC41: - case PDB_InfoVersion_VC50: - case PDB_InfoVersion_VC98: - case PDB_InfoVersion_VC70_DEP: - case PDB_InfoVersion_VC80: - case PDB_InfoVersion_VC110: - case PDB_InfoVersion_VC140: { - NotImplemented; - } break; - default: Assert(!"invalid info stream version"); break; } - + // open string table PDB_StringTable strtab = {0}; MSF_StreamNumber strtab_sn = pdb_find_named_stream(&named_stream_ht, PDB_NAMES_STREAM_NAME); @@ -1757,9 +1764,9 @@ pdb_info_open(MSF_Context *msf, MSF_StreamNumber sn) Arena *arena = arena_alloc(); PDB_InfoContext *info = push_array_no_zero(arena, PDB_InfoContext, 1); info->arena = arena; - info->time_stamp = time_stamp; - info->age = age; - info->guid = guid; + info->time_stamp = parse.time_stamp; + info->age = parse.age; + info->guid = parse.guid; info->flags = flags; info->named_stream_ht = named_stream_ht; info->src_header_block_ht = src_header_block_ht; diff --git a/src/linker/pdb_ext/pdb_builder.h b/src/linker/pdb_ext/pdb_builder.h index bfe996c0..9cf859bd 100644 --- a/src/linker/pdb_ext/pdb_builder.h +++ b/src/linker/pdb_ext/pdb_builder.h @@ -150,6 +150,15 @@ typedef struct //////////////////////////////// // Info +typedef struct PDB_InfoParse +{ + PDB_TpiVersion version; + COFF_TimeStamp time_stamp; + U32 age; + OS_Guid guid; + String8 extra_info; +} PDB_InfoParse; + typedef struct PDB_InfoContext { Arena *arena; @@ -356,6 +365,7 @@ internal OS_Guid pdb_get_guid(PDB_Context *pdb); // Info internal PDB_InfoContext * pdb_info_alloc(U32 age, COFF_TimeStamp time_stamp, OS_Guid guid); +internal void pdb_info_parse_from_data(String8 data, PDB_InfoParse *parse_out); internal PDB_InfoContext * pdb_info_open(MSF_Context *msf, MSF_StreamNumber sn); internal void pdb_info_build(PDB_InfoContext *info, MSF_Context *msf, MSF_StreamNumber sn); internal void pdb_info_release(PDB_InfoContext **info_ptr);