From d34e1a7802ac431501a559cea0fe9cd35eb0627f Mon Sep 17 00:00:00 2001 From: Nikita Smith Date: Fri, 3 Jul 2026 00:26:27 -0700 Subject: [PATCH] implement /OPT:ICF Add identical COMDAT folding using a color-refinement over foldable read-only sections. Hash section contents, relocation targets, and target colors until partitions stabilize, then redirect folded COMDAT symlinks and discard duplicate sections. Honor .llvm_addrsig via /LLVM_ADDRSIG so address-significant sections are not folded, keep separate color space for code, unwind info, and MSVC vftables, and update COMDAT symbol/section mapping after folding. Add linker tests covering function folding, relocation-sensitive folds, alignment, color-space separation, symlink chains, .llvm_addrsig, pdata/xdata, and C++ ICF cases. --- src/linker/lnk.c | 763 ++++++++++++++--- src/linker/lnk.h | 14 +- src/linker/lnk_config.c | 8 + src/linker/lnk_config.h | 4 + src/linker/lnk_log.h | 1 + src/linker/lnk_obj.c | 49 +- src/linker/lnk_obj.h | 7 +- src/linker/lnk_symbol_table.c | 4 +- src/linker/tests/linker_tests.c | 1158 +++++++++++++++++++++++++- src/linker/thread_pool/thread_pool.c | 12 +- src/linker/thread_pool/thread_pool.h | 2 +- 11 files changed, 1869 insertions(+), 153 deletions(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index 58e0ae00..5f4de897 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -171,6 +171,8 @@ lnk_make_default_cmd_line(Arena *arena, LNK_CmdLine user_cmd_line) "/RAD_SORT_IMPORTS", (char*)str8f(scratch.arena, "/RAD_MT_PATH:%s", LNK_MANIFEST_MERGE_TOOL_NAME).str, (char*)str8f(scratch.arena, "/RAD_DATA_DIR_COUNT:%u", PE_DataDirectoryIndex_COUNT).str, + + "/LLVM_ADDRSIG", }; char *push_opts[] = { @@ -234,11 +236,10 @@ lnk_make_default_cmd_line(Arena *arena, LNK_CmdLine user_cmd_line) } // when /FORCE is specified on the command line, do not stop on these errors -#if 0 - if (lnk_cmd_line_has_switch(cmd_line, LNK_CmdSwitch_Force)) { + if (lnk_cmd_line_has_switch(user_cmd_line, LNK_CmdSwitch_Force)) { g_error_mode_arr[LNK_Error_UnresolvedSymbol] = LNK_ErrorMode_Continue; + g_error_mode_arr[LNK_Error_RelocationAgainstRemovedSection] = LNK_ErrorMode_Continue; } -#endif #undef DefaultOpt #undef PushOpt @@ -2397,19 +2398,31 @@ lnk_link_image(TP_Context *tp, TP_Arena *arena, LNK_Config *config, LNK_Inputer scratch_end(debug_scratch); } - // TODO: /FORCE - if (unresolved_symbols_count) { + if (unresolved_symbols_count && !config->force) { lnk_exit(LNK_Error_UnresolvedSymbol); } ProfEnd(); } - // - // discard COMDAT sections that are not referenced - // - if (config->opt_ref == LNK_SwitchState_Yes) { - lnk_opt_ref(tp, symtab, config, link->objs); + { + LNK_Obj **objs = 0; + + // + // discard COMDAT sections that are not referenced + // + if (config->opt_ref == LNK_SwitchState_Yes) { + if (objs == 0) { objs = lnk_array_from_obj_list(scratch.arena, link->objs); } + lnk_opt_ref(tp, symtab, config, objs, link->objs.count); + } + + // + // fold duplicate sections + // + if (config->opt_icf == LNK_SwitchState_Yes) { + if (objs == 0) { objs = lnk_array_from_obj_list(scratch.arena, link->objs); } + lnk_opt_icf(tp, symtab, config, objs, link->objs.count); + } } // @@ -2503,38 +2516,128 @@ lnk_reloc_ref_batch_list_concat_in_place_atomic(LNK_RelocRefsBatchList *list, LN } } +internal U32Array * +lnk_obj_indices_from_section_counts(Arena *arena, U64 worker_count, LNK_Obj **objs, U64 objs_count) +{ + Temp scratch = scratch_begin(&arena, 1); + + U64 *worker_section_counts = push_array(scratch.arena, U64, worker_count); + U64 *worker_obj_counts = push_array(scratch.arena, U64, worker_count); + U32Array *obj_indices = push_array(arena, U32Array, worker_count); + + for EachIndex(obj_idx, objs_count) { + U64 min_worker_idx = 0; + for (U64 worker_idx = 1; worker_idx < worker_count; worker_idx += 1) { + if (worker_section_counts[worker_idx] < worker_section_counts[min_worker_idx]) { + min_worker_idx = worker_idx; + } + } + + worker_section_counts[min_worker_idx] += objs[obj_idx]->header.section_count_no_null; + worker_obj_counts[min_worker_idx] += 1; + } + + for EachIndex(worker_idx, worker_count) { + obj_indices[worker_idx].v = push_array_no_zero(arena, U32, worker_obj_counts[worker_idx]); + } + + MemoryZero(worker_section_counts, sizeof(worker_section_counts[0])*worker_count); + MemoryZero(worker_obj_counts, sizeof(worker_obj_counts[0])*worker_count); + + for EachIndex(obj_idx, objs_count) { + U64 min_worker_idx = 0; + for (U64 worker_idx = 1; worker_idx < worker_count; worker_idx += 1) { + if (worker_section_counts[worker_idx] < worker_section_counts[min_worker_idx]) { + min_worker_idx = worker_idx; + } + } + + U32Array *worker_obj_indices = &obj_indices[min_worker_idx]; + worker_obj_indices->v[worker_obj_counts[min_worker_idx]++] = (U32)obj_idx; + worker_obj_indices->count += 1; + worker_section_counts[min_worker_idx] += objs[obj_idx]->header.section_count_no_null; + } + + scratch_end(scratch); + return obj_indices; +} + +internal B32 +lnk_resolve_reloc_target_symbol(Arena *arena, LNK_SymbolTable *symtab, LNK_ObjSymbolRef symbol, String8 pass_name, LNK_ObjSymbolRef *resolved_symbol_out) +{ + B32 is_resolved = 1; + + Temp temp = temp_begin(arena); + HashMap seen_hm = {0}; + B32 keep_walking = 1; + LNK_ObjSymbolRef result = symbol; + do { + // detect cyclic chains + U64 symbol_key = ((U64)result.obj->input_idx << 32ull) | (U64)result.symbol_idx; + if (hash_map_search_u64_u64(&seen_hm, symbol_key) == 0) { + hash_map_push_u64_u64(temp.arena, &seen_hm, symbol_key, 1); + } else { + COFF_ParsedSymbol symbol_parsed = lnk_parsed_symbol_from_coff_symbol_idx(symbol.obj, symbol.symbol_idx); + lnk_error_obj(LNK_Warning_CyclicSymbol, symbol.obj, "symbol %S forms a cyclic chain (%S)", symbol_parsed.name, pass_name); + MemoryZeroStruct(&result); + is_resolved = 0; + break; + } + + // unpack symbol + COFF_ParsedSymbol result_parsed = lnk_parsed_symbol_from_coff_symbol_idx(result.obj, result.symbol_idx); + COFF_SymbolValueInterpType result_interp = coff_interp_from_parsed_symbol(result_parsed); + + // resolve symbol + LNK_ObjSymbolRef next_ref = {0}; + if (lnk_resolve_symbol(symtab, result, &next_ref)) { + keep_walking = (result_interp == COFF_SymbolValueInterp_Weak || result_interp == COFF_SymbolValueInterp_Undefined); + result = next_ref; + } else { + keep_walking = 0; + } + } while (keep_walking); + + if (resolved_symbol_out) { + *resolved_symbol_out = result; + } + + temp_end(temp); + return is_resolved; +} + internal -THREAD_POOL_TASK_FUNC(lnk_walk_relocs_and_mark_ref_sections_task) +THREAD_POOL_TASK_FUNC(lnk_opt_ref_task) { ProfBeginFunction(); Temp scratch = scratch_begin(0,0); Temp scratch2 = scratch_begin(&scratch.arena, 1); - LNK_OptRefTask *task = raw_task; - LNK_SymbolTable *symtab = task->symtab; - LNK_Config *config = task->config; - LNK_ObjList objs = task->objs; + LNK_OptTask *task = raw_task; + LNK_SymbolTable *symtab = task->symtab; + LNK_Config *config = task->config; + LNK_Obj **objs = task->objs; + U64 objs_count = task->objs_count; - U8 **is_live = 0; - U64 *active_thread_count = 0; + U8 **is_live = 0; + U64 *active_thread_count = 0; LNK_RelocRefsBatchList *global_batch_list = 0; if (task_id == 0) { - active_thread_count = push_array(scratch.arena, U64, 1); + active_thread_count = push_array(scratch.arena, U64, 1); global_batch_list = push_array(scratch.arena, LNK_RelocRefsBatchList, 1); // alloc live flags and set live status on every non-COMDAT section - is_live = push_array_no_zero(scratch.arena, U8 *, objs.count); + is_live = push_array_no_zero(scratch.arena, U8 *, objs_count); { - U64 obj_idx = 0; - for EachNode(n, LNK_ObjNode, task->objs.first) { - is_live[obj_idx] = push_array(scratch.arena, U8, n->data.header.section_count_no_null + 1); + for EachIndex(obj_idx, objs_count) { + LNK_Obj *obj = objs[obj_idx]; - for EachIndex(sect_idx, n->data.header.section_count_no_null) { - is_live[obj_idx][sect_idx + 1] = !(n->data.section_flags[sect_idx] & COFF_SectionFlag_LnkCOMDAT); + is_live[obj_idx] = push_array(scratch.arena, U8, obj->header.section_count_no_null + 1); + + for EachIndex(sect_idx, obj->header.section_count_no_null) { + is_live[obj_idx][sect_idx + 1] = !(obj->section_flags[sect_idx] & COFF_SectionFlag_LnkCOMDAT); } - - obj_idx += 1; } } @@ -2561,8 +2664,9 @@ THREAD_POOL_TASK_FUNC(lnk_walk_relocs_and_mark_ref_sections_task) } // push task for every non-COMDAT section - for EachNode(obj_n, LNK_ObjNode, objs.first) { - LNK_Obj *obj = &obj_n->data; + for EachIndex(obj_idx, objs_count) { + LNK_Obj *obj = objs[obj_idx]; + for EachIndex(sect_idx, obj->header.section_count_no_null) { U32 section_number = sect_idx+1; COFF_SectionFlags section_flags = obj->section_flags[sect_idx]; @@ -2610,37 +2714,7 @@ THREAD_POOL_TASK_FUNC(lnk_walk_relocs_and_mark_ref_sections_task) // reloc -> symbol LNK_ObjSymbolRef ref_symbol = (LNK_ObjSymbolRef){ .obj = batch->v[i].obj, .symbol_idx = reloc->isymbol }; - { - Temp temp = temp_begin(scratch2.arena); - HashMap seen_hm = {0}; - B32 keep_walking = 1; - do { - // detect cyclic chains - U64 symbol_key = ((U64)ref_symbol.obj->input_idx << 32ull) | (U64)ref_symbol.symbol_idx; - if (hash_map_search_u64_u64(&seen_hm, symbol_key) == 0) { - hash_map_push_u64_u64(temp.arena, &seen_hm, symbol_key, 1); - } else { - COFF_ParsedSymbol reloc_parsed = lnk_parsed_symbol_from_coff_symbol_idx(batch->v[i].obj, reloc->isymbol); - lnk_error_obj(LNK_Warning_CyclicSymbol, batch->v[i].obj, "symbol %S forms a cyclic chain (/OPT:REF)", reloc_parsed.name); - MemoryZeroStruct(&ref_symbol); - break; - } - - // unpack symbol - COFF_ParsedSymbol ref_parsed = lnk_parsed_symbol_from_coff_symbol_idx(ref_symbol.obj, ref_symbol.symbol_idx); - COFF_SymbolValueInterpType ref_interp = coff_interp_from_parsed_symbol(ref_parsed); - - // resolve symbol - LNK_ObjSymbolRef next_ref = {0}; - if (lnk_resolve_symbol(symtab, ref_symbol, &next_ref)) { - keep_walking = (ref_interp == COFF_SymbolValueInterp_Weak || ref_interp == COFF_SymbolValueInterp_Undefined); - ref_symbol = next_ref; - } else { - keep_walking = 0; - } - } while (keep_walking); - temp_end(temp); - } + lnk_resolve_reloc_target_symbol(scratch2.arena, symtab, ref_symbol, str8_lit("/OPT:REF"), &ref_symbol); // skip unresolved symbol if (ref_symbol.obj == 0) { continue; } @@ -2674,7 +2748,7 @@ THREAD_POOL_TASK_FUNC(lnk_walk_relocs_and_mark_ref_sections_task) SLLStackPush(stack, stack_n); } - COFF_SectionFlags section_flags = ref_symbol.obj->section_flags[section_number-1]; + COFF_SectionFlags section_flags = ref_symbol.obj->section_flags[section_number-1]; // on first section visit, set live flag and enqueue section U8 was_visited = ins_atomic_u8_eval_assign(&is_live[ref_symbol.obj->input_idx][section_number], 1); @@ -2740,8 +2814,8 @@ THREAD_POOL_TASK_FUNC(lnk_walk_relocs_and_mark_ref_sections_task) if (task_id == 0) { ProfBegin("Remove Unreachable Sections"); - for EachNode(obj_n, LNK_ObjNode, objs.first) { - LNK_Obj *obj = &obj_n->data; + for EachIndex(obj_idx, objs_count) { + LNK_Obj *obj = objs[obj_idx]; for EachIndex(sect_idx, obj->header.section_count_no_null) { U32 section_number = sect_idx+1; COFF_SectionHeader *section_header = lnk_coff_section_header_from_section_number(obj, section_number); @@ -2756,8 +2830,8 @@ THREAD_POOL_TASK_FUNC(lnk_walk_relocs_and_mark_ref_sections_task) enum { Stat_Null, Stat_Code, Stat_Data, Stat_Debug, Stat_Count }; Stat stats[Stat_Count] = {0}; - for EachNode(obj_n, LNK_ObjNode, objs.first) { - LNK_Obj *obj = &obj_n->data; + for EachIndex(obj_idx, objs_count) { + LNK_Obj *obj = objs[obj_idx]; for EachIndex(sect_idx, obj->header.section_count_no_null) { U32 section_number = sect_idx+1; @@ -2813,14 +2887,503 @@ THREAD_POOL_TASK_FUNC(lnk_walk_relocs_and_mark_ref_sections_task) } internal void -lnk_opt_ref(TP_Context *tp, LNK_SymbolTable *symtab, LNK_Config *config, LNK_ObjList objs) +lnk_opt_ref(TP_Context *tp, LNK_SymbolTable *symtab, LNK_Config *config, LNK_Obj **objs, U64 objs_count) { - ProfScope("Mark Live Sections") - tp_for_parallel(tp, - 0, - tp->worker_count, - lnk_walk_relocs_and_mark_ref_sections_task, - &(LNK_OptRefTask){ .symtab = symtab, .config = config, .objs = objs }); + Temp scratch = scratch_begin(0,0); + U32Array *obj_indices = lnk_obj_indices_from_section_counts(scratch.arena, tp->worker_count, objs, objs_count); + + ProfScope("/OPT:REF") + { + LNK_OptTask task = { .symtab = symtab, .config = config, .objs = objs, .objs_count = objs_count, .obj_indices = obj_indices }; + tp_for_parallel(tp, 0, tp->worker_count, lnk_opt_ref_task, &task); + } + + scratch_end(scratch); +} + +typedef enum LNK_ICF_ColorSpace +{ + LNK_ICF_ColorSpace_Null, + LNK_ICF_ColorSpace_Code, + LNK_ICF_ColorSpace_Unwind, + LNK_ICF_ColorSpace_VFTable, + LNK_ICF_ColorSpace_COUNT, +} LNK_ICF_ColorSpace; + +internal String8 +lnk_string_from_icf_color_space(LNK_ICF_ColorSpace color_space) +{ + String8 result = str8_lit("Unknown"); + switch (color_space) { + case LNK_ICF_ColorSpace_Null: { result = str8_lit("Null"); } break; + case LNK_ICF_ColorSpace_Code: { result = str8_lit("Code"); } break; + case LNK_ICF_ColorSpace_Unwind: { result = str8_lit("Unwind"); } break; + case LNK_ICF_ColorSpace_VFTable: { result = str8_lit("VFTables"); } break; + case LNK_ICF_ColorSpace_COUNT: { result = str8_lit("Unknown"); } break; + } + return result; +} + +internal LNK_ICF_ColorSpace +lnk_icf_color_space_from_section(LNK_Obj *obj, U32 sect_idx) +{ + LNK_ICF_ColorSpace result = LNK_ICF_ColorSpace_Null; + + // fold read-only COMDAT sections + COFF_SectionFlags expected_flags = COFF_SectionFlag_LnkCOMDAT | COFF_SectionFlag_MemRead; + COFF_SectionFlags exclude_flags = COFF_SectionFlag_LnkRemove | COFF_SectionFlag_MemWrite | LNK_SECTION_FLAG_NOICF; + + if ((obj->section_flags[sect_idx] & expected_flags) == expected_flags && (obj->section_flags[sect_idx] & exclude_flags) == 0) { + // fold code + if (obj->section_flags[sect_idx] & COFF_SectionFlag_CntCode) { + result = LNK_ICF_ColorSpace_Code; + } + // fold data + else if (obj->section_flags[sect_idx] & COFF_SectionFlag_CntInitializedData) { + COFF_SectionHeader *section_header = lnk_coff_section_header_from_section_number(obj, sect_idx + 1); + String8 section_name = str8_cstring_capped(section_header->name, section_header->name + sizeof(section_header->name)); + + // fold unwind info + if (str8_match(section_name, str8_lit(".xdata"), 0) || str8_match(section_name, str8_lit(".pdata"), 0)) { + result = LNK_ICF_ColorSpace_Unwind; + } else { + // fold MSVC vftables separately from other read-only data + LNK_ObjSymbolRef symlink_ref = {0}; + if (lnk_obj_get_comdat_symlink(obj, sect_idx + 1, &symlink_ref)) { + COFF_ParsedSymbol symlink_symbol = lnk_parsed_symbol_from_coff_symbol_idx(symlink_ref.obj, symlink_ref.symbol_idx); + if (str8_starts_with(symlink_symbol.name, str8_lit("??_7"))) { + result = LNK_ICF_ColorSpace_VFTable; + } + } + } + } + } + + return result; +} + +internal +THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) +{ + ProfBeginFunction(); + Temp scratch = scratch_begin(&arena,1); + + // + // OPT uses a color-refinement algorithm for folding duplicate sections. + // If a color group contains multiple distinct hashes, the group is split + // and a new refinement pass is run. By default, the algorithm loops until + // partitions stabilize. Equivalence is established by comparing cryptographic + // 128-bit hashes; in theory, the chance of collisions are near the birthday + // paradox with BLAKE3 and the other downside it is susceptible to adversarial + // inputs. + // + + LNK_OptTask *task = raw_task; + LNK_Obj **objs = task->objs; + + if (task_id == 0) { + lnk_log(LNK_Log_Debug, "/OPT:ICF:"); + } + + if (task->config->llvm_addrsig == LNK_SwitchState_Yes) { + // .llvm_addrsig is an array of ULEB128 symbol indices, which mark sections + // whose addresses are significant + for EachIndex(i, task->obj_indices[task_id].count) { + U64 obj_idx = task->obj_indices[task_id].v[i]; + LNK_Obj *obj = task->objs[obj_idx]; + + if (obj->llvm_addrsig_sect_idx >= obj->header.section_count_no_null) { continue; } + + String8 symbol_table = lnk_coff_symbol_table_from_obj(obj); + String8 string_table = lnk_coff_string_table_from_obj(obj); + COFF_SectionHeader *section_header = lnk_coff_section_header_from_section_number(obj, obj->llvm_addrsig_sect_idx + 1); + String8 section_data = str8_substr(obj->data, r1u64s(section_header->foff, section_header->fsize)); + + // parse symbol indices and mark selected sections with NOICF flag + for (U64 off = 0; off < section_data.size;) { + U64 symbol_off = off; + U64 symbol_idx = 0; + off += str8_deserial_read_uleb128(section_data, off, &symbol_idx); + if (symbol_off == off) { break; } + + if (symbol_idx < obj->header.symbol_count) { + LNK_ObjSymbolRef target_ref = { .obj = obj, .symbol_idx = symbol_idx }; + B32 is_symbol_found = lnk_resolve_reloc_target_symbol(scratch.arena, task->symtab, target_ref, str8_lit("/OPT:ICF"), &target_ref); + if (is_symbol_found) { + COFF_ParsedSymbol symbol = coff_parse_symbol(target_ref.obj->header, string_table, symbol_table, target_ref.symbol_idx); + if (coff_interp_from_parsed_symbol(symbol) == COFF_SymbolValueInterp_Regular) { + obj->section_flags[symbol.section_number - 1] |= LNK_SECTION_FLAG_NOICF; + } else { + lnk_error_obj(LNK_Error_IllData, obj, ".llvm_addrsig: skip symbol 0x%x at offset 0x%x; symbol index must address a section-based symbol\n", symbol_idx, symbol_off); + } + } + } else { + lnk_error_obj(LNK_Error_IllData, obj, ".llvm_addrsig: contains out of bounds symbol index 0x%x at offset 0x%x\n", symbol_idx, symbol_off); + } + } + } + barrier_wait(tp->barrier); + } + + // + // step 1: fill out color map and contributions + // + + // alloc total section counter + U64 *contrib_counts = 0; + if (task_id == 0) { + contrib_counts = push_array(scratch.arena, U64, task->objs_count); + } + tp_broadcast(&contrib_counts); + + // count contributions + for EachIndex(i, task->obj_indices[task_id].count) { + U64 obj_idx = task->obj_indices[task_id].v[i]; + LNK_Obj *obj = objs[obj_idx]; + for EachIndex(sect_idx, obj->header.section_count_no_null) { + if (lnk_icf_color_space_from_section(obj, sect_idx)) { + contrib_counts[obj_idx] += 1; + } + } + } + barrier_wait(tp->barrier); + + typedef struct { + struct { + U128 hash; + U64 old_color; + } key; + U32 obj_idx; + U32 sect_idx; + } Contrib; + + U64 contrib_count = sum_array_u64(task->objs_count, contrib_counts); + U64 *contrib_offsets = 0; + U64 **color_map = 0; + Contrib *contribs = 0; + U32 *is_part_stable = 0; + U64 *next_color = 0; + if (task_id == 0) { + contrib_offsets = offsets_from_counts_array_u64(scratch.arena, contrib_counts, task->objs_count); + color_map = push_array(scratch.arena, U64 *, task->objs_count); + for EachIndex(obj_idx, task->objs_count) { + LNK_Obj *obj = objs[obj_idx]; + color_map[obj_idx] = push_array(scratch.arena, U64, obj->header.section_count_no_null); + } + contribs = push_array(scratch.arena, Contrib, contrib_count); + is_part_stable = push_array(scratch.arena, U32, 1); + next_color = push_array(scratch.arena, U64, 1); + *next_color = LNK_ICF_ColorSpace_COUNT; + + lnk_log(LNK_Log_Debug, " Contrib count: %S", str8_from_count(scratch.arena, contrib_count)); + } + tp_broadcast(&contrib_offsets); + tp_broadcast(&color_map); + tp_broadcast(&contribs); + tp_broadcast(&is_part_stable); + tp_broadcast(&next_color); + + for EachIndex(i, task->obj_indices[task_id].count) { + U64 obj_idx = task->obj_indices[task_id].v[i]; + LNK_Obj *obj = objs[obj_idx]; + U64 cursor = 0; + for EachIndex(sect_idx, obj->header.section_count_no_null) { + LNK_ICF_ColorSpace color_space = lnk_icf_color_space_from_section(obj, sect_idx); + if (color_space == LNK_ICF_ColorSpace_Null) { + // assign unique color to sections that are not foldable + color_map[obj_idx][sect_idx] = ins_atomic_u64_inc_eval(next_color); + } + else { + // compute contribution index + U64 contrib_idx = contrib_offsets[obj_idx] + cursor++; + + // seed foldable sections with a content-derived color + COFF_SectionHeader *section_header = lnk_coff_section_header_from_section_number(obj, sect_idx + 1); + String8 section_data = str8_substr(obj->data, r1u64s(section_header->foff, section_header->fsize)); + XXH3_state_t hasher; + XXH3_64bits_reset(&hasher); + XXH3_64bits_update(&hasher, &color_space, sizeof(color_space)); + XXH3_64bits_update(&hasher, section_data.str, section_data.size); + + // fill out contribution + contribs[contrib_idx] = (Contrib){ + .obj_idx = safe_cast_u32(obj->input_idx), + .sect_idx = safe_cast_u32(sect_idx), + }; + + // assign content-derived starting color value + color_map[obj_idx][sect_idx] = XXH3_64bits_digest(&hasher) | (1ull << 63); + } + } + } + barrier_wait(tp->barrier); + + // + // step 2: refine equivalence classes + // + + U64 iter_count = 0; + for (;; iter_count += 1) { + barrier_wait(tp->barrier); + // reset color status tracker + if (task_id == 0) { + *is_part_stable = 1; + } + barrier_wait(tp->barrier); + + // compute colored hashes + for EachIndex(i, task->obj_indices[task_id].count) { + U64 obj_idx = task->obj_indices[task_id].v[i]; + LNK_Obj *obj = objs[obj_idx]; + COFF_SectionHeader *section_table = lnk_coff_section_table_from_obj(obj); + U64 contrib_cursor = 0; + for EachIndex(sect_idx, obj->header.section_count_no_null) { + LNK_ICF_ColorSpace color_space = lnk_icf_color_space_from_section(obj, sect_idx); + if (color_space == LNK_ICF_ColorSpace_Null) { continue; } + + // parse relocations + COFF_SectionHeader *section_header = section_table + sect_idx; + COFF_RelocArray relocs = lnk_coff_relocs_from_section_header(obj, section_header); + + U64 contrib_idx = contrib_offsets[obj_idx] + contrib_cursor; + contrib_cursor += 1; + contribs[contrib_idx].key.old_color = color_map[obj_idx][sect_idx]; + + // parse section data + String8 section_data = str8_substr(obj->data, r1u64s(section_header->foff, section_header->fsize)); + + blake3_hasher hasher; blake3_hasher_init(&hasher); + + // mix section non-recursive properties + blake3_hasher_update(&hasher, section_data.str, section_data.size); + blake3_hasher_update(&hasher, &relocs.count, sizeof(relocs.count)); + blake3_hasher_update(&hasher, &color_space, sizeof(color_space)); + + for EachIndex(reloc_idx, relocs.count) { + COFF_Reloc *r = &relocs.v[reloc_idx]; + + // resolve symbol referenced by the relocation + LNK_ObjSymbolRef target_ref = { .obj = obj, .symbol_idx = r->isymbol }; + B32 is_symbol_found = lnk_resolve_reloc_target_symbol(scratch.arena, task->symtab, target_ref, str8_lit("/OPT:ICF"), &target_ref); + + COFF_SymbolValueInterpType target_interp; + U64 target_id; + U32 target_value; + + if (is_symbol_found) { + // parse COFF symbol and interpret the target symbol kind + COFF_ParsedSymbol target_symbol = lnk_parsed_symbol_from_coff_symbol_idx(target_ref.obj, target_ref.symbol_idx); + target_interp = coff_interp_from_parsed_symbol(target_symbol); + + switch (target_interp) { + case COFF_SymbolValueInterp_Regular: { + // color relocation with the referenced section + target_id = color_map[target_ref.obj->input_idx][target_symbol.section_number - 1]; + target_value = target_symbol.value; + } break; + default: { + // color relocation with unique symbol ref + target_id = Compose64Bit(target_ref.obj->input_idx, target_ref.symbol_idx); + target_value = target_symbol.value; + } break; + } + } else { + // color relocation with unique symbol ref + target_interp = max_U32; + target_id = Compose64Bit(obj_idx, r->isymbol); + target_value = 0; + } + + // mix relocation properties + blake3_hasher_update(&hasher, &r->apply_off, sizeof(r->apply_off)); + blake3_hasher_update(&hasher, &r->type, sizeof(r->type)); + blake3_hasher_update(&hasher, &target_interp, sizeof(target_interp)); + blake3_hasher_update(&hasher, &target_id, sizeof(target_id)); + blake3_hasher_update(&hasher, &target_value, sizeof(target_value)); + } + + // finalize section hash + U128 hash; + blake3_hasher_finalize(&hasher, (U8*)&hash, sizeof(hash)); + + // update contribution hash and color + contribs[contrib_idx].key.hash = hash; + } + } + barrier_wait(tp->barrier); + + // group sections by (color, hash) and handle color splits + if (task_id == 0) { + Temp temp = temp_begin(scratch.arena); + + HashMap new_color_hm = {0}; + HashMap old_color_hm = {0}; + U64 start_next_color = *next_color; + + for EachIndex(contrib_idx, contrib_count) { + Contrib *contrib = &contribs[contrib_idx]; + U64 *new_color_ptr = hash_map_search_string_u64(&new_color_hm, str8_struct(&contrib->key)); + if (new_color_ptr == 0) { + U64 color; + if (hash_map_search_u64_u64(&old_color_hm, contrib->key.old_color) == 0) { + color = contrib->key.old_color; + hash_map_push_u64_u64(temp.arena, &old_color_hm, contrib->key.old_color, color); + } else { + // generate a new color for the hash + color = ++*next_color; + *is_part_stable = 0; + } + new_color_ptr = &hash_map_push_string_u64(temp.arena, &new_color_hm, str8_struct(&contrib->key), color)->v.value.value_u64; + } + color_map[contrib->obj_idx][contrib->sect_idx] = *new_color_ptr; + } + + U64 split_count = *next_color - start_next_color; + lnk_log(LNK_Log_Debug, " Pass %llu found %S splits", iter_count, str8_from_count(scratch.arena, split_count)); + + temp_end(temp); + } + barrier_wait(tp->barrier); + + // stop iterating when partitions stabilize + if (*is_part_stable) { break; } + } + barrier_wait(tp->barrier); + + // + // step 3: flag folded sections for removal + // + + if (task_id == 0) { + HashMap leader_hm = { 0 }; + for EachIndex(contrib_idx, contrib_count) { + Contrib *contrib = &contribs[contrib_idx]; + U64 color = color_map[contrib->obj_idx][contrib->sect_idx]; + if (hash_map_search_u64_raw(&leader_hm, color) == 0) { + hash_map_push_u64_raw(scratch.arena, &leader_hm, color, contrib); + } + } + + typedef struct { U64 count; U64 size; } FoldStats; + FoldStats fold_stats[LNK_ICF_ColorSpace_COUNT] = {0}; + + for EachIndex(contrib_idx, contrib_count) { + Contrib *contrib = &contribs[contrib_idx]; + Contrib *leader = hash_map_search_u64_raw(&leader_hm, color_map[contrib->obj_idx][contrib->sect_idx]); + if (leader == 0 || leader == contrib) { continue; } + + LNK_Obj *contrib_obj = objs[contrib->obj_idx]; + LNK_Obj *leader_obj = objs[leader->obj_idx]; + + // update fold stats + if (lnk_get_log_status(LNK_Log_Debug)) { + COFF_SectionHeader *section_header = lnk_coff_section_header_from_section_number(contrib_obj, contrib->sect_idx + 1); + LNK_ICF_ColorSpace color_space = lnk_icf_color_space_from_section(contrib_obj, contrib->sect_idx); + fold_stats[color_space].count += 1; + fold_stats[color_space].size += section_header->fsize; + } + + // update leader with largest alignment + U64 leader_align = coff_align_size_from_section_flags(leader_obj->section_flags[leader->sect_idx]); + U64 contrib_align = coff_align_size_from_section_flags(contrib_obj->section_flags[contrib->sect_idx]); + if (leader_align < contrib_align) { + leader_obj->section_flags[leader->sect_idx] &= ~(COFF_SectionFlag_AlignMask << COFF_SectionFlag_AlignShift); + leader_obj->section_flags[leader->sect_idx] |= coff_section_flag_from_align_size(contrib_align); + } + + // update COMDAT leader symbol link + Assert(leader_obj->comdats[leader->sect_idx] != max_U32); + LNK_ObjSymbolRef leader_symlink = { leader_obj, leader_obj->comdats[leader->sect_idx] }; + contrib_obj->symlinks[contrib->sect_idx + 1] = leader_symlink; + + // discard folded section + contrib_obj->section_flags[contrib->sect_idx] |= COFF_SectionFlag_LnkRemove; + + #if LNK_PARANOID + String8 section_name = lnk_obj_section_name_from_section_number(contrib_obj, contrib->sect_idx+1); + String8 leader_name = lnk_obj_section_name_from_section_number(leader_obj, leader->sect_idx+1); + lnk_log(LNK_Log_Debug, "fold %.*s[SECT%X \"%.*s\"] ==> %.*s[SECT%X \"%.*s\"]", str8_varg(lnk_loc_from_obj(scratch.arena, contrib_obj)), contrib->sect_idx+1, str8_varg(section_name), str8_varg(lnk_loc_from_obj(scratch.arena, leader_obj)), leader->sect_idx+1, str8_varg(leader_name)); + #endif + } + + if (lnk_get_log_status(LNK_Log_Debug)) { + U64 total_count = 0, total_size = 0; + for (LNK_ICF_ColorSpace color_space = (LNK_ICF_ColorSpace)(LNK_ICF_ColorSpace_Null + 1); color_space < LNK_ICF_ColorSpace_COUNT; color_space += 1) { + lnk_log(LNK_Log_Debug, " %-8S: %M, %.*s sections", lnk_string_from_icf_color_space(color_space), fold_stats[color_space].size, str8_varg(str8_from_count(scratch.arena, fold_stats[color_space].count))); + total_count += fold_stats[color_space].count; + total_size += fold_stats[color_space].size; + } + lnk_log(LNK_Log_Debug, " %-8s: %M, %.*s sections", "Total", total_size, str8_varg(str8_from_count(scratch.arena, total_count))); + } + } + barrier_wait(tp->barrier); + + // + // step 4: flatten COMDAT symlink chains so subsequent passes can assume symlinks are single hop + // + + for EachIndex(i, task->obj_indices[task_id].count) { + U64 obj_idx = task->obj_indices[task_id].v[i]; + LNK_Obj *obj = objs[obj_idx]; + for EachIndex(sect_idx, obj->header.section_count_no_null) { + U64 section_number = sect_idx + 1; + + LNK_ObjSymbolRef symlink_ref = {0}; + if (!lnk_obj_get_comdat_symlink(obj, section_number, &symlink_ref)) { continue; } + + { + Temp temp = temp_begin(scratch.arena); + + HashMap seen_hm = {0}; + U64 hop_count = 0; + U64 hop_cap = 1024; + for(; hop_count < hop_cap; hop_count += 1) { + COFF_ParsedSymbol symlink_parsed = lnk_parsed_symbol_from_coff_symbol_idx(symlink_ref.obj, symlink_ref.symbol_idx); + + LNK_ObjSymbolRef next_symlink_ref = {0}; + if (!lnk_obj_get_comdat_symlink(symlink_ref.obj, symlink_parsed.section_number, &next_symlink_ref)) { break; } + + if (MemoryMatchStruct(&next_symlink_ref, &symlink_ref)) { break; } + + if (hash_map_search_string_u64(&seen_hm, str8_struct(&next_symlink_ref)) != 0) { + lnk_error_obj(LNK_Error_IllData, obj, "recursive COMDAT symlink in SECT%X", section_number); + MemoryZeroStruct(&symlink_ref); + break; + } + + symlink_ref = next_symlink_ref; + hash_map_push_string_u64(temp.arena, &seen_hm, str8_copy(temp.arena, str8_struct(&symlink_ref)), 1); + } + if (hop_count >= hop_cap) { + lnk_error_obj(LNK_Error_IllData, obj, "failed to flatten symlink for SECT%X; max number of hops reached", section_number); + MemoryZeroStruct(&symlink_ref); + } + + temp_end(temp); + } + + obj->symlinks[section_number] = symlink_ref; + } + } + barrier_wait(tp->barrier); + + scratch_end(scratch); + ProfEnd(); +} + +internal void +lnk_opt_icf(TP_Context *tp, LNK_SymbolTable *symtab, LNK_Config *config, LNK_Obj **objs, U64 objs_count) +{ + Temp scratch = scratch_begin(0,0); + U32Array *obj_indices = lnk_obj_indices_from_section_counts(scratch.arena, tp->worker_count, objs, objs_count); + + ProfScope("/OPT:ICF") + { + LNK_OptTask task = { .symtab = symtab, .config = config, .objs = objs, .objs_count = objs_count, .obj_indices = obj_indices }; + tp_for_parallel(tp, 0, tp->worker_count, lnk_opt_icf_task, &task); + } + + scratch_end(scratch); } internal @@ -2935,11 +3498,10 @@ THREAD_POOL_TASK_FUNC(lnk_set_comdat_leaders_contribs_task) if (~obj->section_flags[sect_idx] & COFF_SectionFlag_LnkCOMDAT) { continue; } - LNK_Symbol *symlink = lnk_obj_get_comdat_symlink(obj, section_number); - if (symlink == 0) { continue; } + LNK_ObjSymbolRef symlink_ref = {0}; + if ( ! lnk_obj_get_comdat_symlink(obj, section_number, &symlink_ref)) { continue; } - COFF_ParsedSymbol symlink_parsed = lnk_parsed_from_symbol(symlink); - LNK_ObjSymbolRef symlink_ref = lnk_ref_from_symbol(symlink); + COFF_ParsedSymbol symlink_parsed = lnk_parsed_symbol_from_coff_symbol_idx(symlink_ref.obj, symlink_ref.symbol_idx); task->sect_map[obj_idx][sect_idx] = task->sect_map[symlink_ref.obj->input_idx][symlink_parsed.section_number - 1]; } ProfEnd(); @@ -2981,35 +3543,30 @@ THREAD_POOL_TASK_FUNC(lnk_patch_comdat_leaders_task) symbol = lnk_parsed_symbol_from_coff_symbol_idx(obj, symbol_idx); COFF_SymbolValueInterpType interp = coff_interp_symbol(symbol.section_number, symbol.value, symbol.storage_class); - if (interp == COFF_SymbolValueInterp_Regular) { - LNK_Symbol *symlink = lnk_obj_get_comdat_symlink(obj, symbol.section_number); - if (symlink) { - LNK_ObjSymbolRef symlink_ref = lnk_ref_from_symbol(symlink); - if (symlink_ref.obj != obj) { - U32 section_number; - U32 value; - if (symbol.storage_class == COFF_SymStorageClass_External) { - // COMDAT leader may be at a different offset, so update this symbol with leader's offset - COFF_ParsedSymbol parsed_symlink = lnk_parsed_from_symbol(symlink); - section_number = symbol.section_number; - value = parsed_symlink.value; - } else { - // COMDAT section may have static symbols which are now invalid to relocate against - section_number = lnk_obj_get_removed_section_number(obj); - value = max_U32; - task->u.patch_symtabs.was_symbol_patched[obj_idx][symbol_idx] = 1; - } + if (interp != COFF_SymbolValueInterp_Regular) { continue; } - if (obj->header.is_big_obj) { - COFF_Symbol32 *symbol32 = symbol.raw_symbol; - symbol32->section_number = section_number; - symbol32->value = value; - } else { - COFF_Symbol16 *symbol16 = symbol.raw_symbol; - symbol16->section_number = (U16)section_number; - symbol16->value = value; - } - } + LNK_ObjSymbolRef symlink_ref = {0}; + if ( ! lnk_obj_get_comdat_symlink(obj, symbol.section_number, &symlink_ref)) { continue; } + + COFF_ParsedSymbol parsed_symlink = lnk_parsed_symbol_from_coff_symbol_idx(symlink_ref.obj, symlink_ref.symbol_idx); + if (symlink_ref.obj == obj && parsed_symlink.section_number == symbol.section_number) { continue; } + + B32 is_static_comdat_leader = symbol.storage_class == COFF_SymStorageClass_Static && + obj->comdats[symbol.section_number-1] == symbol_idx; + + if (symbol.storage_class == COFF_SymStorageClass_External || is_static_comdat_leader) { + // COMDAT leader may be at a different offset, so update this symbol with leader's offset + U32 section_number = symbol.section_number; + U32 value = parsed_symlink.value; + + if (obj->header.is_big_obj) { + COFF_Symbol32 *symbol32 = symbol.raw_symbol; + symbol32->section_number = section_number; + symbol32->value = value; + } else { + COFF_Symbol16 *symbol16 = symbol.raw_symbol; + symbol16->section_number = (U16)section_number; + symbol16->value = value; } } } diff --git a/src/linker/lnk.h b/src/linker/lnk.h index d6388d0a..b6a154ca 100644 --- a/src/linker/lnk.h +++ b/src/linker/lnk.h @@ -74,6 +74,7 @@ typedef struct LNK_Inputer #define LNK_NULL_SYMBOL "*** RAD_NULL_SYMBOL ***" #define LNK_SECTION_FLAG_DEBUG (1 << 0) +#define LNK_SECTION_FLAG_NOICF (1 << 1) typedef U8 LNK_LibMemberFlags; enum @@ -215,10 +216,12 @@ typedef struct typedef struct { - LNK_SymbolTable *symtab; - LNK_Config *config; - LNK_ObjList objs; -} LNK_OptRefTask; + LNK_SymbolTable *symtab; + LNK_Config *config; + LNK_Obj **objs; + U64 objs_count; + U32Array *obj_indices; +} LNK_OptTask; typedef struct { @@ -390,7 +393,8 @@ internal LNK_LinkResult lnk_link_image (TP_Context *tp, TP_Arena *arena, LNK_Con // --- Optimizations ----------------------------------------------------------- -internal void lnk_opt_ref(TP_Context *tp, LNK_SymbolTable *symtab, LNK_Config *config, LNK_ObjList objs); +internal void lnk_opt_ref(TP_Context *tp, LNK_SymbolTable *symtab, LNK_Config *config, LNK_Obj **objs, U64 objs_count); +internal void lnk_opt_icf(TP_Context *tp, LNK_SymbolTable *symtab, LNK_Config *config, LNK_Obj **objs, U64 objs_count); // --- Win32 Image ------------------------------------------------------------- diff --git a/src/linker/lnk_config.c b/src/linker/lnk_config.c index ac624c0e..f4dc456e 100644 --- a/src/linker/lnk_config.c +++ b/src/linker/lnk_config.c @@ -24,6 +24,7 @@ global read_only LNK_CmdSwitch g_cmd_switch_map[] = { LNK_CmdSwitch_FailIfMismatch, 1, "FAILIFMISMATCH", "{id=value}", "Fails to link if same ids have conflicting values." }, { LNK_CmdSwitch_FileAlign, 0, "FILEALIGN", ":#", "Set section alignment in the file." }, { LNK_CmdSwitch_Fixed, 0, "FIXED", "[:NO]", "Load the image at the default base address." }, + { LNK_CmdSwitch_Force, 0, "FORCE", "", "Force image output despite errors." }, { LNK_CmdSwitch_FunctionPadMin, 0, "FUNCTIONPADMIN", ":#", "Minimum function byte size." }, { LNK_CmdSwitch_Heap, 0, "HEAP", "RESERVE[,COMMIT]", "Set reserve and commit size for the heap." }, { LNK_CmdSwitch_HighEntropyVa, 0, "HIGHENTROPYVA", "[:NO]", "Indicate that image supports full 64-bit address space ASLR." }, @@ -106,6 +107,8 @@ global read_only LNK_CmdSwitch g_cmd_switch_map[] = { LNK_CmdSwitch_RadTypeServer, 0, "RAD_TYPE_SERVER", ":FILENAME", "Merge types and store them in the specified file. The filename must have the .rrt extension." }, + { LNK_CmdSwitch_LLVM_AddrSig, 0, "LLVM_ADDRSIG", "[:NO]", "Use .llvm_addrsig to guide ICF." }, + { LNK_CmdSwitch_Help, 0, "HELP", "", "" }, { LNK_CmdSwitch_Help, 0, "?", "", "" }, }; @@ -2179,6 +2182,10 @@ lnk_apply_cmd_option_to_config(LNK_Config *config, String8 cmd_name, String8List lnk_error_cmd_switch(LNK_Error_Cmdl, obj, cmd_switch, "missing type server file path"); } } break; + + case LNK_CmdSwitch_LLVM_AddrSig: { + lnk_cmd_switch_parse_flag(obj, cmd_switch, value_strings, &config->llvm_addrsig); + } break; } scratch_end(scratch); @@ -2208,6 +2215,7 @@ lnk_config_init(LNK_CmdLine cmd_line) config->arena = arena; config->raw_cmd_line = str8_list_copy(arena, &cmd_line.raw_cmd_line); config->work_dir = get_current_path(arena); + config->force = lnk_cmd_line_has_switch(cmd_line, LNK_CmdSwitch_Force); // apply command line switches for EachNode(cmd, LNK_CmdOption, cmd_line.first_option) { diff --git a/src/linker/lnk_config.h b/src/linker/lnk_config.h index a781d664..67ae4c3f 100644 --- a/src/linker/lnk_config.h +++ b/src/linker/lnk_config.h @@ -143,6 +143,8 @@ typedef enum LNK_CmdSwitch_RadTypeServer, LNK_CmdSwitch_RadTypeServer_MatchObj, + LNK_CmdSwitch_LLVM_AddrSig, + LNK_CmdSwitch_Help, LNK_CmdSwitch_Count @@ -316,6 +318,7 @@ typedef struct LNK_Config U64 function_pad_min; U64 *manifest_resource_id; B32 no_default_libs; + B32 force; LNK_SwitchState infer_asan_libs; Version link_ver; Version os_ver; @@ -387,6 +390,7 @@ typedef struct LNK_Config String8 type_server_name; LNK_SwitchState type_server; LNK_SwitchState sort_imports; + LNK_SwitchState llvm_addrsig; } LNK_Config; // --- MSVC Error Codes -------------------------------------------------------- diff --git a/src/linker/lnk_log.h b/src/linker/lnk_log.h index 8c087009..b10c836b 100644 --- a/src/linker/lnk_log.h +++ b/src/linker/lnk_log.h @@ -24,6 +24,7 @@ typedef enum LNK_Log_Count } LNK_LogType; +// TODO: factor into an xlist with explicitly defined error levels and warnings typedef enum { LNK_Error_Null, diff --git a/src/linker/lnk_obj.c b/src/linker/lnk_obj.c index c2f9649e..c3144acf 100644 --- a/src/linker/lnk_obj.c +++ b/src/linker/lnk_obj.c @@ -100,7 +100,7 @@ THREAD_POOL_TASK_FUNC(lnk_obj_initer) COFF_SectionFlags *section_flags = push_array_no_zero(arena, COFF_SectionFlags, header.section_count_no_null); for (U64 sect_idx = 0; sect_idx < header.section_count_no_null; sect_idx += 1) { COFF_SectionHeader *coff_sect_header = &coff_section_table[sect_idx]; - section_flags[sect_idx] = coff_sect_header->flags; + section_flags[sect_idx] = coff_sect_header->flags & ~3; // linker reserves low 2 bits for internal flags String8 sect_name = coff_name_from_section_header(raw_coff_string_table, coff_sect_header); if (~section_flags[sect_idx] & COFF_SectionFlag_CntUninitializedData) { if (coff_sect_header->fsize > 0) { @@ -344,6 +344,7 @@ THREAD_POOL_TASK_FUNC(lnk_obj_initer) obj->debug_t_sect_idx = ~0; obj->debug_p_sect_idx = ~0; obj->debug_h_sect_idx = ~0; + obj->llvm_addrsig_sect_idx = ~0; } internal @@ -362,6 +363,19 @@ THREAD_POOL_TASK_FUNC(lnk_obj_find_debug_t) } } +internal +THREAD_POOL_TASK_FUNC(lnk_obj_find_llvm_addrsig) +{ + LNK_Obj *obj = &((LNK_ObjNode *)raw_task)[task_id].data; + for EachIndex(sect_idx, obj->header.section_count_no_null) { + String8 section_name = lnk_obj_section_name_from_sect_idx(obj, sect_idx); + if (str8_match(section_name, str8_lit(".llvm_addrsig"), 0)) { + obj->llvm_addrsig_sect_idx = sect_idx; + break; + } + } +} + internal LNK_ObjNode * lnk_obj_from_input_many(TP_Context *tp, TP_Arena *arena, LNK_Config *config, U64 inputs_count, LNK_Input **inputs) { @@ -369,10 +383,12 @@ lnk_obj_from_input_many(TP_Context *tp, TP_Arena *arena, LNK_Config *config, U64 if (inputs_count) { objs = push_array(arena->v[0], LNK_ObjNode, inputs_count); tp_for_parallel(tp, arena, inputs_count, lnk_obj_initer, &(LNK_ObjIniter){ .inputs = inputs, .objs = objs, .machine = config->machine }); - if (lnk_do_debug_info(config)) { tp_for_parallel(tp, arena, inputs_count, lnk_obj_find_debug_t, objs); } + if (config->opt_icf == LNK_SwitchState_Yes) { + tp_for_parallel(tp, arena, inputs_count, lnk_obj_find_llvm_addrsig, objs); + } } return objs; } @@ -451,19 +467,26 @@ THREAD_POOL_TASK_FUNC(lnk_input_coff_symbol_table) } } -internal LNK_SymbolHashTrie ** +internal LNK_ObjSymbolRef * lnk_symlinks_from_obj(Arena *arena, LNK_SymbolTable *symtab, LNK_Obj *obj) { - LNK_SymbolHashTrie **symlinks = push_array(arena, LNK_SymbolHashTrie *, obj->header.section_count_no_null+1); + LNK_ObjSymbolRef *symlinks = push_array(arena, LNK_ObjSymbolRef, obj->header.section_count_no_null+1); COFF_ParsedSymbol symbol; for (U64 symbol_idx = 0; symbol_idx < obj->header.symbol_count; symbol_idx += (1 + symbol.aux_symbol_count)) { symbol = lnk_parsed_symbol_from_coff_symbol_idx(obj, symbol_idx); COFF_SymbolValueInterpType interp = coff_interp_symbol(symbol.section_number, symbol.value, symbol.storage_class); - if (interp == COFF_SymbolValueInterp_Regular && symbol.aux_symbol_count == 0 && symbol.storage_class == COFF_SymStorageClass_External) { + if (interp == COFF_SymbolValueInterp_Regular) { LNK_ObjSection section = lnk_obj_section_from_section_number(obj, symbol.section_number); if (*section.flags & COFF_SectionFlag_LnkCOMDAT) { - if (symlinks[symbol.section_number] == 0 || symbol.value == 0) { - symlinks[symbol.section_number] = lnk_symbol_table_search_(symtab, symbol.name); + if (symbol.aux_symbol_count == 0 && symbol.storage_class == COFF_SymStorageClass_External) { + if (symlinks[symbol.section_number].obj == 0 || symbol.value == 0) { + LNK_SymbolHashTrie *link_symbol = lnk_symbol_table_search_(symtab, symbol.name); + if (link_symbol) { + symlinks[symbol.section_number] = lnk_ref_from_symbol(link_symbol->symbol); + } + } + } else if (symlinks[symbol.section_number].obj == 0 && symbol.storage_class == COFF_SymStorageClass_Static && symbol.aux_symbol_count > 0) { + symlinks[symbol.section_number] = (LNK_ObjSymbolRef){ obj, symbol_idx }; } } } @@ -543,11 +566,15 @@ lnk_obj_get_removed_section_number(LNK_Obj *obj) return obj->header.is_big_obj ? LNK_REMOVED_SECTION_NUMBER_32 : LNK_REMOVED_SECTION_NUMBER_16; } -internal LNK_Symbol * -lnk_obj_get_comdat_symlink(LNK_Obj *obj, U64 section_number) +internal B32 +lnk_obj_get_comdat_symlink(LNK_Obj *obj, U64 section_number, LNK_ObjSymbolRef *symlink_out) { - LNK_SymbolHashTrie *symlink = obj->symlinks[section_number]; - return symlink ? symlink->symbol : 0; + LNK_ObjSymbolRef symlink = obj->symlinks[section_number]; + B32 is_valid = symlink.obj != 0; + if (is_valid && symlink_out) { + *symlink_out = symlink; + } + return is_valid; } internal COFF_SectionHeader * diff --git a/src/linker/lnk_obj.h b/src/linker/lnk_obj.h index 9a21de59..339f2152 100644 --- a/src/linker/lnk_obj.h +++ b/src/linker/lnk_obj.h @@ -22,7 +22,7 @@ typedef struct LNK_Obj // COMDAT U32 *comdats; U32Node **associated_sections; - LNK_SymbolHashTrie **symlinks; + LNK_ObjSymbolRef *symlinks; // link struct LNK_LibMemberRef *link_member; @@ -33,6 +33,9 @@ typedef struct LNK_Obj U32 debug_p_sect_idx; U32 debug_h_sect_idx; + // ICF + U32 llvm_addrsig_sect_idx; + // @type_server Rng1U64 ti_range; CV_TypeIndex *ti_map; @@ -139,7 +142,7 @@ internal U32 lnk_obj_get_vol_md(LNK_Obj *obj); internal struct LNK_Lib * lnk_obj_get_lib(LNK_Obj *obj); internal String8 lnk_obj_get_lib_path(LNK_Obj *obj); internal U32 lnk_obj_get_removed_section_number(LNK_Obj *obj); -internal LNK_Symbol * lnk_obj_get_comdat_symlink(LNK_Obj *obj, U64 section_number); +internal B32 lnk_obj_get_comdat_symlink(LNK_Obj *obj, U64 section_number, LNK_ObjSymbolRef *symlink_out); // --- Symbol & Section Helpers ------------------------------------------------ diff --git a/src/linker/lnk_symbol_table.c b/src/linker/lnk_symbol_table.c index 8cb6c4e5..1c4a7655 100644 --- a/src/linker/lnk_symbol_table.c +++ b/src/linker/lnk_symbol_table.c @@ -717,8 +717,8 @@ lnk_resolve_symbol(LNK_SymbolTable *symtab, LNK_ObjSymbolRef symbol, LNK_ObjSymb COFF_SymbolValueInterpType symbol_interp = coff_interp_symbol(symbol_parsed.section_number, symbol_parsed.value, symbol_parsed.storage_class); switch (symbol_interp) { case COFF_SymbolValueInterp_Regular: { - LNK_Symbol *symlink = lnk_obj_get_comdat_symlink(symbol.obj, symbol_parsed.section_number); - *symbol_out = symlink ? lnk_ref_from_symbol(symlink) : symbol; + LNK_ObjSymbolRef symlink = {0}; + *symbol_out = lnk_obj_get_comdat_symlink(symbol.obj, symbol_parsed.section_number, &symlink) ? symlink : symbol; } break; case COFF_SymbolValueInterp_Weak: { LNK_Symbol *defn = lnk_symbol_table_search(symtab, symbol_parsed.name); diff --git a/src/linker/tests/linker_tests.c b/src/linker/tests/linker_tests.c index 3221abe0..bd1c8d56 100644 --- a/src/linker/tests/linker_tests.c +++ b/src/linker/tests/linker_tests.c @@ -3,15 +3,15 @@ // TODO: // [x] defer_duplicate_imp_link +// [x] fold_two_funcs +// [x] same_but_different +// [x] fold_diamond +// [x] cyclic_icf +// [x] fold_with_largest_align // [ ] opt_ref_comdat_undef_section // [ ] opt_ref_weak_alias_comdat // [ ] reloc_apply_off_out_of_bounds // [ ] lib_member_reloc_apply_off_out_of_bounds -// [ ] fold_two_funcs -// [ ] same_but_different -// [ ] fold_diamond -// [ ] cyclic_icf -// [ ] fold_with_largest_align // [ ] relocate_undefined_section_symbol //////////////////////////////// @@ -2510,6 +2510,63 @@ TEST(simple_lib_test) T_Ok(*data_addr32nb == data_sect->voff); } +TEST(lib_member_imp_and_regular_symbol_queued_once) +{ + T_Ok(t_write_def_lib("rust_style.rlib", (T_COFF_DefLib){ + .emit_second_member = 1, + .members = (T_COFF_DefLibMember[]){ + { + .type = T_COFF_DefLibMember_Obj, + .obj = { + .path = str8_lit("core-9f9efb2036858c45.core.78298229696da45f-cgu.0.rcgu.o"), + .machine = T_COFF_DefSetMachine(X64), + .sections = (T_COFF_DefSection[]){ + { "text", ".text", str8_lit_comp("\xc3"), .flags = "rx:code@1" }, + { "idata", ".idata", str8_lit_comp("\x00\x00\x00\x00\x00\x00\x00\x00"), .flags = "r:data@8" }, + {0} + }, + .symbols = (T_COFF_DefSymbol[]){ + T_COFF_DefSymbol_ExternFunc("foo", "text", 0), + T_COFF_DefSymbol_Extern("__imp_foo", "idata", 0), + {0} + } + } + }, + {0} + } + })); + + T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ + .machine = T_COFF_DefSetMachine(X64), + .sections = (T_COFF_DefSection[]){ + { + "text", ".text", + str8_lit_comp( + "\x48\xC7\xC0\x00\x00\x00\x00" + "\x48\xC7\xC1\x00\x00\x00\x00" + "\xC3" + ), + .flags = "rx:code@1", + .relocs = (T_COFF_DefReloc[]){ + T_COFF_DefReloc(X64_Addr32Nb, 3, "foo"), + T_COFF_DefReloc(X64_Addr32Nb, 10, "__imp_foo"), + {0} + } + }, + {0} + }, + .symbols = (T_COFF_DefSymbol[]){ + T_COFF_DefSymbol_Extern("entry", "text", 0), + T_COFF_DefSymbol_Undef("foo"), + T_COFF_DefSymbol_Undef("__imp_foo"), + {0} + } + })); + + t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj rust_style.rlib"); + T_Ok(g_last_exit_code == 0); +} + #if OS_WINDOWS TEST(import_export) { @@ -4749,6 +4806,87 @@ TEST(relocate_undefined_section_symbol) } #endif +#if 1 +TEST(weak_alias_comdat_duplicate_fallback) +{ + U64 dummy_count = 64; + + for EachIndex(obj_idx, 2) { + char prefix = obj_idx == 0 ? 'a' : 'b'; + T_COFF_DefSection *sections = push_array(arena, T_COFF_DefSection, dummy_count + 5); + T_COFF_DefSymbol *symbols = push_array(arena, T_COFF_DefSymbol, dummy_count*2 + 11); + U64 section_idx = 0; + U64 symbol_idx = 0; + + for EachIndex(i, dummy_count) { + char *id = (char *)str8f(arena, "dummy_%c_%I64u", prefix, i).str; + char *name = (char *)str8f(arena, "?dummy_%c_%I64u@@YAXXZ", prefix, i).str; + sections[section_idx++] = (T_COFF_DefSection){ id, ".text", str8_lit_comp("\xC3"), .flags = "rx:code@16", .raw_flags = COFF_SectionFlag_LnkCOMDAT }; + symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_Secdef(id, COFF_ComdatSelect_Any); + symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_ExternFunc(name, id, 0); + } + + sections[section_idx++] = (T_COFF_DefSection){ "text0", ".text", str8_lit_comp("\x33\xC0"), .flags = "rx:code@16", .raw_flags = COFF_SectionFlag_LnkCOMDAT }; + sections[section_idx++] = (T_COFF_DefSection){ "text1", ".text", str8_lit_comp("\x33\xC0"), .flags = "rx:code@16", .raw_flags = COFF_SectionFlag_LnkCOMDAT }; + sections[section_idx++] = (T_COFF_DefSection){ "xdata0", ".xdata", str8_lit_comp("\x01\x00\x00\x00"), .flags = "r:data@4", .raw_flags = COFF_SectionFlag_LnkCOMDAT }; + sections[section_idx++] = (T_COFF_DefSection){ "text2", ".text", str8_lit_comp("\x33\xC0\xC3"), .flags = "rx:code@16", .raw_flags = COFF_SectionFlag_LnkCOMDAT }; + + symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_Secdef("text0", COFF_ComdatSelect_Any); + symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_ExternFunc("??_G__shared_count@__Cr@std@@MEAAPEAXI@Z", "text0", 0); + symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_Secdef("text1", COFF_ComdatSelect_Any); + symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_ExternFunc("??_G__shared_weak_count@__Cr@std@@MEAAPEAXI@Z", "text1", 0); + symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_Associative("xdata0", "text0"); + symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_Secdef("text2", COFF_ComdatSelect_Any); + + if (obj_idx == 0) { + symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_ExternFunc("??_G?$__shared_ptr_emplace@A@@UEAAPEAXI@Z", "text2", 0); + symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_Weak("??_E?$__shared_ptr_emplace@A@@UEAAPEAXI@Z", COFF_WeakExt_SearchAlias, "??_G?$__shared_ptr_emplace@A@@UEAAPEAXI@Z"); + } else { + symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_ExternFunc("??_G?$__shared_ptr_emplace@B@@UEAAPEAXI@Z", "text2", 0); + symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_Weak("??_E?$__shared_ptr_emplace@B@@UEAAPEAXI@Z", COFF_WeakExt_SearchAlias, "??_G?$__shared_ptr_emplace@B@@UEAAPEAXI@Z"); + } + symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_Weak("??_E__shared_count@__Cr@std@@MEAAPEAXI@Z", COFF_WeakExt_SearchAlias, "??_G__shared_count@__Cr@std@@MEAAPEAXI@Z"); + symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_Weak("??_E__shared_weak_count@__Cr@std@@MEAAPEAXI@Z", COFF_WeakExt_SearchAlias, "??_G__shared_weak_count@__Cr@std@@MEAAPEAXI@Z"); + + T_Ok(t_write_def_obj(obj_idx == 0 ? "a.obj" : "b.obj", (T_COFF_DefObj){ + .machine = T_COFF_DefSetMachine(X64), + .sections = sections, + .symbols = symbols, + })); + } + + T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ + .machine = T_COFF_DefSetMachine(X64), + .sections = (T_COFF_DefSection[]){ + { + "text", ".text", + str8_lit_comp( + "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm + "\xC3" // ret + ), + .flags = "rx:code", + .relocs = (T_COFF_DefReloc[]){ + T_COFF_DefReloc(X64_Addr32Nb, 3, "??_E__shared_count@__Cr@std@@MEAAPEAXI@Z"), + {0} + } + }, + {0} + }, + .symbols = (T_COFF_DefSymbol[]){ + T_COFF_DefSymbol_Extern("entry", "text", 0), + T_COFF_DefSymbol_Undef("??_E__shared_count@__Cr@std@@MEAAPEAXI@Z"), + {0} + } + })); + + t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe a.obj b.obj entry.obj"); + T_Ok(g_last_exit_code == 0); + + t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe b.obj a.obj entry.obj"); + T_Ok(g_last_exit_code == 0); +} +#endif + #if 1 TEST(opt_ref_weak_alias_comdat) { @@ -4840,6 +4978,18 @@ TEST(fail_if_mismatch) else T_Ok(g_last_exit_code != 0); } +TEST(guardsym_directive) +{ + T_Ok(t_write_entry_obj()); + + // MSVC link accepts GUARDSYM without treating the named symbol as /INCLUDE. + String8 guardsym = t_make_obj_with_directive(arena, str8_lit("/GUARDSYM:missing,S")); + T_Ok(t_write_file(str8_lit("guardsym.obj"), guardsym)); + + t_invoke_linkerf("entry.obj guardsym.obj /entry:entry /subsystem:console /out:guardsym.exe"); + T_Ok(g_last_exit_code == 0); +} + TEST(long_section_name) { Arch arch = Arch_x64; @@ -6963,9 +7113,236 @@ TEST(determ_test) #endif -#if 0 +internal B32 t_read_exe_data_vaddrs(Arena *arena, String8 exe_path, U64 *vaddrs, U64 count); -TEST(fold_two_funcs) +#if OS_WINDOWS +TEST(ms_link_icfs_identical_comdats) +{ + U8 same_text[] = { + 0x48, 0x31, 0xc0, // xor rax, rax + 0xc3 // ret + }; + U8 entry_text[] = { + 0xc3, // ret + }; + U8 addresses[] = { + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + }; + + T_Ok(t_write_def_obj("ms_icf.obj", (T_COFF_DefObj){ + .machine = T_COFF_DefSetMachine(X64), + .sections = (T_COFF_DefSection[]){ + { "entry", ".text$mn", str8_array_fixed(entry_text), .flags = "rx:code@1" }, + { "a", ".text$mn", str8_array_fixed(same_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, + { "b", ".text$mn", str8_array_fixed(same_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, + { + "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@1", + .relocs = (T_COFF_DefReloc[]){ + T_COFF_DefReloc(X64_Addr64, 0, "a"), + T_COFF_DefReloc(X64_Addr64, 8, "b"), + {0} + } + }, + {0} + }, + .symbols = (T_COFF_DefSymbol[]){ + T_COFF_DefSymbol_Secdef("a", COFF_ComdatSelect_Any), + T_COFF_DefSymbol_Secdef("b", COFF_ComdatSelect_Any), + T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), + T_COFF_DefSymbol_ExternFunc("a", "a", 0), + T_COFF_DefSymbol_ExternFunc("b", "b", 0), + T_COFF_DefSymbol_Extern("addresses", "addresses", 0), + {0} + } + })); + + B32 is_invoke_ok = t_invoke(str8_lit("link.exe"), str8_lit("/nologo /nodefaultlib /subsystem:console /entry:entry /out:ms_icf.exe /opt:ref,icf /include:a /include:b /include:addresses ms_icf.obj"), max_U64); + T_Ok(is_invoke_ok); + T_Ok(g_last_exit_code == 0); + + String8 exe = t_read_file(arena, str8_lit("ms_icf.exe")); + T_Ok(exe.size); + + PE_BinInfo pe = pe_bin_info_from_data(arena, exe); + COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; + String8 string_table = str8_substr(exe, pe.string_table_range); + COFF_SectionHeader *data_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); + T_Ok(data_section != 0); + T_Ok(data_section->foff + sizeof(addresses) <= exe.size); + + String8 data = str8_substr(exe, r1u64(data_section->foff, data_section->foff + sizeof(addresses))); + U64 a_addr = 0; + U64 b_addr = 0; + str8_deserial_read_struct(data, 0, &a_addr); + str8_deserial_read_struct(data, 8, &b_addr); + T_Ok(a_addr != 0); + T_Ok(a_addr == b_addr); // COMDAT are folded +} + +#if 0 +TEST(ms_link_icf_section_flag_eligibility) +{ + U8 ret_text[] = { + 0xc3, // ret + }; + U8 data_bytes[] = { + 1, 2, 3, 4, 5, 6, 7, 8, + }; + U8 addresses[] = { + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + }; + + T_Ok(t_write_def_obj("ms_icf_flags.obj", (T_COFF_DefObj){ + .machine = T_COFF_DefSetMachine(X64), + .sections = (T_COFF_DefSection[]){ + { "entry", ".text$mn", str8_array_fixed(ret_text), .flags = "rx:code@1" }, + { + "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@1", + .relocs = (T_COFF_DefReloc[]){ + T_COFF_DefReloc(X64_Addr64, 0, "code_comdat_a"), + T_COFF_DefReloc(X64_Addr64, 8, "code_comdat_b"), + T_COFF_DefReloc(X64_Addr64, 16, "code_plain_a"), + T_COFF_DefReloc(X64_Addr64, 24, "code_plain_b"), + T_COFF_DefReloc(X64_Addr64, 32, "rdata_comdat_a"), + T_COFF_DefReloc(X64_Addr64, 40, "rdata_comdat_b"), + T_COFF_DefReloc(X64_Addr64, 48, "wdata_comdat_a"), + T_COFF_DefReloc(X64_Addr64, 56, "wdata_comdat_b"), + T_COFF_DefReloc(X64_Addr64, 64, "ro_code_comdat_a"), + T_COFF_DefReloc(X64_Addr64, 72, "ro_code_comdat_b"), + T_COFF_DefReloc(X64_Addr64, 80, "rw_code_comdat_a"), + T_COFF_DefReloc(X64_Addr64, 88, "rw_code_comdat_b"), + {0} + } + }, + { "code_comdat_a", ".text$mn", str8_array_fixed(ret_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, + { "code_comdat_b", ".text$mn", str8_array_fixed(ret_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, + { "code_plain_a", ".text$mn", str8_array_fixed(ret_text), .flags = "rx:code@1" }, + { "code_plain_b", ".text$mn", str8_array_fixed(ret_text), .flags = "rx:code@1" }, + { "ro_code_comdat_a", ".text$mn", str8_array_fixed(ret_text), .flags = "r:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, + { "ro_code_comdat_b", ".text$mn", str8_array_fixed(ret_text), .flags = "r:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, + { "rw_code_comdat_a", ".text$mn", str8_array_fixed(ret_text), .flags = "rw:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, + { "rw_code_comdat_b", ".text$mn", str8_array_fixed(ret_text), .flags = "rw:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, + { "rdata_comdat_a", ".rdata$mn", str8_array_fixed(data_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, + { "rdata_comdat_b", ".rdata$mn", str8_array_fixed(data_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, + { "wdata_comdat_a", ".data$mn", str8_array_fixed(data_bytes), .flags = "rw:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, + { "wdata_comdat_b", ".data$mn", str8_array_fixed(data_bytes), .flags = "rw:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, + + {0} + }, + .symbols = (T_COFF_DefSymbol[]){ + T_COFF_DefSymbol_Secdef("code_comdat_a", COFF_ComdatSelect_Any), + T_COFF_DefSymbol_Secdef("code_comdat_b", COFF_ComdatSelect_Any), + T_COFF_DefSymbol_Secdef("rdata_comdat_a", COFF_ComdatSelect_Any), + T_COFF_DefSymbol_Secdef("rdata_comdat_b", COFF_ComdatSelect_Any), + T_COFF_DefSymbol_Secdef("wdata_comdat_a", COFF_ComdatSelect_Any), + T_COFF_DefSymbol_Secdef("wdata_comdat_b", COFF_ComdatSelect_Any), + T_COFF_DefSymbol_Secdef("ro_code_comdat_a", COFF_ComdatSelect_Any), + T_COFF_DefSymbol_Secdef("ro_code_comdat_b", COFF_ComdatSelect_Any), + T_COFF_DefSymbol_Secdef("rw_code_comdat_a", COFF_ComdatSelect_Any), + T_COFF_DefSymbol_Secdef("rw_code_comdat_b", COFF_ComdatSelect_Any), + T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), + T_COFF_DefSymbol_ExternFunc("code_comdat_a", "code_comdat_a", 0), + T_COFF_DefSymbol_ExternFunc("code_comdat_b", "code_comdat_b", 0), + T_COFF_DefSymbol_ExternFunc("code_plain_a", "code_plain_a", 0), + T_COFF_DefSymbol_ExternFunc("code_plain_b", "code_plain_b", 0), + T_COFF_DefSymbol_ExternFunc("ro_code_comdat_a", "ro_code_comdat_a", 0), + T_COFF_DefSymbol_ExternFunc("ro_code_comdat_b", "ro_code_comdat_b", 0), + T_COFF_DefSymbol_ExternFunc("rw_code_comdat_a", "rw_code_comdat_a", 0), + T_COFF_DefSymbol_ExternFunc("rw_code_comdat_b", "rw_code_comdat_b", 0), + T_COFF_DefSymbol_Extern("rdata_comdat_a", "rdata_comdat_a", 0), + T_COFF_DefSymbol_Extern("rdata_comdat_b", "rdata_comdat_b", 0), + T_COFF_DefSymbol_Extern("wdata_comdat_a", "wdata_comdat_a", 0), + T_COFF_DefSymbol_Extern("wdata_comdat_b", "wdata_comdat_b", 0), + T_COFF_DefSymbol_Extern("addresses", "addresses", 0), + {0} + } + })); + + B32 is_invoke_ok = t_invoke(str8_lit("link.exe"), str8_lit("/nologo /nodefaultlib /subsystem:console /entry:entry /out:ms_icf_flags.exe /opt:ref,icf /include:addresses ms_icf_flags.obj"), max_U64); + T_Ok(is_invoke_ok); + T_Ok(g_last_exit_code == 0); + + String8 exe = t_read_file(arena, str8_lit("ms_icf_flags.exe")); + T_Ok(exe.size); + + PE_BinInfo pe = pe_bin_info_from_data(arena, exe); + COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; + String8 string_table = str8_substr(exe, pe.string_table_range); + COFF_SectionHeader *data_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); + T_Ok(data_section != 0); + T_Ok(data_section->foff + sizeof(addresses) <= exe.size); + + String8 data = str8_substr(exe, r1u64(data_section->foff, data_section->foff + sizeof(addresses))); + U64 code_comdat_a_addr = 0; + U64 code_comdat_b_addr = 0; + U64 code_plain_a_addr = 0; + U64 code_plain_b_addr = 0; + U64 rdata_comdat_a_addr = 0; + U64 rdata_comdat_b_addr = 0; + U64 wdata_comdat_a_addr = 0; + U64 wdata_comdat_b_addr = 0; + U64 ro_code_comdat_a_addr = 0; + U64 ro_code_comdat_b_addr = 0; + U64 rw_code_comdat_a_addr = 0; + U64 rw_code_comdat_b_addr = 0; + str8_deserial_read_struct(data, 0, &code_comdat_a_addr); + str8_deserial_read_struct(data, 8, &code_comdat_b_addr); + str8_deserial_read_struct(data, 16, &code_plain_a_addr); + str8_deserial_read_struct(data, 24, &code_plain_b_addr); + str8_deserial_read_struct(data, 32, &rdata_comdat_a_addr); + str8_deserial_read_struct(data, 40, &rdata_comdat_b_addr); + str8_deserial_read_struct(data, 48, &wdata_comdat_a_addr); + str8_deserial_read_struct(data, 56, &wdata_comdat_b_addr); + str8_deserial_read_struct(data, 64, &ro_code_comdat_a_addr); + str8_deserial_read_struct(data, 72, &ro_code_comdat_b_addr); + str8_deserial_read_struct(data, 80, &rw_code_comdat_a_addr); + str8_deserial_read_struct(data, 88, &rw_code_comdat_b_addr); + + T_Ok(code_comdat_a_addr != 0); + T_Ok(code_comdat_a_addr == code_comdat_b_addr); // executable code COMDATs fold + T_Ok(ro_code_comdat_a_addr == ro_code_comdat_b_addr); // read-only code COMDATs fold + T_Ok(code_plain_a_addr != code_plain_b_addr); // non-COMDAT code does not fold + T_Ok(wdata_comdat_a_addr != wdata_comdat_b_addr); // writable data COMDATs do not fold + T_Ok(rw_code_comdat_a_addr != rw_code_comdat_b_addr); // writable code COMDATs do not + T_Ok(rdata_comdat_a_addr == rdata_comdat_b_addr); // read-only data COMDATs fold +} +#endif + +#endif + +#if 1 + +internal B32 +t_read_exe_data_vaddrs(Arena *arena, String8 exe_path, U64 *vaddrs, U64 count) +{ + B32 result = 0; + String8 exe = t_read_file(arena, exe_path); + if (exe.size) { + PE_BinInfo pe = pe_bin_info_from_data(arena, exe); + COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; + String8 string_table = str8_substr(exe, pe.string_table_range); + COFF_SectionHeader *data_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); + if (data_section != 0 && data_section->foff + count*sizeof(U64) <= exe.size) { + String8 data = str8_substr(exe, r1u64(data_section->foff, data_section->foff + count*sizeof(U64))); + result = str8_deserial_read_array(data, 0, vaddrs, count); + } + } + return result; +} + +TEST(icf_fold_two_funcs) { U8 same_text[] = { 0x48, 0x31, 0xc0, // xor rax, rax @@ -7004,7 +7381,7 @@ TEST(fold_two_funcs) t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:icf ident_funcs.obj"); T_Ok(g_last_exit_code == 0); - String8 exe = t_read_file(arena, str8_lit("ident_funcs.exe")); + String8 exe = t_read_file(arena, str8_lit("a.exe")); T_Ok(exe.size); PE_BinInfo pe = pe_bin_info_from_data(arena, exe); @@ -7036,8 +7413,7 @@ TEST(fold_two_funcs) T_Ok(str8_match(text_data, str8_array_fixed(expected_text), 0)); } - -TEST(same_but_different) +TEST(icf_same_but_different) { U8 text[] = { 0xe8, 0x00, 0x00, 0x00, 0x00, // call $ @@ -7138,8 +7514,7 @@ TEST(same_but_different) } } - -TEST(fold_diamond) +TEST(icf_fold_diamond) { U8 call_b_and_c[] = { 0xe8, 0x00, 0x00, 0x00, 0x00, @@ -7231,8 +7606,7 @@ TEST(fold_diamond) } } - -TEST(cyclic_icf) +TEST(icf_cyclic_icf) { U8 text[] = { 0xe8, 0x00, 0x00, 0x00, 0x00, @@ -7242,14 +7616,14 @@ TEST(cyclic_icf) .machine = T_COFF_DefSetMachine(X64), .sections = (T_COFF_DefSection[]){ { - "a", ".text", str8_array_fixed(text), .flags = "rx:code", + "a", ".text", str8_array_fixed(text), .flags = "rx:code", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Rel32, 1, "b"), {0} } }, { - "b", ".text", str8_array_fixed(text), .flags = "rx:code", + "b", ".text", str8_array_fixed(text), .flags = "rx:code", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Rel32, 1, "a"), {0} @@ -7258,6 +7632,8 @@ TEST(cyclic_icf) {0} }, .symbols = (T_COFF_DefSymbol[]){ + T_COFF_DefSymbol_Secdef("a", COFF_ComdatSelect_NoDuplicates), + T_COFF_DefSymbol_Secdef("b", COFF_ComdatSelect_NoDuplicates), T_COFF_DefSymbol_Extern("a", "a", 0), T_COFF_DefSymbol_Static("b", "b", 0), {0} @@ -7270,11 +7646,8 @@ TEST(cyclic_icf) // validate output { U8 expected_text[] = { - 0xe8, 0x0b, 0x00, 0x00, 0x00, // a + 0xe8, 0xfb, 0xff, 0xff, 0xff, // a and b folded into a self-call 0xc3, - 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, - 0xe8, 0xeb, 0xff, 0xff, 0xff, // b - 0xc3, }; String8 exe = t_read_file(arena, str8_lit("a.exe")); @@ -7293,8 +7666,190 @@ TEST(cyclic_icf) } } +// ICF must preserve identical sections with different symbol targets +TEST(icf_reloc_target_symbol_types_do_not_fold) +{ + U8 reloc_text[] = { + 0x48, 0xc7, 0xc0, 0x00, 0x00, 0x00, 0x00, // mov rax, target + 0xc3 // ret + }; + U8 ret_text[] = { + 0xc3, // ret + }; + U8 target_data[] = { + 0x00, + }; + U8 addresses[3 * sizeof(U64)] = {0}; -TEST(fold_with_largest_align) + T_Ok(t_write_def_obj("icf_interp_entry.obj", (T_COFF_DefObj){ + .machine = T_COFF_DefSetMachine(X64), + .sections = (T_COFF_DefSection[]){ + { "entry", ".text", str8_array_fixed(ret_text), .flags = "rx:code@1" }, + { + "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@1", + .relocs = (T_COFF_DefReloc[]){ + T_COFF_DefReloc(X64_Addr64, 0, "fn_regular"), + T_COFF_DefReloc(X64_Addr64, 8, "fn_common"), + T_COFF_DefReloc(X64_Addr64, 16, "fn_abs"), + {0} + } + }, + {0} + }, + .symbols = (T_COFF_DefSymbol[]){ + T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), + T_COFF_DefSymbol_Extern("addresses", "addresses", 0), + T_COFF_DefSymbol_Undef("fn_regular"), + T_COFF_DefSymbol_Undef("fn_common"), + T_COFF_DefSymbol_Undef("fn_abs"), + {0} + } + })); + + T_Ok(t_write_def_obj("icf_interp_regular.obj", (T_COFF_DefObj){ + .machine = T_COFF_DefSetMachine(X64), + .sections = (T_COFF_DefSection[]){ + { "fn", ".text$mn", str8_array_fixed(reloc_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Rel32, 3, "target_regular"), {0} } }, + { "target", ".rdata$mn", str8_array_fixed(target_data), .flags = "r:data@1" }, + {0} + }, + .symbols = (T_COFF_DefSymbol[]){ + T_COFF_DefSymbol_Secdef("fn", COFF_ComdatSelect_NoDuplicates), + T_COFF_DefSymbol_ExternFunc("fn_regular", "fn", 0), + T_COFF_DefSymbol_Extern("target_regular", "target", 0), + {0} + } + })); + + T_Ok(t_write_def_obj("icf_interp_common.obj", (T_COFF_DefObj){ + .machine = T_COFF_DefSetMachine(X64), + .sections = (T_COFF_DefSection[]){ + { "fn", ".text$mn", str8_array_fixed(reloc_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Rel32, 3, "target_common"), {0} } }, + {0} + }, + .symbols = (T_COFF_DefSymbol[]){ + T_COFF_DefSymbol_Secdef("fn", COFF_ComdatSelect_NoDuplicates), + T_COFF_DefSymbol_ExternFunc("fn_common", "fn", 0), + T_COFF_DefSymbol_Common("target_common", 8), + {0} + } + })); + + T_Ok(t_write_def_obj("icf_interp_abs.obj", (T_COFF_DefObj){ + .machine = T_COFF_DefSetMachine(X64), + .sections = (T_COFF_DefSection[]){ + { "fn", ".text$mn", str8_array_fixed(reloc_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Rel32, 3, "target_abs"), {0} } }, + {0} + }, + .symbols = (T_COFF_DefSymbol[]){ + T_COFF_DefSymbol_Secdef("fn", COFF_ComdatSelect_NoDuplicates), + T_COFF_DefSymbol_ExternFunc("fn_abs", "fn", 0), + T_COFF_DefSymbol_AbsExtern("target_abs", 0x1234), + {0} + } + })); + + t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:ref,icf icf_interp_entry.obj icf_interp_regular.obj icf_interp_common.obj icf_interp_abs.obj"); + T_Ok(g_last_exit_code == 0); + + U64 fn_vaddrs[ArrayCount(addresses) / sizeof(U64)] = {0}; + T_Ok(t_read_exe_data_vaddrs(arena, str8_lit("a.exe"), fn_vaddrs, ArrayCount(fn_vaddrs))); + + for EachElement(i, fn_vaddrs) { + for (U64 j = i + 1; j < ArrayCount(fn_vaddrs); j += 1) { + T_Ok(fn_vaddrs[i] != fn_vaddrs[j]); + } + } +} + +// ICF must preserve sections with unresolved target symbols (with /FORCE) +TEST(icf_unresolved_reloc_targets_do_not_fold) +{ + U8 reloc_text[] = { + 0x48, 0xc7, 0xc0, 0x00, 0x00, 0x00, 0x00, // mov rax, target + 0xc3 // ret + }; + U8 ret_text[] = { + 0xc3, // ret + }; + U8 addresses[2 * sizeof(U64)] = {0}; + + T_Ok(t_write_def_obj("icf_unresolved_entry.obj", (T_COFF_DefObj){ + .machine = T_COFF_DefSetMachine(X64), + .sections = (T_COFF_DefSection[]){ + { "entry", ".text", str8_array_fixed(ret_text), .flags = "rx:code@1" }, + { + "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@1", + .relocs = (T_COFF_DefReloc[]){ + T_COFF_DefReloc(X64_Addr64, 0, "fn_undef"), + T_COFF_DefReloc(X64_Addr64, 8, "fn_weak"), + {0} + } + }, + {0} + }, + .symbols = (T_COFF_DefSymbol[]){ + T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), + T_COFF_DefSymbol_Extern("addresses", "addresses", 0), + T_COFF_DefSymbol_Undef("fn_undef"), + T_COFF_DefSymbol_Undef("fn_weak"), + {0} + } + })); + + T_Ok(t_write_def_obj("icf_unresolved_undef.obj", (T_COFF_DefObj){ + .machine = T_COFF_DefSetMachine(X64), + .sections = (T_COFF_DefSection[]){ + { "fn", ".text$mn", str8_array_fixed(reloc_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Rel32, 3, "target_undef"), {0} } }, + {0} + }, + .symbols = (T_COFF_DefSymbol[]){ + T_COFF_DefSymbol_Secdef("fn", COFF_ComdatSelect_NoDuplicates), + T_COFF_DefSymbol_ExternFunc("fn_undef", "fn", 0), + T_COFF_DefSymbol_Undef("target_undef"), + {0} + } + })); + + T_Ok(t_write_def_obj("icf_unresolved_weak.obj", (T_COFF_DefObj){ + .machine = T_COFF_DefSetMachine(X64), + .sections = (T_COFF_DefSection[]){ + { "fn", ".text$mn", str8_array_fixed(reloc_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Rel32, 3, "target_weak"), {0} } }, + {0} + }, + .symbols = (T_COFF_DefSymbol[]){ + T_COFF_DefSymbol_Secdef("fn", COFF_ComdatSelect_NoDuplicates), + T_COFF_DefSymbol_ExternFunc("fn_weak", "fn", 0), + T_COFF_DefSymbol_AbsExtern("target_weak_fallback", 0), + T_COFF_DefSymbol_Weak("target_weak", COFF_WeakExt_NoLibrary, "target_weak_fallback"), + {0} + } + })); + + t_invoke_linkerf("/force /subsystem:console /entry:entry /out:a.exe /opt:ref,icf icf_unresolved_entry.obj icf_unresolved_undef.obj icf_unresolved_weak.obj"); + T_Ok(g_last_exit_code == 0); + + String8 exe = t_read_file(arena, str8_lit("a.exe")); + T_Ok(exe.size); + + PE_BinInfo pe = pe_bin_info_from_data(arena, exe); + COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; + String8 string_table = str8_substr(exe, pe.string_table_range); + COFF_SectionHeader *data_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); + T_Ok(data_section != 0); + T_Ok(data_section->foff + sizeof(addresses) <= exe.size); + + String8 data = str8_substr(exe, r1u64(data_section->foff, data_section->foff + sizeof(addresses))); + U64 undef_vaddr = 0; + U64 weak_vaddr = 0; + str8_deserial_read_struct(data, 0, &undef_vaddr); + str8_deserial_read_struct(data, 8, &weak_vaddr); + T_Ok(undef_vaddr != 0); + T_Ok(weak_vaddr != 0); + T_Ok(undef_vaddr != weak_vaddr); +} + +TEST(icf_fold_with_largest_align) { U8 text[] = { 0x48, 0xc7, 0xc0, 0x01, 0x00, 0x00, 0x00, // mov rax, 1 @@ -7360,10 +7915,10 @@ TEST(fold_with_largest_align) T_Ok(t_write_file(str8_lit("a.obj"), a_obj)); T_Ok(t_write_file(str8_lit("b.obj"), b_obj)); - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe a.obj"); + t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:icf a.obj"); T_Ok(g_last_exit_code == 0); - t_invoke_linkerf("/subsystem:console /entry:entry /out:b.exe b.obj"); + t_invoke_linkerf("/subsystem:console /entry:entry /out:b.exe /opt:icf b.obj"); T_Ok(g_last_exit_code == 0); U8 expected_text[] = { @@ -7415,6 +7970,563 @@ TEST(fold_with_largest_align) } } +TEST(icf_identical_bytes_different_color_spaces_do_not_fold) +{ + U8 same_bytes[] = { + 0xc3, + }; + U8 entry_text[] = { + 0xc3, + }; + U8 addresses[2 * sizeof(U64)] = {0}; + + T_Ok(t_write_def_obj("icf_color_spaces.obj", (T_COFF_DefObj){ + .machine = T_COFF_DefSetMachine(X64), + .sections = (T_COFF_DefSection[]){ + { "entry", ".text", str8_array_fixed(entry_text), .flags = "rx:code@1" }, + { "text", ".text$mn", str8_array_fixed(same_bytes), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, + { "vftable", ".rdata$mn", str8_array_fixed(same_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, + { + "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@1", + .relocs = (T_COFF_DefReloc[]){ + T_COFF_DefReloc(X64_Addr64, 0, "text"), + T_COFF_DefReloc(X64_Addr64, 8, "??_7type@@6B@"), + {0} + } + }, + {0} + }, + .symbols = (T_COFF_DefSymbol[]){ + T_COFF_DefSymbol_Secdef("text", COFF_ComdatSelect_NoDuplicates), + T_COFF_DefSymbol_Secdef("vftable", COFF_ComdatSelect_NoDuplicates), + T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), + T_COFF_DefSymbol_ExternFunc("text", "text", 0), + T_COFF_DefSymbol_Extern("??_7type@@6B@", "vftable", 0), + T_COFF_DefSymbol_Extern("addresses", "addresses", 0), + {0} + } + })); + + t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_color_spaces.obj"); + T_Ok(g_last_exit_code == 0); + + String8 exe = t_read_file(arena, str8_lit("a.exe")); + T_Ok(exe.size); + + PE_BinInfo pe = pe_bin_info_from_data(arena, exe); + COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; + String8 string_table = str8_substr(exe, pe.string_table_range); + COFF_SectionHeader *data_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); + T_Ok(data_section != 0); + T_Ok(data_section->foff + sizeof(addresses) <= exe.size); + + String8 data = str8_substr(exe, r1u64(data_section->foff, data_section->foff + sizeof(addresses))); + U64 text_vaddr = 0; + U64 vftable_vaddr = 0; + str8_deserial_read_struct(data, 0, &text_vaddr); + str8_deserial_read_struct(data, 8, &vftable_vaddr); + T_Ok(text_vaddr != 0); + T_Ok(vftable_vaddr != 0); + T_Ok(text_vaddr != vftable_vaddr); +} + +TEST(icf_multihop_reloc_target_colors_do_not_fold) +{ + U8 call_text[] = { + 0xe8, 0x00, 0x00, 0x00, 0x00, + 0xc3, + }; + U8 return_1[] = { + 0x48, 0xc7, 0xc0, 0x01, 0x00, 0x00, 0x00, + 0xc3, + }; + U8 return_2[] = { + 0x48, 0xc7, 0xc0, 0x02, 0x00, 0x00, 0x00, + 0xc3, + }; + U8 entry_text[] = { + 0xc3, + }; + U8 addresses[6 * sizeof(U64)] = {0}; + + T_Ok(t_write_def_obj("icf_multihop.obj", (T_COFF_DefObj){ + .machine = T_COFF_DefSetMachine(X64), + .sections = (T_COFF_DefSection[]){ + { "entry", ".text", str8_array_fixed(entry_text), .flags = "rx:code@1" }, + { + "top_a", ".text$mn", str8_array_fixed(call_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, + .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Rel32, 1, "mid_a"), {0} } + }, + { + "top_b", ".text$mn", str8_array_fixed(call_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, + .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Rel32, 1, "mid_b"), {0} } + }, + { + "mid_a", ".text$mn", str8_array_fixed(call_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, + .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Rel32, 1, "leaf_a"), {0} } + }, + { + "mid_b", ".text$mn", str8_array_fixed(call_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, + .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Rel32, 1, "leaf_b"), {0} } + }, + { "leaf_a", ".text$mn", str8_array_fixed(return_1), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, + { "leaf_b", ".text$mn", str8_array_fixed(return_2), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, + { + "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@1", + .relocs = (T_COFF_DefReloc[]){ + T_COFF_DefReloc(X64_Addr64, 0, "top_a"), + T_COFF_DefReloc(X64_Addr64, 8, "top_b"), + T_COFF_DefReloc(X64_Addr64, 16, "mid_a"), + T_COFF_DefReloc(X64_Addr64, 24, "mid_b"), + T_COFF_DefReloc(X64_Addr64, 32, "leaf_a"), + T_COFF_DefReloc(X64_Addr64, 40, "leaf_b"), + {0} + } + }, + {0} + }, + .symbols = (T_COFF_DefSymbol[]){ + T_COFF_DefSymbol_Secdef("top_a", COFF_ComdatSelect_NoDuplicates), + T_COFF_DefSymbol_Secdef("top_b", COFF_ComdatSelect_NoDuplicates), + T_COFF_DefSymbol_Secdef("mid_a", COFF_ComdatSelect_NoDuplicates), + T_COFF_DefSymbol_Secdef("mid_b", COFF_ComdatSelect_NoDuplicates), + T_COFF_DefSymbol_Secdef("leaf_a", COFF_ComdatSelect_NoDuplicates), + T_COFF_DefSymbol_Secdef("leaf_b", COFF_ComdatSelect_NoDuplicates), + T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), + T_COFF_DefSymbol_ExternFunc("top_a", "top_a", 0), + T_COFF_DefSymbol_ExternFunc("top_b", "top_b", 0), + T_COFF_DefSymbol_ExternFunc("mid_a", "mid_a", 0), + T_COFF_DefSymbol_ExternFunc("mid_b", "mid_b", 0), + T_COFF_DefSymbol_ExternFunc("leaf_a", "leaf_a", 0), + T_COFF_DefSymbol_ExternFunc("leaf_b", "leaf_b", 0), + T_COFF_DefSymbol_Extern("addresses", "addresses", 0), + {0} + } + })); + + t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_multihop.obj"); + T_Ok(g_last_exit_code == 0); + + U64 vaddrs[ArrayCount(addresses) / sizeof(U64)] = {0}; + T_Ok(t_read_exe_data_vaddrs(arena, str8_lit("a.exe"), vaddrs, ArrayCount(vaddrs))); + for EachIndex(i, ArrayCount(vaddrs)) { + T_Ok(vaddrs[i] != 0); + } + T_Ok(vaddrs[0] != vaddrs[1]); + T_Ok(vaddrs[2] != vaddrs[3]); + T_Ok(vaddrs[4] != vaddrs[5]); +} + +TEST(icf_comdat_symlink_chain) +{ + U8 ret_small[] = { 0xc3 }; + U8 ret_large[] = { 0xc3, 0x90 }; + U8 entry_text[] = { 0xc3 }; + U8 addresses[2 * sizeof(U64)] = {0}; + + T_Ok(t_write_def_obj("icf_chain_leader.obj", (T_COFF_DefObj){ + .machine = T_COFF_DefSetMachine(X64), + .sections = (T_COFF_DefSection[]){ + { "leader", ".text$mn", str8_array_fixed(ret_large), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, + {0} + }, + .symbols = (T_COFF_DefSymbol[]){ + T_COFF_DefSymbol_Secdef("leader", COFF_ComdatSelect_NoDuplicates), + T_COFF_DefSymbol_ExternFunc("leader", "leader", 0), + {0} + } + })); + + T_Ok(t_write_def_obj("icf_chain_duplicate.obj", (T_COFF_DefObj){ + .machine = T_COFF_DefSetMachine(X64), + .sections = (T_COFF_DefSection[]){ + { "dup", ".text$mn", str8_array_fixed(ret_small), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, + { + "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@1", + .relocs = (T_COFF_DefReloc[]){ + T_COFF_DefReloc(X64_Addr64, 0, "local_dup"), + T_COFF_DefReloc(X64_Addr64, 8, "leader"), + {0} + } + }, + {0} + }, + .symbols = (T_COFF_DefSymbol[]){ + T_COFF_DefSymbol_Secdef("dup", COFF_ComdatSelect_Largest), + T_COFF_DefSymbol_ExternFunc("dup", "dup", 0), + T_COFF_DefSymbol_Static("local_dup", "dup", 0), + T_COFF_DefSymbol_Extern("addresses", "addresses", 0), + T_COFF_DefSymbol_UndefFunc("leader"), + {0} + } + })); + + T_Ok(t_write_def_obj("icf_chain_selected.obj", (T_COFF_DefObj){ + .machine = T_COFF_DefSetMachine(X64), + .sections = (T_COFF_DefSection[]){ + { "dup", ".text$mn", str8_array_fixed(ret_large), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, + {0} + }, + .symbols = (T_COFF_DefSymbol[]){ + T_COFF_DefSymbol_Secdef("dup", COFF_ComdatSelect_Largest), + T_COFF_DefSymbol_ExternFunc("dup", "dup", 0), + {0} + } + })); + + T_Ok(t_write_def_obj("icf_chain_entry.obj", (T_COFF_DefObj){ + .machine = T_COFF_DefSetMachine(X64), + .sections = (T_COFF_DefSection[]){ + { "entry", ".text", str8_array_fixed(entry_text), .flags = "rx:code@1" }, + {0} + }, + .symbols = (T_COFF_DefSymbol[]){ + T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), + {0} + } + })); + + t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_chain_leader.obj icf_chain_duplicate.obj icf_chain_selected.obj icf_chain_entry.obj"); + T_Ok(g_last_exit_code == 0); + + U64 vaddrs[ArrayCount(addresses) / sizeof(U64)] = {0}; + T_Ok(t_read_exe_data_vaddrs(arena, str8_lit("a.exe"), vaddrs, ArrayCount(vaddrs))); + T_Ok(vaddrs[0] != 0); + T_Ok(vaddrs[0] == vaddrs[1]); +} + +TEST(icf_llvm_addrsig) +{ + char *main_c = "int foo() { return 123; }\n" + "int bar() { return 123; }\n" + "int main() {\n" + "int (*fn)() = &foo;\n" + "return fn != bar;\n" + "}\n"; + String8 main_path = t_make_file_path(arena, str8_lit("main.c")); + T_Ok(write_data_to_file_path(main_path, str8_cstring(main_c))); + + String8 main_obj_path = t_make_file_path(arena, str8_lit("main.obj"));; + t_invoke(t_clang_path(), str8f(arena, "%S -o %S -c -ffunction-sections -target x86_64-pc-windows-msvc", main_path, main_obj_path), max_U64); + T_Ok(g_last_exit_code == 0); + + t_invoke_linkerf("%S /opt:icf /out:a.exe libcmt.lib", main_obj_path); + T_Ok(g_last_exit_code == 0); + + String8 a_path = t_make_file_path(arena, str8_lit("a.exe")); + t_invoke(a_path, str8_zero(), max_U64); + if (t_id_linker() == Linker_radlink) { + T_Ok(g_last_exit_code == 1); + } else { + T_Ok(g_last_exit_code == 0); + } + + if (t_id_linker() == Linker_radlink) { + t_invoke_linkerf("%S /opt:icf /out:a.exe libcmt.lib /llvm_addrsig:no", main_obj_path); + T_Ok(g_last_exit_code == 0); + + t_invoke(a_path, str8_zero(), max_U64); + T_Ok(g_last_exit_code == 0); + } + +} + +// .llvm_addrsig can name an undefined external whose definition is in another +// object; ICF must parse and mark the resolved symbol's object, not the referrer. +TEST(icf_llvm_addrsig_external_symbol) +{ + char *ref_c = "extern int ext_sig();\n" + "int (*ext_sig_addr)() = &ext_sig;\n" + "int entry() { return ext_sig_addr(); }\n"; + char *def_c = "int dummy0() { return 0; }\n" + "int dummy1() { return 1; }\n" + "int dummy2() { return 2; }\n" + "int dummy3() { return 3; }\n" + "int dummy4() { return 4; }\n" + "int ext_sig() { return 0; }\n"; + String8 ref_path = t_make_file_path(arena, str8_lit("ref.c")); + String8 def_path = t_make_file_path(arena, str8_lit("def.c")); + T_Ok(write_data_to_file_path(ref_path, str8_cstring(ref_c))); + T_Ok(write_data_to_file_path(def_path, str8_cstring(def_c))); + + String8 ref_obj_path = t_make_file_path(arena, str8_lit("ref.obj")); + String8 def_obj_path = t_make_file_path(arena, str8_lit("def.obj")); + t_invoke(t_clang_path(), str8f(arena, "%S -o %S -c -ffunction-sections -target x86_64-pc-windows-msvc", ref_path, ref_obj_path), max_U64); + T_Ok(g_last_exit_code == 0); + t_invoke(t_clang_path(), str8f(arena, "%S -o %S -c -ffunction-sections -target x86_64-pc-windows-msvc", def_path, def_obj_path), max_U64); + T_Ok(g_last_exit_code == 0); + + t_invoke_linkerf("%S %S /subsystem:console /entry:entry /opt:icf /out:addrsig_ext.exe libcmt.lib", ref_obj_path, def_obj_path); + T_Ok(g_last_exit_code == 0); +} + +TEST(icf_pdata_xdata_fold) +{ + U8 ret_text[] = { 0xc3 }; + U8 xdata[] = { 0x01, 0x00, 0x00, 0x00 }; + PE_IntelPdata pdata = {0}; + U8 entry_text[] = { 0xc3 }; + U8 addresses[4 * sizeof(U64)] = {0}; + + T_Ok(t_write_def_obj("icf_pdata_xdata_fold.obj", (T_COFF_DefObj){ + .machine = T_COFF_DefSetMachine(X64), + .sections = (T_COFF_DefSection[]){ + { "entry", ".text", str8_array_fixed(entry_text), .flags = "rx:code@1" }, + { "fn_a", ".text$mn", str8_array_fixed(ret_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, + { "fn_b", ".text$mn", str8_array_fixed(ret_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, + { "xdata_a", ".xdata", str8_array_fixed(xdata), .flags = "r:data@4", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, + { "xdata_b", ".xdata", str8_array_fixed(xdata), .flags = "r:data@4", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, + { + "pdata_a", ".pdata", str8_struct(&pdata), .flags = "r:data@4", .raw_flags = COFF_SectionFlag_LnkCOMDAT, + .relocs = (T_COFF_DefReloc[]){ + T_COFF_DefReloc(X64_Addr32Nb, OffsetOf(PE_IntelPdata, voff_first), "fn_a"), + T_COFF_DefReloc(X64_Addr32Nb, OffsetOf(PE_IntelPdata, voff_one_past_last), "fn_a"), + T_COFF_DefReloc(X64_Addr32Nb, OffsetOf(PE_IntelPdata, voff_unwind_info), "$unwind$a"), + {0} + } + }, + { + "pdata_b", ".pdata", str8_struct(&pdata), .flags = "r:data@4", .raw_flags = COFF_SectionFlag_LnkCOMDAT, + .relocs = (T_COFF_DefReloc[]){ + T_COFF_DefReloc(X64_Addr32Nb, OffsetOf(PE_IntelPdata, voff_first), "fn_b"), + T_COFF_DefReloc(X64_Addr32Nb, OffsetOf(PE_IntelPdata, voff_one_past_last), "fn_b"), + T_COFF_DefReloc(X64_Addr32Nb, OffsetOf(PE_IntelPdata, voff_unwind_info), "$unwind$b"), + {0} + } + }, + { + "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@1", + .relocs = (T_COFF_DefReloc[]){ + T_COFF_DefReloc(X64_Addr64, 0, "$pdata$a"), + T_COFF_DefReloc(X64_Addr64, 8, "$pdata$b"), + T_COFF_DefReloc(X64_Addr64, 16, "$unwind$a"), + T_COFF_DefReloc(X64_Addr64, 24, "$unwind$b"), + {0} + } + }, + {0} + }, + .symbols = (T_COFF_DefSymbol[]){ + T_COFF_DefSymbol_Secdef("fn_a", COFF_ComdatSelect_NoDuplicates), + T_COFF_DefSymbol_Secdef("fn_b", COFF_ComdatSelect_NoDuplicates), + T_COFF_DefSymbol_Secdef("xdata_a", COFF_ComdatSelect_NoDuplicates), + T_COFF_DefSymbol_Secdef("xdata_b", COFF_ComdatSelect_NoDuplicates), + T_COFF_DefSymbol_Secdef("pdata_a", COFF_ComdatSelect_NoDuplicates), + T_COFF_DefSymbol_Secdef("pdata_b", COFF_ComdatSelect_NoDuplicates), + T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), + T_COFF_DefSymbol_ExternFunc("fn_a", "fn_a", 0), + T_COFF_DefSymbol_ExternFunc("fn_b", "fn_b", 0), + T_COFF_DefSymbol_Extern("$unwind$a", "xdata_a", 0), + T_COFF_DefSymbol_Extern("$unwind$b", "xdata_b", 0), + T_COFF_DefSymbol_Extern("$pdata$a", "pdata_a", 0), + T_COFF_DefSymbol_Extern("$pdata$b", "pdata_b", 0), + T_COFF_DefSymbol_Extern("addresses", "addresses", 0), + {0} + } + })); + + t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_pdata_xdata_fold.obj"); + T_Ok(g_last_exit_code == 0); + + U64 vaddrs[4] = {0}; + T_Ok(t_read_exe_data_vaddrs(arena, str8_lit("a.exe"), vaddrs, ArrayCount(vaddrs))); + T_Ok(vaddrs[0] != 0); + T_Ok(vaddrs[0] == vaddrs[1]); + T_Ok(vaddrs[2] != 0); + T_Ok(vaddrs[2] == vaddrs[3]); +} + +TEST(icf_pdata_differs_by_function_color) +{ + U8 ret_1[] = { 0xb8, 1, 0, 0, 0, 0xc3 }; + U8 ret_2[] = { 0xb8, 2, 0, 0, 0, 0xc3 }; + U8 xdata[] = { 0x01, 0x00, 0x00, 0x00 }; + PE_IntelPdata pdata = {0}; + U8 entry_text[] = { 0xc3 }; + U8 addresses[3 * sizeof(U64)] = {0}; + + T_Ok(t_write_def_obj("icf_pdata_diff.obj", (T_COFF_DefObj){ + .machine = T_COFF_DefSetMachine(X64), + .sections = (T_COFF_DefSection[]){ + { "entry", ".text", str8_array_fixed(entry_text), .flags = "rx:code@1" }, + { "fn_a", ".text$mn", str8_array_fixed(ret_1), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, + { "fn_b", ".text$mn", str8_array_fixed(ret_2), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, + { "xdata_a", ".xdata", str8_array_fixed(xdata), .flags = "r:data@4", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, + { "xdata_b", ".xdata", str8_array_fixed(xdata), .flags = "r:data@4", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, + { "pdata_a", ".pdata", str8_struct(&pdata), .flags = "r:data@4", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr32Nb, 0, "fn_a"), T_COFF_DefReloc(X64_Addr32Nb, 4, "fn_a"), T_COFF_DefReloc(X64_Addr32Nb, 8, "$unwind$a"), {0} } }, + { "pdata_b", ".pdata", str8_struct(&pdata), .flags = "r:data@4", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr32Nb, 0, "fn_b"), T_COFF_DefReloc(X64_Addr32Nb, 4, "fn_b"), T_COFF_DefReloc(X64_Addr32Nb, 8, "$unwind$b"), {0} } }, + { "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@1", .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr64, 0, "$pdata$a"), T_COFF_DefReloc(X64_Addr64, 8, "$pdata$b"), T_COFF_DefReloc(X64_Addr64, 16, "$unwind$a"), {0} } }, + {0} + }, + .symbols = (T_COFF_DefSymbol[]){ + T_COFF_DefSymbol_Secdef("fn_a", COFF_ComdatSelect_NoDuplicates), T_COFF_DefSymbol_Secdef("fn_b", COFF_ComdatSelect_NoDuplicates), + T_COFF_DefSymbol_Secdef("xdata_a", COFF_ComdatSelect_NoDuplicates), T_COFF_DefSymbol_Secdef("xdata_b", COFF_ComdatSelect_NoDuplicates), + T_COFF_DefSymbol_Secdef("pdata_a", COFF_ComdatSelect_NoDuplicates), T_COFF_DefSymbol_Secdef("pdata_b", COFF_ComdatSelect_NoDuplicates), + T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), T_COFF_DefSymbol_ExternFunc("fn_a", "fn_a", 0), T_COFF_DefSymbol_ExternFunc("fn_b", "fn_b", 0), + T_COFF_DefSymbol_Extern("$unwind$a", "xdata_a", 0), T_COFF_DefSymbol_Extern("$unwind$b", "xdata_b", 0), + T_COFF_DefSymbol_Extern("$pdata$a", "pdata_a", 0), T_COFF_DefSymbol_Extern("$pdata$b", "pdata_b", 0), T_COFF_DefSymbol_Extern("addresses", "addresses", 0), + {0} + } + })); + + t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_pdata_diff.obj"); + T_Ok(g_last_exit_code == 0); + + U64 vaddrs[3] = {0}; + T_Ok(t_read_exe_data_vaddrs(arena, str8_lit("a.exe"), vaddrs, ArrayCount(vaddrs))); + T_Ok(vaddrs[0] != 0); + T_Ok(vaddrs[1] != 0); + T_Ok(vaddrs[0] != vaddrs[1]); +} + +TEST(icf_vftable_and_vbtable_policy) +{ + U8 table_bytes[sizeof(U64)] = {0}; + U8 target_a[] = { 1 }; + U8 target_b[] = { 2 }; + U8 entry_text[] = { 0xc3 }; + U8 addresses[8 * sizeof(U64)] = {0}; + + T_Ok(t_write_def_obj("icf_tables.obj", (T_COFF_DefObj){ + .machine = T_COFF_DefSetMachine(X64), + .sections = (T_COFF_DefSection[]){ + { "entry", ".text", str8_array_fixed(entry_text), .flags = "rx:code@1" }, + { "vf_a", ".rdata$mn", str8_array_fixed(table_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, + { "vf_b", ".rdata$mn", str8_array_fixed(table_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, + { "vb_a", ".rdata$mn", str8_array_fixed(table_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, + { "vb_b", ".rdata$mn", str8_array_fixed(table_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, + { "rtti_a", ".rdata$mn", str8_array_fixed(target_a), .flags = "r:data@1" }, + { "rtti_b", ".rdata$mn", str8_array_fixed(target_b), .flags = "r:data@1" }, + { "vf_ref_a", ".rdata$mn", str8_array_fixed(table_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr64, 0, "rtti_a"), {0} } }, + { "vf_ref_b", ".rdata$mn", str8_array_fixed(table_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr64, 0, "rtti_b"), {0} } }, + { "vb_ref_a", ".rdata$mn", str8_array_fixed(table_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr64, 0, "rtti_a"), {0} } }, + { "vb_ref_b", ".rdata$mn", str8_array_fixed(table_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr64, 0, "rtti_b"), {0} } }, + { "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@1", .relocs = (T_COFF_DefReloc[]){ + T_COFF_DefReloc(X64_Addr64, 0, "??_7a@@6B@"), T_COFF_DefReloc(X64_Addr64, 8, "??_7b@@6B@"), + T_COFF_DefReloc(X64_Addr64, 16, "??_8a@@7B@"), T_COFF_DefReloc(X64_Addr64, 24, "??_8b@@7B@"), + T_COFF_DefReloc(X64_Addr64, 32, "??_7ra@@6B@"), T_COFF_DefReloc(X64_Addr64, 40, "??_7rb@@6B@"), + T_COFF_DefReloc(X64_Addr64, 48, "??_8ra@@7B@"), T_COFF_DefReloc(X64_Addr64, 56, "??_8rb@@7B@"), {0} } }, + {0} + }, + .symbols = (T_COFF_DefSymbol[]){ + T_COFF_DefSymbol_Secdef("vf_a", COFF_ComdatSelect_NoDuplicates), T_COFF_DefSymbol_Secdef("vf_b", COFF_ComdatSelect_NoDuplicates), + T_COFF_DefSymbol_Secdef("vb_a", COFF_ComdatSelect_NoDuplicates), T_COFF_DefSymbol_Secdef("vb_b", COFF_ComdatSelect_NoDuplicates), + T_COFF_DefSymbol_Secdef("vf_ref_a", COFF_ComdatSelect_NoDuplicates), T_COFF_DefSymbol_Secdef("vf_ref_b", COFF_ComdatSelect_NoDuplicates), + T_COFF_DefSymbol_Secdef("vb_ref_a", COFF_ComdatSelect_NoDuplicates), T_COFF_DefSymbol_Secdef("vb_ref_b", COFF_ComdatSelect_NoDuplicates), + T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), + T_COFF_DefSymbol_Extern("??_7a@@6B@", "vf_a", 0), T_COFF_DefSymbol_Extern("??_7b@@6B@", "vf_b", 0), + T_COFF_DefSymbol_Extern("??_8a@@7B@", "vb_a", 0), T_COFF_DefSymbol_Extern("??_8b@@7B@", "vb_b", 0), + T_COFF_DefSymbol_Extern("??_7ra@@6B@", "vf_ref_a", 0), T_COFF_DefSymbol_Extern("??_7rb@@6B@", "vf_ref_b", 0), + T_COFF_DefSymbol_Extern("??_8ra@@7B@", "vb_ref_a", 0), T_COFF_DefSymbol_Extern("??_8rb@@7B@", "vb_ref_b", 0), + T_COFF_DefSymbol_Extern("rtti_a", "rtti_a", 0), T_COFF_DefSymbol_Extern("rtti_b", "rtti_b", 0), T_COFF_DefSymbol_Extern("addresses", "addresses", 0), + {0} + } + })); + + t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_tables.obj"); + T_Ok(g_last_exit_code == 0); + + U64 vaddrs[8] = {0}; + T_Ok(t_read_exe_data_vaddrs(arena, str8_lit("a.exe"), vaddrs, ArrayCount(vaddrs))); + T_Ok(vaddrs[0] == vaddrs[1]); + T_Ok(vaddrs[2] != vaddrs[3]); + T_Ok(vaddrs[4] != vaddrs[5]); + T_Ok(vaddrs[6] != vaddrs[7]); +} + +TEST(icf_readonly_non_vftable_data_policy) +{ + U8 data_bytes[] = { 1, 2, 3, 4 }; + U8 entry_text[] = { 0xc3 }; + U8 addresses[2 * sizeof(U64)] = {0}; + + T_Ok(t_write_def_obj("icf_rdata_policy.obj", (T_COFF_DefObj){ + .machine = T_COFF_DefSetMachine(X64), + .sections = (T_COFF_DefSection[]){ + { "entry", ".text", str8_array_fixed(entry_text), .flags = "rx:code@1" }, + { "data_a", ".rdata$mn", str8_array_fixed(data_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, + { "data_b", ".rdata$mn", str8_array_fixed(data_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, + { "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@1", .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr64, 0, "data_a"), T_COFF_DefReloc(X64_Addr64, 8, "data_b"), {0} } }, + {0} + }, + .symbols = (T_COFF_DefSymbol[]){ + T_COFF_DefSymbol_Secdef("data_a", COFF_ComdatSelect_NoDuplicates), T_COFF_DefSymbol_Secdef("data_b", COFF_ComdatSelect_NoDuplicates), + T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), T_COFF_DefSymbol_Extern("data_a", "data_a", 0), T_COFF_DefSymbol_Extern("data_b", "data_b", 0), T_COFF_DefSymbol_Extern("addresses", "addresses", 0), + {0} + } + })); + + t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_rdata_policy.obj"); + T_Ok(g_last_exit_code == 0); + + U64 vaddrs[2] = {0}; + T_Ok(t_read_exe_data_vaddrs(arena, str8_lit("a.exe"), vaddrs, ArrayCount(vaddrs))); + T_Ok(vaddrs[0] != 0); + T_Ok(vaddrs[1] != 0); + T_Ok(vaddrs[0] != vaddrs[1]); +} + +TEST(icf_cpp_identical_functions_fold) +{ + char source[] = + "extern \"C\" __declspec(noinline) int a(void) { return 42; }\n" + "extern \"C\" __declspec(noinline) int b(void) { return 42; }\n" + "extern \"C\" int (* volatile pa)(void) = a;\n" + "extern \"C\" int (* volatile pb)(void) = b;\n" + "extern \"C\" int entry(void) { return pa == pb ? 0 : 1; }\n"; + + T_Ok(t_write_file(str8_lit("icf_cpp_fold.cpp"), str8_cstring(source))); + T_Ok(t_invoke_cl("/nologo /c /O2 /Gy /Zc:preprocessor /Fo:icf_cpp_fold.obj icf_cpp_fold.cpp")); + T_Ok(g_last_exit_code == 0); + + t_invoke_linkerf("/nodefaultlib /subsystem:console /entry:entry /out:icf_cpp_fold.exe /opt:ref,icf /include:pa /include:pb icf_cpp_fold.obj"); + T_Ok(g_last_exit_code == 0); + + U64 vaddrs[2] = {0}; + T_Ok(t_read_exe_data_vaddrs(arena, str8_lit("icf_cpp_fold.exe"), vaddrs, ArrayCount(vaddrs))); + T_Ok(vaddrs[0] != 0); + T_Ok(vaddrs[0] == vaddrs[1]); +} + +TEST(icf_cpp_multihop_functions_do_not_fold) +{ + char source[] = + "__declspec(noinline) int leaf_a(void) { return 1; }\n" + "__declspec(noinline) int leaf_b(void) { return 2; }\n" + "__declspec(noinline) int mid_a(void) { return leaf_a(); }\n" + "__declspec(noinline) int mid_b(void) { return leaf_b(); }\n" + "__declspec(noinline) int top_a(void) { return mid_a(); }\n" + "__declspec(noinline) int top_b(void) { return mid_b(); }\n" + "int (* volatile p_top_a)(void) = top_a;\n" + "int (* volatile p_top_b)(void) = top_b;\n" + "int (* volatile p_mid_a)(void) = mid_a;\n" + "int (* volatile p_mid_b)(void) = mid_b;\n" + "int (* volatile p_leaf_a)(void) = leaf_a;\n" + "int (* volatile p_leaf_b)(void) = leaf_b;\n" + "int entry(void) {\n" + " if (p_top_a == p_top_b) { return 1; }\n" + " if (p_mid_a == p_mid_b) { return 2; }\n" + " if (p_leaf_a == p_leaf_b) { return 3; }\n" + " return 0;\n" + "}\n"; + + T_Ok(t_write_file(str8_lit("icf_cpp_multihop.c"), str8_cstring(source))); + T_Ok(t_invoke_cl("/nologo /c /O2 /Gy /Zc:preprocessor /Fo:icf_cpp_multihop.obj icf_cpp_multihop.c")); + T_Ok(g_last_exit_code == 0); + + t_invoke_linkerf("/nodefaultlib /subsystem:console /entry:entry /out:icf_cpp_multihop.exe /opt:ref,icf /include:p_top_a /include:p_top_b /include:p_mid_a /include:p_mid_b /include:p_leaf_a /include:p_leaf_b icf_cpp_multihop.obj"); + T_Ok(g_last_exit_code == 0); + + U64 vaddrs[6] = {0}; + T_Ok(t_read_exe_data_vaddrs(arena, str8_lit("icf_cpp_multihop.exe"), vaddrs, ArrayCount(vaddrs))); + for EachIndex(i, ArrayCount(vaddrs)) { + T_Ok(vaddrs[i] != 0); + } + T_Ok(vaddrs[0] != vaddrs[1]); + T_Ok(vaddrs[2] != vaddrs[3]); + T_Ok(vaddrs[4] != vaddrs[5]); +} + #endif #if 0 diff --git a/src/linker/thread_pool/thread_pool.c b/src/linker/thread_pool/thread_pool.c index 0d16213c..3dc2c146 100644 --- a/src/linker/thread_pool/thread_pool.c +++ b/src/linker/thread_pool/thread_pool.c @@ -4,7 +4,7 @@ internal void tp_run_tasks(TP_Context *pool, TP_Worker *worker) { - barrier_wait(pool->barrier); + barrier_wait(pool->run_barrier); for (;;) { S64 task_left = ins_atomic_u64_dec_eval(&pool->task_left); @@ -19,14 +19,11 @@ tp_run_tasks(TP_Context *pool, TP_Worker *worker) U64 task_id = pool->task_count - (task_left+1); pool->task_func(arena, worker->id, task_id, pool->task_data, pool); - // cache task count so we dont touch pool memory after atomic inc - U64 task_count = pool->task_count; - // update task done count ins_atomic_u64_inc_eval(&pool->task_done); } - barrier_wait(pool->barrier); + barrier_wait(pool->run_barrier); } internal void @@ -74,6 +71,7 @@ tp_alloc(Arena *arena, U32 worker_count, U32 max_worker_count, String8 name) // init pool TP_Context *pool = push_array(arena, TP_Context, 1); pool->exec_semaphore = exec_semaphore; + pool->run_barrier = barrier_alloc(worker_count); pool->barrier = barrier_alloc(worker_count); pool->is_live = 1; pool->worker_count = worker_count; @@ -113,6 +111,7 @@ tp_release(TP_Context *pool) if (is_shared) { semaphore_release(pool->exec_semaphore); } + barrier_release(pool->run_barrier); barrier_release(pool->barrier); MemoryZeroStruct(pool); @@ -194,13 +193,14 @@ tp_for_parallel(TP_Context *pool, TP_Arena *task_arena, U64 task_count, TP_TaskF // if we are in shared mode -> ping if (*pool->exec_semaphore.u64) { - U64 drop_count64 = Min(task_count, pool->worker_count); + U64 drop_count64 = pool->worker_count - 1; U32 drop_count = safe_cast_u32(drop_count64); semaphore_drop_count(pool->exec_semaphore, drop_count); } // run tasks on main worker tp_run_tasks(pool, pool->worker_arr); + Assert(pool->task_done == task_count); } } diff --git a/src/linker/thread_pool/thread_pool.h b/src/linker/thread_pool/thread_pool.h index 8d7f8f22..f89f1785 100644 --- a/src/linker/thread_pool/thread_pool.h +++ b/src/linker/thread_pool/thread_pool.h @@ -32,6 +32,7 @@ typedef struct TP_Context Semaphore exec_semaphore; Semaphore task_semaphore; Semaphore main_semaphore; + Barrier run_barrier; Barrier barrier; void *broadcast; U64 broadcast_size; @@ -58,4 +59,3 @@ internal void tp_temp_end(TP_Temp temp); internal void tp_for_parallel(TP_Context *pool, TP_Arena *arena, U64 task_count, TP_TaskFunc *task_func, void *task_data); internal Rng1U64 * tp_divide_work(Arena *arena, U64 item_count, U32 worker_count); #define tp_broadcast(p) tp_broadcast_(tp, task_id, p, sizeof(*p)) -