mirror of
https://github.com/Ed94/raddebugger.git
synced 2026-08-08 06:48:41 +00:00
optimize ICF refinement and COMDAT handling
Cache immutable section and relocation metadata before color refinement, then reuse generated-tagged concurrent tables. Parallelize grouping and fold marking. Include associative COMDAT descendants in contribution hashes and canonicalize relocation targets to selected COMDAT leaders.
This commit is contained in:
Nikita Smith
committed by
Ryan Fleury
Ryan Fleury
parent
abfea7c33a
commit
b84d32d9fb
@@ -34,6 +34,23 @@ void_list_push(Arena *arena, VoidList *list, void *v)
|
||||
return n;
|
||||
}
|
||||
|
||||
internal void
|
||||
u32_list_push_node(U32List *list, U32Node *n)
|
||||
{
|
||||
SLLQueuePush(list->first, list->last, n);
|
||||
list->count += 1;
|
||||
}
|
||||
|
||||
internal U32Node *
|
||||
u32_list_push(Arena *arena, U32List *list, U32 data)
|
||||
{
|
||||
U32Node *n = push_array(arena, U32Node, 1);
|
||||
n->next = 0;
|
||||
n->data = data;
|
||||
u32_list_push_node(list, n);
|
||||
return n;
|
||||
}
|
||||
|
||||
internal void
|
||||
u64_list_push_node(U64List *list, U64Node *n)
|
||||
{
|
||||
@@ -264,4 +281,3 @@ s64_array_from_list(Arena *arena, S64List *list)
|
||||
for EachNode(n, S64Node, list->first) { result.v[result.count++] = n->v; }
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ typedef struct U64Node { U64 data; struct U64Node *next; } U64Node;
|
||||
typedef struct S64Node { S64 v; struct S64Node *next; } S64Node;
|
||||
|
||||
typedef struct VoidList { U64 count; VoidNode *first, *last; } VoidList;
|
||||
typedef struct U32List { U64 count; U32Node *first, *last; } U32List;
|
||||
typedef struct U64List { U64 count; U64Node *first, *last; } U64List;
|
||||
typedef struct S64List { U64 count; S64Node *first, *last; } S64List;
|
||||
|
||||
@@ -20,6 +21,9 @@ internal U64 void_list_count_nodes (VoidNode *head);
|
||||
internal void void_node_concat (VoidNode **head, VoidNode *node);
|
||||
internal void void_node_concat_atomic(VoidNode **head, VoidNode *node);
|
||||
|
||||
internal void u32_list_push_node (U32List *list, U32Node *n);
|
||||
internal U32Node * u32_list_push (Arena *arena, U32List *list, U32 v);
|
||||
|
||||
internal void u64_list_push_node (U64List *list, U64Node *n);
|
||||
internal U64Node * u64_list_push (Arena *arena, U64List *list, U64 v);
|
||||
internal void u64_list_concat_in_place(U64List *list, U64List *to_concat);
|
||||
@@ -44,4 +48,3 @@ internal void s64_list_push_node (S64List *list, S64Node *n);
|
||||
internal S64Node * s64_list_push (Arena *arena, S64List *list, S64 v);
|
||||
internal void s64_list_concat_in_place(S64List *list, S64List *to_concat);
|
||||
internal S64Array s64_array_from_list (Arena *arena, S64List *list);
|
||||
|
||||
|
||||
+419
-198
@@ -2740,27 +2740,13 @@ THREAD_POOL_TASK_FUNC(lnk_opt_ref_task)
|
||||
if (ref_interp == COFF_SymbolValueInterp_Regular) {
|
||||
Temp temp = temp_begin(scratch2.arena);
|
||||
|
||||
HashMap visited_sections_hm = {0};
|
||||
U32Node *stack = push_array(temp.arena, U32Node, 1);
|
||||
stack->data = ref_parsed.section_number;
|
||||
do {
|
||||
U32 section_number = stack->data;
|
||||
SLLStackPop(stack);
|
||||
U32List associated_sections = lnk_obj_collect_associated_sections(temp.arena, ref_symbol.obj, ref_parsed.section_number, 0);
|
||||
|
||||
// is section number valid?
|
||||
if (section_number == 0 || section_number > ref_symbol.obj->header.section_count_no_null) { continue; }
|
||||
// visit root section
|
||||
u32_list_push(temp.arena, &associated_sections, ref_parsed.section_number);
|
||||
|
||||
// detect cyclic associative sections
|
||||
if (hash_map_search_u64_u64(&visited_sections_hm, section_number)) { continue; }
|
||||
hash_map_push_u64_u64(temp.arena, &visited_sections_hm, section_number, 1);
|
||||
|
||||
// push associated section
|
||||
for EachNode(associated_n, U32Node, ref_symbol.obj->associated_sections[section_number]) {
|
||||
if (hash_map_search_u64_u64(&visited_sections_hm, associated_n->data)) { continue; }
|
||||
U32Node *stack_n = push_array(temp.arena, U32Node, 1);
|
||||
stack_n->data = associated_n->data;
|
||||
SLLStackPush(stack, stack_n);
|
||||
}
|
||||
for EachNode(section_n, U32Node, associated_sections.first) {
|
||||
U32 section_number = section_n->data;
|
||||
|
||||
COFF_SectionFlags section_flags = ref_symbol.obj->section_flags[section_number-1];
|
||||
|
||||
@@ -2790,8 +2776,7 @@ THREAD_POOL_TASK_FUNC(lnk_opt_ref_task)
|
||||
}
|
||||
|
||||
batch->v[batch->count++] = refs;
|
||||
|
||||
} while (stack);
|
||||
}
|
||||
|
||||
temp_end(temp);
|
||||
}
|
||||
@@ -2976,6 +2961,17 @@ lnk_icf_color_space_from_section(LNK_Obj *obj, U32 sect_idx)
|
||||
return result;
|
||||
}
|
||||
|
||||
internal void
|
||||
lnk_icf_atomic_min_u64(U64 *dst, U64 value)
|
||||
{
|
||||
// preserve stable leaders despite concurrent insertion
|
||||
for (U64 old_value = ins_atomic_u64_eval(dst); value < old_value;) {
|
||||
U64 observed = ins_atomic_u64_eval_cond_assign(dst, value, old_value);
|
||||
if (observed == old_value) { break; }
|
||||
old_value = observed;
|
||||
}
|
||||
}
|
||||
|
||||
// NOTE: OPT uses a color-refinement algorithm for folding duplicate sections.
|
||||
// If a color group contains multiple distinct hashes, the group is split
|
||||
// and a new refinement round is run. By default, the algorithm loops until
|
||||
@@ -2986,16 +2982,56 @@ lnk_icf_color_space_from_section(LNK_Obj *obj, U32 sect_idx)
|
||||
THREAD_POOL_TASK_FUNC(lnk_opt_icf_task)
|
||||
{
|
||||
ProfBeginFunction();
|
||||
|
||||
typedef struct { U128 hash; U64 old_color; } ColorKey;
|
||||
|
||||
// only target colors vary between rounds, so cache non-recursive relocation data
|
||||
// and rehash target colors each round
|
||||
typedef struct {
|
||||
U64 *color;
|
||||
U64 static_id;
|
||||
U32 value;
|
||||
COFF_SymbolValueInterpType interp;
|
||||
} RelocTarget;
|
||||
|
||||
typedef struct {
|
||||
ColorKey key;
|
||||
U128 static_hash;
|
||||
RelocTarget **reloc_targets;
|
||||
U64 reloc_count;
|
||||
U64 color_slot_idx;
|
||||
U32 obj_idx;
|
||||
U32 sect_idx;
|
||||
LNK_ICF_ColorSpace color_space;
|
||||
} Contrib;
|
||||
|
||||
// reuse both tables without clearing them between refinement rounds
|
||||
typedef struct {
|
||||
U64 state; // generation << 2 | (0 = empty, 1 = initializing, 2 = ready)
|
||||
ColorKey key;
|
||||
U64 first_contrib_idx;
|
||||
U64 old_color_slot_idx;
|
||||
U64 color;
|
||||
} ColorHashSlot;
|
||||
|
||||
typedef struct { ColorHashSlot *slots; U64 slots_count; } ColorHashTable;
|
||||
|
||||
typedef struct {
|
||||
U64 state; // generation << 2 | (0 = empty, 1 = initializing, 2 = ready)
|
||||
U64 old_color;
|
||||
U64 first_contrib_idx;
|
||||
} OldColorHashSlot;
|
||||
|
||||
typedef struct { OldColorHashSlot *slots; U64 slots_count; } OldColorHashTable;
|
||||
|
||||
Temp scratch = scratch_begin(&arena,1);
|
||||
Temp scratch2 = scratch_begin(&scratch.arena,1); // retain relocation metadata through temporary associated-section traversals
|
||||
|
||||
LNK_OptTask *task = raw_task;
|
||||
LNK_Obj **objs = task->objs;
|
||||
|
||||
if (task_id == 0) {
|
||||
lnk_log(LNK_Log_Debug, "/OPT:ICF:");
|
||||
}
|
||||
|
||||
if (task->config->llvm_addrsig == LNK_SwitchState_Yes) {
|
||||
ProfBegin("Flag significant sections");
|
||||
// .llvm_addrsig is an array of ULEB128 symbol indices, which mark sections
|
||||
// whose addresses are significant
|
||||
for EachIndex(i, task->obj_indices[task_id].count) {
|
||||
@@ -3038,6 +3074,7 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task)
|
||||
}
|
||||
}
|
||||
}
|
||||
ProfEnd();
|
||||
barrier_wait(tp->barrier);
|
||||
}
|
||||
|
||||
@@ -3045,15 +3082,14 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task)
|
||||
// step 1: fill out color map and contributions
|
||||
//
|
||||
|
||||
U64 *contrib_counts = 0;
|
||||
{
|
||||
// alloc total section counter
|
||||
U64 *contrib_counts = 0;
|
||||
if (task_id == 0) {
|
||||
contrib_counts = push_array(scratch.arena, U64, task->objs_count);
|
||||
}
|
||||
tp_broadcast(&contrib_counts);
|
||||
|
||||
// count contributions
|
||||
ProfBegin("Count Contributions");
|
||||
for EachIndex(i, task->obj_indices[task_id].count) {
|
||||
U64 obj_idx = task->obj_indices[task_id].v[i];
|
||||
LNK_Obj *obj = objs[obj_idx];
|
||||
@@ -3063,206 +3099,388 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task)
|
||||
}
|
||||
}
|
||||
}
|
||||
ProfEnd();
|
||||
barrier_wait(tp->barrier);
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
struct {
|
||||
U128 hash;
|
||||
U64 old_color;
|
||||
} key;
|
||||
U32 obj_idx;
|
||||
U32 sect_idx;
|
||||
} Contrib;
|
||||
|
||||
U64 contrib_count = sum_array_u64(task->objs_count, contrib_counts);
|
||||
U64 *noncontrib_offsets = 0;
|
||||
U64 *contrib_offsets = 0;
|
||||
U64 **color_map = 0;
|
||||
Contrib *contribs = 0;
|
||||
Rng1U64 *contrib_ranges = 0;
|
||||
U64 *split_counts = 0;
|
||||
U64 *split_offsets = 0;
|
||||
U32 *is_part_stable = 0;
|
||||
U64 *next_color = 0;
|
||||
if (task_id == 0) {
|
||||
contrib_offsets = offsets_from_counts_array_u64(scratch.arena, contrib_counts, task->objs_count);
|
||||
ProfBegin("Init");
|
||||
|
||||
noncontrib_offsets = push_array(scratch.arena, U64, task->objs_count);
|
||||
U64 noncontrib_count = 0;
|
||||
for EachIndex(obj_idx, task->objs_count) {
|
||||
noncontrib_offsets[obj_idx] = noncontrib_count;
|
||||
noncontrib_count += objs[obj_idx]->header.section_count_no_null - contrib_counts[obj_idx];
|
||||
}
|
||||
|
||||
color_map = push_array(scratch.arena, U64 *, task->objs_count);
|
||||
for EachIndex(obj_idx, task->objs_count) {
|
||||
LNK_Obj *obj = objs[obj_idx];
|
||||
color_map[obj_idx] = push_array(scratch.arena, U64, obj->header.section_count_no_null);
|
||||
}
|
||||
|
||||
contrib_offsets = offsets_from_counts_array_u64(scratch.arena, contrib_counts, task->objs_count);
|
||||
contribs = push_array(scratch.arena, Contrib, contrib_count);
|
||||
contrib_ranges = tp_divide_work(scratch.arena, contrib_count, tp->worker_count);
|
||||
split_counts = push_array(scratch.arena, U64, tp->worker_count);
|
||||
split_offsets = push_array(scratch.arena, U64, tp->worker_count + 1);
|
||||
is_part_stable = push_array(scratch.arena, U32, 1);
|
||||
next_color = push_array(scratch.arena, U64, 1);
|
||||
*next_color = LNK_ICF_ColorSpace_COUNT;
|
||||
*next_color = LNK_ICF_ColorSpace_COUNT + noncontrib_count;
|
||||
|
||||
lnk_log(LNK_Log_Debug, " Contrib count: %S", str8_from_count(scratch.arena, contrib_count));
|
||||
|
||||
ProfEnd();
|
||||
}
|
||||
tp_broadcast(&contrib_offsets);
|
||||
tp_broadcast(&noncontrib_offsets);
|
||||
tp_broadcast(&color_map);
|
||||
tp_broadcast(&contribs);
|
||||
tp_broadcast(&contrib_ranges);
|
||||
tp_broadcast(&split_counts);
|
||||
tp_broadcast(&split_offsets);
|
||||
tp_broadcast(&is_part_stable);
|
||||
tp_broadcast(&next_color);
|
||||
|
||||
ProfBegin("Compute Hashes");
|
||||
HashMap reloc_target_hm = {0}; // cache source-symbol resolution so refinement only reads colors and hashes
|
||||
for EachIndex(i, task->obj_indices[task_id].count) {
|
||||
U64 obj_idx = task->obj_indices[task_id].v[i];
|
||||
LNK_Obj *obj = objs[obj_idx];
|
||||
U64 cursor = 0;
|
||||
U64 noncontrib_cursor = 0;
|
||||
for EachIndex(sect_idx, obj->header.section_count_no_null) {
|
||||
LNK_ICF_ColorSpace color_space = lnk_icf_color_space_from_section(obj, sect_idx);
|
||||
if (color_space == LNK_ICF_ColorSpace_Null) {
|
||||
// assign unique color to sections that are not foldable
|
||||
color_map[obj_idx][sect_idx] = ins_atomic_u64_inc_eval(next_color);
|
||||
// assign colors in object order to avoid a contended atomic allocator
|
||||
color_map[obj_idx][sect_idx] = LNK_ICF_ColorSpace_COUNT + noncontrib_offsets[obj_idx] + noncontrib_cursor++;
|
||||
continue;
|
||||
}
|
||||
else {
|
||||
|
||||
// compute contribution index
|
||||
U64 contrib_idx = contrib_offsets[obj_idx] + cursor++;
|
||||
|
||||
// seed foldable sections with a content-derived color
|
||||
COFF_SectionHeader *section_header = lnk_coff_section_header_from_section_number(obj, sect_idx + 1);
|
||||
String8 section_data = str8_substr(obj->data, r1u64s(section_header->foff, section_header->fsize));
|
||||
XXH3_state_t hasher;
|
||||
XXH3_64bits_reset(&hasher);
|
||||
XXH3_64bits_update(&hasher, &color_space, sizeof(color_space));
|
||||
XXH3_64bits_update(&hasher, section_data.str, section_data.size);
|
||||
|
||||
// fill out contribution
|
||||
contribs[contrib_idx] = (Contrib){
|
||||
Contrib *contrib = &contribs[contrib_idx];
|
||||
*contrib = (Contrib){
|
||||
.obj_idx = safe_cast_u32(obj->input_idx),
|
||||
.sect_idx = safe_cast_u32(sect_idx),
|
||||
.color_space = color_space,
|
||||
};
|
||||
|
||||
// assign content-derived starting color value
|
||||
color_map[obj_idx][sect_idx] = XXH3_64bits_digest(&hasher) | (1ull << 63);
|
||||
Temp temp = temp_begin(scratch.arena);
|
||||
// include associative children in their parent COMDAT's identity
|
||||
COFF_SectionFlags associated_filter = COFF_SectionFlag_LnkRemove | COFF_SectionFlag_LnkInfo | COFF_SectionFlag_MemDiscardable | LNK_SECTION_FLAG_DEBUG;
|
||||
U32List associated_sections = lnk_obj_collect_associated_sections(temp.arena, obj, sect_idx + 1, associated_filter);
|
||||
u32_list_push(temp.arena, &associated_sections, sect_idx + 1);
|
||||
|
||||
for EachNode(associated_n, U32Node, associated_sections.first) {
|
||||
COFF_SectionHeader *associated_header = lnk_coff_section_header_from_section_number(obj, associated_n->data);
|
||||
COFF_RelocArray associated_relocs = lnk_coff_relocs_from_section_header(obj, associated_header);
|
||||
contrib->reloc_count += associated_relocs.count;
|
||||
}
|
||||
if (contrib->reloc_count) {
|
||||
contrib->reloc_targets = push_array(scratch2.arena, RelocTarget *, contrib->reloc_count);
|
||||
}
|
||||
|
||||
blake3_hasher hasher; blake3_hasher_init(&hasher);
|
||||
blake3_hasher_update(&hasher, &color_space, sizeof(color_space));
|
||||
|
||||
U64 reloc_cursor = 0;
|
||||
for EachNode(associated_n, U32Node, associated_sections.first) {
|
||||
COFF_SectionHeader *associated_header = lnk_coff_section_header_from_section_number(obj, associated_n->data);
|
||||
String8 associated_data = str8_substr(obj->data, r1u64s(associated_header->foff, associated_header->fsize));
|
||||
COFF_RelocArray associated_relocs = lnk_coff_relocs_from_section_header(obj, associated_header);
|
||||
|
||||
blake3_hasher_update(&hasher, associated_data.str, associated_data.size);
|
||||
blake3_hasher_update(&hasher, &associated_relocs.count, sizeof(associated_relocs.count));
|
||||
|
||||
for EachIndex(reloc_idx, associated_relocs.count) {
|
||||
COFF_Reloc *r = &associated_relocs.v[reloc_idx];
|
||||
|
||||
U64 reloc_key = Compose64Bit(obj->input_idx, r->isymbol);
|
||||
RelocTarget *target = hash_map_search_u64_raw(&reloc_target_hm, reloc_key);
|
||||
if (target == 0) {
|
||||
target = push_array(scratch2.arena, RelocTarget, 1);
|
||||
*target = (RelocTarget){0};
|
||||
|
||||
LNK_ObjSymbolRef target_ref = { .obj = obj, .symbol_idx = r->isymbol };
|
||||
B32 is_symbol_found = lnk_resolve_reloc_target_symbol(scratch2.arena, task->symtab, target_ref, str8_lit("/OPT:ICF"), &target_ref);
|
||||
if (is_symbol_found) {
|
||||
COFF_ParsedSymbol target_symbol = lnk_parsed_symbol_from_coff_symbol_idx(target_ref.obj, target_ref.symbol_idx);
|
||||
target->interp = coff_interp_from_parsed_symbol(target_symbol);
|
||||
target->value = target_symbol.value;
|
||||
|
||||
switch (target->interp) {
|
||||
case COFF_SymbolValueInterp_Regular: {
|
||||
LNK_Obj *target_obj = target_ref.obj;
|
||||
U32 target_sect = target_symbol.section_number;
|
||||
|
||||
// use the selected COMDAT leader so equivalent targets hash alike
|
||||
if (target_sect != 0 && target_sect <= target_obj->header.section_count_no_null &&
|
||||
target_obj->section_flags[target_sect - 1] & COFF_SectionFlag_LnkCOMDAT) {
|
||||
LNK_ObjSymbolRef leader_ref = {0};
|
||||
if (lnk_obj_get_comdat_symlink(target_obj, target_sect, &leader_ref)) {
|
||||
COFF_ParsedSymbol leader_symbol = lnk_parsed_symbol_from_coff_symbol_idx(leader_ref.obj, leader_ref.symbol_idx);
|
||||
if (leader_symbol.section_number != 0 && leader_symbol.section_number <= leader_ref.obj->header.section_count_no_null) {
|
||||
target_obj = leader_ref.obj;
|
||||
target_sect = leader_symbol.section_number;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
target->color = &color_map[target_obj->input_idx][target_sect - 1];
|
||||
} break;
|
||||
default: {
|
||||
target->static_id = Compose64Bit(target_ref.obj->input_idx, target_ref.symbol_idx);
|
||||
} break;
|
||||
}
|
||||
} else {
|
||||
target->interp = max_U32;
|
||||
target->static_id = Compose64Bit(obj_idx, r->isymbol);
|
||||
}
|
||||
|
||||
hash_map_push_u64_raw(scratch2.arena, &reloc_target_hm, reloc_key, target);
|
||||
}
|
||||
|
||||
contrib->reloc_targets[reloc_cursor++] = target;
|
||||
blake3_hasher_update(&hasher, &r->apply_off, sizeof(r->apply_off));
|
||||
blake3_hasher_update(&hasher, &r->type, sizeof(r->type));
|
||||
blake3_hasher_update(&hasher, &target->interp, sizeof(target->interp));
|
||||
blake3_hasher_update(&hasher, &target->value, sizeof(target->value));
|
||||
}
|
||||
}
|
||||
Assert(reloc_cursor == contrib->reloc_count);
|
||||
blake3_hasher_finalize(&hasher, (U8 *)&contrib->static_hash, sizeof(contrib->static_hash));
|
||||
|
||||
// seed foldable sections with their immutable content and relocation shape
|
||||
color_map[obj_idx][sect_idx] = hash_map_hasher(str8_struct(&contrib->static_hash)) | (1ull << 63);
|
||||
temp_end(temp);
|
||||
}
|
||||
}
|
||||
ProfEnd();
|
||||
barrier_wait(tp->barrier);
|
||||
|
||||
ColorHashTable color_table = {0};
|
||||
OldColorHashTable old_color_table = {0};
|
||||
U64 *table_generation = 0;
|
||||
if (task_id == 0) {
|
||||
ProfBegin("Alloc hash tables");
|
||||
color_table.slots_count = u64_up_to_pow2(Max(2, contrib_count*2));
|
||||
color_table.slots = push_array(scratch.arena, ColorHashSlot, color_table.slots_count);
|
||||
old_color_table.slots_count = color_table.slots_count;
|
||||
old_color_table.slots = push_array(scratch.arena, OldColorHashSlot, old_color_table.slots_count);
|
||||
table_generation = push_array_no_zero(scratch.arena, U64, 1);
|
||||
*table_generation = 0;
|
||||
ProfEnd();
|
||||
}
|
||||
tp_broadcast(&color_table);
|
||||
tp_broadcast(&old_color_table);
|
||||
tp_broadcast(&table_generation);
|
||||
|
||||
//
|
||||
// step 2: refine equivalence classes
|
||||
//
|
||||
|
||||
U64 iter_count = 0;
|
||||
for (;; iter_count += 1) {
|
||||
ProfBegin("Round #%llu", iter_count);
|
||||
|
||||
barrier_wait(tp->barrier);
|
||||
// reset color status tracker
|
||||
|
||||
if (task_id == 0) {
|
||||
// reset color status tracker
|
||||
*is_part_stable = 1;
|
||||
|
||||
// update hash tables generations
|
||||
Assert(*table_generation < (max_U64 >> 2));
|
||||
*table_generation += 1;
|
||||
}
|
||||
barrier_wait(tp->barrier);
|
||||
|
||||
// compute colored hashes
|
||||
for EachIndex(i, task->obj_indices[task_id].count) {
|
||||
U64 obj_idx = task->obj_indices[task_id].v[i];
|
||||
LNK_Obj *obj = objs[obj_idx];
|
||||
COFF_SectionHeader *section_table = lnk_coff_section_table_from_obj(obj);
|
||||
U64 contrib_cursor = 0;
|
||||
for EachIndex(sect_idx, obj->header.section_count_no_null) {
|
||||
LNK_ICF_ColorSpace color_space = lnk_icf_color_space_from_section(obj, sect_idx);
|
||||
if (color_space == LNK_ICF_ColorSpace_Null) { continue; }
|
||||
// unpack the table generation
|
||||
U64 table_generation_value = *table_generation;
|
||||
U64 initializing_state = (table_generation_value << 2) | 1;
|
||||
U64 ready_state = (table_generation_value << 2) | 2;
|
||||
|
||||
// parse relocations
|
||||
COFF_SectionHeader *section_header = section_table + sect_idx;
|
||||
COFF_RelocArray relocs = lnk_coff_relocs_from_section_header(obj, section_header);
|
||||
|
||||
U64 contrib_idx = contrib_offsets[obj_idx] + contrib_cursor;
|
||||
contrib_cursor += 1;
|
||||
contribs[contrib_idx].key.old_color = color_map[obj_idx][sect_idx];
|
||||
|
||||
// parse section data
|
||||
String8 section_data = str8_substr(obj->data, r1u64s(section_header->foff, section_header->fsize));
|
||||
ProfBegin("Compute colored hashes");
|
||||
for EachInRange(contrib_idx, contrib_ranges[task_id]) {
|
||||
Contrib *contrib = &contribs[contrib_idx];
|
||||
contrib->key.old_color = color_map[contrib->obj_idx][contrib->sect_idx];
|
||||
|
||||
blake3_hasher hasher; blake3_hasher_init(&hasher);
|
||||
|
||||
// mix section non-recursive properties
|
||||
blake3_hasher_update(&hasher, section_data.str, section_data.size);
|
||||
blake3_hasher_update(&hasher, &relocs.count, sizeof(relocs.count));
|
||||
blake3_hasher_update(&hasher, &color_space, sizeof(color_space));
|
||||
|
||||
for EachIndex(reloc_idx, relocs.count) {
|
||||
COFF_Reloc *r = &relocs.v[reloc_idx];
|
||||
|
||||
// resolve symbol referenced by the relocation
|
||||
LNK_ObjSymbolRef target_ref = { .obj = obj, .symbol_idx = r->isymbol };
|
||||
B32 is_symbol_found = lnk_resolve_reloc_target_symbol(scratch.arena, task->symtab, target_ref, str8_lit("/OPT:ICF"), &target_ref);
|
||||
|
||||
COFF_SymbolValueInterpType target_interp;
|
||||
U64 target_id;
|
||||
U32 target_value;
|
||||
|
||||
if (is_symbol_found) {
|
||||
// parse COFF symbol and interpret the target symbol kind
|
||||
COFF_ParsedSymbol target_symbol = lnk_parsed_symbol_from_coff_symbol_idx(target_ref.obj, target_ref.symbol_idx);
|
||||
target_interp = coff_interp_from_parsed_symbol(target_symbol);
|
||||
|
||||
switch (target_interp) {
|
||||
case COFF_SymbolValueInterp_Regular: {
|
||||
// color relocation with the referenced section
|
||||
target_id = color_map[target_ref.obj->input_idx][target_symbol.section_number - 1];
|
||||
target_value = target_symbol.value;
|
||||
} break;
|
||||
default: {
|
||||
// color relocation with unique symbol ref
|
||||
target_id = Compose64Bit(target_ref.obj->input_idx, target_ref.symbol_idx);
|
||||
target_value = target_symbol.value;
|
||||
} break;
|
||||
}
|
||||
} else {
|
||||
// color relocation with unique symbol ref
|
||||
target_interp = max_U32;
|
||||
target_id = Compose64Bit(obj_idx, r->isymbol);
|
||||
target_value = 0;
|
||||
}
|
||||
|
||||
// mix relocation properties
|
||||
blake3_hasher_update(&hasher, &r->apply_off, sizeof(r->apply_off));
|
||||
blake3_hasher_update(&hasher, &r->type, sizeof(r->type));
|
||||
blake3_hasher_update(&hasher, &target_interp, sizeof(target_interp));
|
||||
blake3_hasher_update(&hasher, &contrib->static_hash, sizeof(contrib->static_hash));
|
||||
for EachIndex(reloc_idx, contrib->reloc_count) {
|
||||
RelocTarget *target = contrib->reloc_targets[reloc_idx];
|
||||
U64 target_id = target->color ? *target->color : target->static_id;
|
||||
blake3_hasher_update(&hasher, &target_id, sizeof(target_id));
|
||||
blake3_hasher_update(&hasher, &target_value, sizeof(target_value));
|
||||
}
|
||||
|
||||
// finalize section hash
|
||||
U128 hash;
|
||||
blake3_hasher_finalize(&hasher, (U8*)&hash, sizeof(hash));
|
||||
blake3_hasher_finalize(&hasher, (U8 *)&hash, sizeof(hash));
|
||||
|
||||
// update contribution hash and color
|
||||
contribs[contrib_idx].key.hash = hash;
|
||||
// insert the colored hash into the concurrent table
|
||||
contrib->key.hash = hash;
|
||||
|
||||
Assert(color_table.slots_count > 0 && (color_table.slots_count & (color_table.slots_count - 1)) == 0);
|
||||
U64 table_hash = hash_map_hasher(str8_struct(&contrib->key));
|
||||
U64 color_slot_idx = table_hash & (color_table.slots_count - 1);
|
||||
for (;;) {
|
||||
ColorHashSlot *color_slot = &color_table.slots[color_slot_idx];
|
||||
U64 state = ins_atomic_u64_eval(&color_slot->state);
|
||||
|
||||
if ((state >> 2) != table_generation_value) {
|
||||
if (ins_atomic_u64_eval_cond_assign(&color_slot->state, initializing_state, state) == state) {
|
||||
color_slot->key = contrib->key;
|
||||
color_slot->first_contrib_idx = contrib_idx;
|
||||
contrib->color_slot_idx = color_slot_idx;
|
||||
ins_atomic_u64_eval_assign(&color_slot->state, ready_state);
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (state == initializing_state) {
|
||||
do { state = ins_atomic_u64_eval(&color_slot->state); } while (state == initializing_state);
|
||||
continue;
|
||||
}
|
||||
|
||||
Assert(state == ready_state);
|
||||
|
||||
if (color_slot->key.old_color == contrib->key.old_color && u128_match(color_slot->key.hash, contrib->key.hash)) {
|
||||
lnk_icf_atomic_min_u64(&color_slot->first_contrib_idx, contrib_idx);
|
||||
contrib->color_slot_idx = color_slot_idx;
|
||||
break;
|
||||
}
|
||||
|
||||
color_slot_idx = (color_slot_idx + 1) & (color_table.slots_count - 1);
|
||||
}
|
||||
}
|
||||
ProfEnd();
|
||||
barrier_wait(tp->barrier);
|
||||
|
||||
// group sections by (color, hash) and handle color splits
|
||||
if (task_id == 0) {
|
||||
Temp temp = temp_begin(scratch.arena);
|
||||
|
||||
HashMap new_color_hm = {0};
|
||||
HashMap old_color_hm = {0};
|
||||
U64 start_next_color = *next_color;
|
||||
|
||||
for EachIndex(contrib_idx, contrib_count) {
|
||||
// publish one old-color record per color group
|
||||
ProfBegin("Index color groups");
|
||||
Rng1U64 contrib_range = contrib_ranges[task_id];
|
||||
for EachInRange(contrib_idx, contrib_range) {
|
||||
Contrib *contrib = &contribs[contrib_idx];
|
||||
U64 *new_color_ptr = hash_map_search_string_u64(&new_color_hm, str8_struct(&contrib->key));
|
||||
if (new_color_ptr == 0) {
|
||||
U64 color;
|
||||
if (hash_map_search_u64_u64(&old_color_hm, contrib->key.old_color) == 0) {
|
||||
color = contrib->key.old_color;
|
||||
hash_map_push_u64_u64(temp.arena, &old_color_hm, contrib->key.old_color, color);
|
||||
} else {
|
||||
// generate a new color for the hash
|
||||
color = ++*next_color;
|
||||
*is_part_stable = 0;
|
||||
ColorHashSlot *slot = &color_table.slots[contrib->color_slot_idx];
|
||||
if (ins_atomic_u64_eval(&slot->first_contrib_idx) == contrib_idx) {
|
||||
Assert(old_color_table.slots_count > 0 && (old_color_table.slots_count & (old_color_table.slots_count - 1)) == 0);
|
||||
U64 old_color = slot->key.old_color;
|
||||
U64 table_hash = hash_map_hasher(str8_struct(&old_color));
|
||||
U64 old_slot_idx = table_hash & (old_color_table.slots_count - 1);
|
||||
for (;;) {
|
||||
OldColorHashSlot *old_color_slot = &old_color_table.slots[old_slot_idx];
|
||||
U64 state = ins_atomic_u64_eval(&old_color_slot->state);
|
||||
|
||||
if ((state >> 2) != table_generation_value) {
|
||||
if (ins_atomic_u64_eval_cond_assign(&old_color_slot->state, initializing_state, state) == state) {
|
||||
old_color_slot->old_color = old_color;
|
||||
old_color_slot->first_contrib_idx = contrib_idx;
|
||||
ins_atomic_u64_eval_assign(&old_color_slot->state, ready_state);
|
||||
break;
|
||||
}
|
||||
new_color_ptr = &hash_map_push_string_u64(temp.arena, &new_color_hm, str8_struct(&contrib->key), color)->v.value.value_u64;
|
||||
}
|
||||
color_map[contrib->obj_idx][contrib->sect_idx] = *new_color_ptr;
|
||||
continue;
|
||||
}
|
||||
|
||||
U64 split_count = *next_color - start_next_color;
|
||||
lnk_log(LNK_Log_Debug, " Round %llu found %S splits", iter_count, str8_from_count(scratch.arena, split_count));
|
||||
if (state == initializing_state) {
|
||||
do { state = ins_atomic_u64_eval(&old_color_slot->state); } while (state == initializing_state);
|
||||
continue;
|
||||
}
|
||||
|
||||
temp_end(temp);
|
||||
Assert(state == ready_state);
|
||||
|
||||
if (old_color_slot->old_color == old_color) {
|
||||
lnk_icf_atomic_min_u64(&old_color_slot->first_contrib_idx, contrib_idx);
|
||||
break;
|
||||
}
|
||||
|
||||
old_slot_idx = (old_slot_idx + 1) & (old_color_table.slots_count - 1);
|
||||
}
|
||||
slot->old_color_slot_idx = old_slot_idx;
|
||||
}
|
||||
}
|
||||
ProfEnd();
|
||||
barrier_wait(tp->barrier);
|
||||
|
||||
// count split groups in deterministic contribution order
|
||||
ProfBegin("Count color splits");
|
||||
U64 split_count = 0;
|
||||
for EachInRange(contrib_idx, contrib_range) {
|
||||
Contrib *contrib = &contribs[contrib_idx];
|
||||
ColorHashSlot *color_slot = &color_table.slots[contrib->color_slot_idx];
|
||||
if (ins_atomic_u64_eval(&color_slot->first_contrib_idx) != contrib_idx) { continue; }
|
||||
|
||||
OldColorHashSlot *old_color_slot = &old_color_table.slots[color_slot->old_color_slot_idx];
|
||||
if (ins_atomic_u64_eval(&old_color_slot->first_contrib_idx) != contrib_idx) {
|
||||
split_count += 1;
|
||||
}
|
||||
}
|
||||
split_counts[task_id] = split_count;
|
||||
ProfEnd();
|
||||
barrier_wait(tp->barrier);
|
||||
|
||||
// assign deterministic color ranges with a small serial prefix sum
|
||||
if (task_id == 0) {
|
||||
ProfBegin("Prefix color splits");
|
||||
|
||||
U64 start_next_color = *next_color;
|
||||
U64 total_split_count = 0;
|
||||
for EachIndex(worker_id, tp->worker_count) {
|
||||
split_offsets[worker_id] = total_split_count;
|
||||
total_split_count += split_counts[worker_id];
|
||||
}
|
||||
split_offsets[tp->worker_count] = start_next_color;
|
||||
|
||||
*next_color += total_split_count;
|
||||
*is_part_stable = (total_split_count == 0);
|
||||
|
||||
lnk_log(LNK_Log_Debug, " Round %llu found %S splits", iter_count, str8_from_count(scratch.arena, total_split_count));
|
||||
|
||||
ProfEnd();
|
||||
}
|
||||
barrier_wait(tp->barrier);
|
||||
|
||||
// assign old and split colors in deterministic contribution order
|
||||
ProfBegin("Assign colors");
|
||||
U64 next_split_color = split_offsets[tp->worker_count] + split_offsets[task_id];
|
||||
for EachInRange(contrib_idx, contrib_range) {
|
||||
Contrib *contrib = &contribs[contrib_idx];
|
||||
ColorHashSlot *color_slot = &color_table.slots[contrib->color_slot_idx];
|
||||
if (ins_atomic_u64_eval(&color_slot->first_contrib_idx) != contrib_idx) { continue; }
|
||||
|
||||
OldColorHashSlot *old_color_slot = &old_color_table.slots[color_slot->old_color_slot_idx];
|
||||
if (ins_atomic_u64_eval(&old_color_slot->first_contrib_idx) == contrib_idx) {
|
||||
color_slot->color = color_slot->key.old_color;
|
||||
} else {
|
||||
color_slot->color = ++next_split_color;
|
||||
}
|
||||
}
|
||||
ProfEnd();
|
||||
barrier_wait(tp->barrier);
|
||||
|
||||
// update colors for this worker's contributions
|
||||
ProfBegin("Update color map");
|
||||
for EachIndex(i, task->obj_indices[task_id].count) {
|
||||
U64 obj_idx = task->obj_indices[task_id].v[i];
|
||||
Rng1U64 obj_contrib_range = r1u64(contrib_offsets[obj_idx], contrib_offsets[obj_idx] + contrib_counts[obj_idx]);
|
||||
for EachInRange(contrib_idx, obj_contrib_range) {
|
||||
Contrib *contrib = &contribs[contrib_idx];
|
||||
color_map[contrib->obj_idx][contrib->sect_idx] = color_table.slots[contrib->color_slot_idx].color;
|
||||
}
|
||||
}
|
||||
ProfEnd();
|
||||
barrier_wait(tp->barrier);
|
||||
|
||||
ProfEnd(); // round prof
|
||||
|
||||
// stop iterating when partitions stabilize
|
||||
if (*is_part_stable) { break; }
|
||||
}
|
||||
@@ -3272,49 +3490,47 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task)
|
||||
// step 3: flag folded sections for removal
|
||||
//
|
||||
|
||||
if (task_id == 0) {
|
||||
HashMap leader_hm = { 0 };
|
||||
for EachIndex(contrib_idx, contrib_count) {
|
||||
Contrib *contrib = &contribs[contrib_idx];
|
||||
U64 color = color_map[contrib->obj_idx][contrib->sect_idx];
|
||||
if (hash_map_search_u64_raw(&leader_hm, color) == 0) {
|
||||
hash_map_push_u64_raw(scratch.arena, &leader_hm, color, contrib);
|
||||
}
|
||||
}
|
||||
|
||||
typedef struct { U64 count; U64 size; } FoldStats;
|
||||
FoldStats fold_stats[LNK_ICF_ColorSpace_COUNT] = {0};
|
||||
FoldStats *fold_stats = 0;
|
||||
if (task_id == 0 && lnk_get_log_status(LNK_Log_Debug)) {
|
||||
fold_stats = push_array(scratch.arena, FoldStats, tp->worker_count * LNK_ICF_ColorSpace_COUNT);
|
||||
}
|
||||
tp_broadcast(&fold_stats);
|
||||
|
||||
for EachIndex(contrib_idx, contrib_count) {
|
||||
ProfBegin("Flag Folds");
|
||||
FoldStats *local_fold_stats = fold_stats ? fold_stats + (task_id * LNK_ICF_ColorSpace_COUNT) : 0;
|
||||
for EachInRange(contrib_idx, contrib_ranges[task_id]) {
|
||||
Contrib *contrib = &contribs[contrib_idx];
|
||||
Contrib *leader = hash_map_search_u64_raw(&leader_hm, color_map[contrib->obj_idx][contrib->sect_idx]);
|
||||
if (leader == 0 || leader == contrib) { continue; }
|
||||
ColorHashSlot *color_slot = &color_table.slots[contrib->color_slot_idx];
|
||||
Contrib *leader = &contribs[ins_atomic_u64_eval(&color_slot->first_contrib_idx)];
|
||||
if (leader == contrib) { continue; }
|
||||
|
||||
LNK_Obj *contrib_obj = objs[contrib->obj_idx];
|
||||
LNK_Obj *leader_obj = objs[leader->obj_idx];
|
||||
|
||||
// update fold stats
|
||||
if (lnk_get_log_status(LNK_Log_Debug)) {
|
||||
if (local_fold_stats) {
|
||||
COFF_SectionHeader *section_header = lnk_coff_section_header_from_section_number(contrib_obj, contrib->sect_idx + 1);
|
||||
LNK_ICF_ColorSpace color_space = lnk_icf_color_space_from_section(contrib_obj, contrib->sect_idx);
|
||||
fold_stats[color_space].count += 1;
|
||||
fold_stats[color_space].size += section_header->fsize;
|
||||
LNK_ICF_ColorSpace color_space = contrib->color_space;
|
||||
local_fold_stats[color_space].count += 1;
|
||||
local_fold_stats[color_space].size += section_header->fsize;
|
||||
}
|
||||
|
||||
// update leader with largest alignment
|
||||
U64 leader_align = coff_align_size_from_section_flags(leader_obj->section_flags[leader->sect_idx]);
|
||||
U64 contrib_align = coff_align_size_from_section_flags(contrib_obj->section_flags[contrib->sect_idx]);
|
||||
if (leader_align < contrib_align) {
|
||||
leader_obj->section_flags[leader->sect_idx] &= ~(COFF_SectionFlag_AlignMask << COFF_SectionFlag_AlignShift);
|
||||
leader_obj->section_flags[leader->sect_idx] |= coff_section_flag_from_align_size(contrib_align);
|
||||
COFF_SectionFlags *leader_flags = &leader_obj->section_flags[leader->sect_idx];
|
||||
for (COFF_SectionFlags old_flags = ins_atomic_u32_eval((U32 *)leader_flags);;) {
|
||||
U64 leader_align = coff_align_size_from_section_flags(old_flags);
|
||||
if (leader_align >= contrib_align) { break; }
|
||||
|
||||
COFF_SectionFlags new_flags = old_flags;
|
||||
new_flags &= ~(COFF_SectionFlag_AlignMask << COFF_SectionFlag_AlignShift);
|
||||
new_flags |= coff_section_flag_from_align_size(contrib_align);
|
||||
COFF_SectionFlags observed = ins_atomic_u32_eval_cond_assign((U32 *)leader_flags, new_flags, old_flags);
|
||||
if (observed == old_flags) { break; }
|
||||
old_flags = observed;
|
||||
}
|
||||
|
||||
// update COMDAT leader symbol link
|
||||
Assert(leader_obj->comdats[leader->sect_idx] != max_U32);
|
||||
LNK_ObjSymbolRef leader_symlink = { leader_obj, leader_obj->comdats[leader->sect_idx] };
|
||||
contrib_obj->symlinks[contrib->sect_idx + 1] = leader_symlink;
|
||||
|
||||
// discard folded section
|
||||
contrib_obj->symlinks[contrib->sect_idx + 1] = (LNK_ObjSymbolRef){ leader_obj, leader_obj->comdats[leader->sect_idx] };
|
||||
contrib_obj->section_flags[contrib->sect_idx] |= COFF_SectionFlag_LnkRemove;
|
||||
|
||||
#if LNK_PARANOID
|
||||
@@ -3323,23 +3539,34 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task)
|
||||
lnk_log(LNK_Log_Debug, "fold %.*s[SECT%X \"%.*s\"] ==> %.*s[SECT%X \"%.*s\"]", str8_varg(lnk_loc_from_obj(scratch.arena, contrib_obj)), contrib->sect_idx+1, str8_varg(section_name), str8_varg(lnk_loc_from_obj(scratch.arena, leader_obj)), leader->sect_idx+1, str8_varg(leader_name));
|
||||
#endif
|
||||
}
|
||||
ProfEnd();
|
||||
barrier_wait(tp->barrier);
|
||||
|
||||
if (task_id == 0 && fold_stats) {
|
||||
FoldStats total_stats[LNK_ICF_ColorSpace_COUNT] = {0};
|
||||
for EachIndex(worker_id, tp->worker_count) {
|
||||
FoldStats *worker_stats = fold_stats + (worker_id * LNK_ICF_ColorSpace_COUNT);
|
||||
for (LNK_ICF_ColorSpace color_space = (LNK_ICF_ColorSpace)(LNK_ICF_ColorSpace_Null + 1); color_space < LNK_ICF_ColorSpace_COUNT; color_space += 1) {
|
||||
total_stats[color_space].count += worker_stats[color_space].count;
|
||||
total_stats[color_space].size += worker_stats[color_space].size;
|
||||
}
|
||||
}
|
||||
|
||||
if (lnk_get_log_status(LNK_Log_Debug)) {
|
||||
U64 total_count = 0, total_size = 0;
|
||||
for (LNK_ICF_ColorSpace color_space = (LNK_ICF_ColorSpace)(LNK_ICF_ColorSpace_Null + 1); color_space < LNK_ICF_ColorSpace_COUNT; color_space += 1) {
|
||||
lnk_log(LNK_Log_Debug, " %-8S: %M, %.*s sections", lnk_string_from_icf_color_space(color_space), fold_stats[color_space].size, str8_varg(str8_from_count(scratch.arena, fold_stats[color_space].count)));
|
||||
total_count += fold_stats[color_space].count;
|
||||
total_size += fold_stats[color_space].size;
|
||||
lnk_log(LNK_Log_Debug, " %-8S: %M, %.*s sections", lnk_string_from_icf_color_space(color_space), total_stats[color_space].size, str8_varg(str8_from_count(scratch.arena, total_stats[color_space].count)));
|
||||
total_count += total_stats[color_space].count;
|
||||
total_size += total_stats[color_space].size;
|
||||
}
|
||||
lnk_log(LNK_Log_Debug, " %-8s: %M, %.*s sections", "Total", total_size, str8_varg(str8_from_count(scratch.arena, total_count)));
|
||||
}
|
||||
}
|
||||
barrier_wait(tp->barrier);
|
||||
|
||||
//
|
||||
// step 4: flatten COMDAT symlink chains so subsequent passes can assume symlinks are single hop
|
||||
//
|
||||
|
||||
ProfBegin("Flatten COMDAT Symbol Links");
|
||||
for EachIndex(i, task->obj_indices[task_id].count) {
|
||||
U64 obj_idx = task->obj_indices[task_id].v[i];
|
||||
LNK_Obj *obj = objs[obj_idx];
|
||||
@@ -3349,26 +3576,20 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task)
|
||||
LNK_ObjSymbolRef symlink_ref = {0};
|
||||
if (!lnk_obj_get_comdat_symlink(obj, section_number, &symlink_ref)) { continue; }
|
||||
|
||||
{
|
||||
Temp temp = temp_begin(scratch.arena);
|
||||
|
||||
HashMap seen_hm = {0};
|
||||
U64 hop_count = 0;
|
||||
U64 hop_cap = 1024;
|
||||
for(; hop_count < hop_cap; hop_count += 1) {
|
||||
COFF_ParsedSymbol symlink_parsed = lnk_parsed_symbol_from_coff_symbol_idx(symlink_ref.obj, symlink_ref.symbol_idx);
|
||||
|
||||
LNK_ObjSymbolRef next_symlink_ref = {0};
|
||||
if (!lnk_obj_get_comdat_symlink(symlink_ref.obj, symlink_parsed.section_number, &next_symlink_ref)) { break; }
|
||||
|
||||
if (MemoryMatchStruct(&next_symlink_ref, &symlink_ref)) { break; }
|
||||
|
||||
if (hash_map_search_string_u64(&seen_hm, str8_struct(&next_symlink_ref)) != 0) {
|
||||
lnk_error_obj(LNK_Error_IllData, obj, "recursive COMDAT symlink in SECT%X", section_number);
|
||||
MemoryZeroStruct(&symlink_ref);
|
||||
break;
|
||||
}
|
||||
|
||||
symlink_ref = next_symlink_ref;
|
||||
hash_map_push_string_u64(temp.arena, &seen_hm, str8_copy(temp.arena, str8_struct(&symlink_ref)), 1);
|
||||
}
|
||||
@@ -3376,15 +3597,15 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task)
|
||||
lnk_error_obj(LNK_Error_IllData, obj, "failed to flatten symlink for SECT%X; max number of hops reached", section_number);
|
||||
MemoryZeroStruct(&symlink_ref);
|
||||
}
|
||||
|
||||
temp_end(temp);
|
||||
}
|
||||
|
||||
obj->symlinks[section_number] = symlink_ref;
|
||||
}
|
||||
}
|
||||
ProfEnd();
|
||||
barrier_wait(tp->barrier);
|
||||
|
||||
scratch_end(scratch2);
|
||||
scratch_end(scratch);
|
||||
ProfEnd();
|
||||
}
|
||||
@@ -3392,16 +3613,16 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task)
|
||||
internal void
|
||||
lnk_opt_icf(TP_Context *tp, LNK_SymbolTable *symtab, LNK_Config *config, LNK_Obj **objs, U64 objs_count)
|
||||
{
|
||||
ProfBegin("/OPT:ICF");
|
||||
Temp scratch = scratch_begin(0,0);
|
||||
U32Array *obj_indices = lnk_obj_indices_from_section_counts(scratch.arena, tp->worker_count, objs, objs_count);
|
||||
|
||||
ProfScope("/OPT:ICF")
|
||||
{
|
||||
lnk_log(LNK_Log_Debug, "/OPT:ICF:");
|
||||
U32Array *obj_indices = lnk_obj_indices_from_section_counts(scratch.arena, tp->worker_count, objs, objs_count);
|
||||
LNK_OptTask task = { .symtab = symtab, .config = config, .objs = objs, .objs_count = objs_count, .obj_indices = obj_indices };
|
||||
tp_for_parallel(tp, 0, tp->worker_count, lnk_opt_icf_task, &task);
|
||||
}
|
||||
|
||||
scratch_end(scratch);
|
||||
ProfEnd();
|
||||
}
|
||||
|
||||
internal int
|
||||
|
||||
@@ -517,6 +517,45 @@ lnk_symlinks_from_obj(Arena *arena, LNK_SymbolTable *symtab, LNK_Obj *obj)
|
||||
return symlinks;
|
||||
}
|
||||
|
||||
internal U32List
|
||||
lnk_obj_collect_associated_sections(Arena *arena, LNK_Obj *obj, U32 root_section, COFF_SectionFlags skip_flags)
|
||||
{
|
||||
Temp scratch = scratch_begin(&arena, 1);
|
||||
|
||||
// track each child before enqueueing it because COFF associations can cycle
|
||||
HashMap seen_hm = {0};
|
||||
U32List queue = {0};
|
||||
|
||||
U32Node root_n = { root_section };
|
||||
u32_list_push_node(&queue, &root_n);
|
||||
hash_map_push_u64_u64(scratch.arena, &seen_hm, root_section, 1);
|
||||
|
||||
// walk the complete descendant chain because associated COMDATs can nest
|
||||
for EachNode(parent_n, U32Node, queue.first) {
|
||||
for EachNode(associated_n, U32Node, obj->associated_sections[parent_n->data]) {
|
||||
U32 child_section = associated_n->data;
|
||||
|
||||
if (child_section == 0) { continue; }
|
||||
if (hash_map_search_u64_u64(&seen_hm, child_section)) { continue; }
|
||||
if (obj->section_flags[child_section - 1] & skip_flags) { continue; }
|
||||
|
||||
hash_map_push_u64_u64(scratch.arena, &seen_hm, child_section, 1);
|
||||
u32_list_push(arena, &queue, child_section);
|
||||
}
|
||||
}
|
||||
|
||||
// return only child sections so callers choose whether the root participates
|
||||
U32List result = {0};
|
||||
if (queue.count > 1) {
|
||||
result.first = queue.first->next;
|
||||
result.last = queue.last;
|
||||
result.count = queue.count - 1;
|
||||
}
|
||||
|
||||
scratch_end(scratch);
|
||||
return result;
|
||||
}
|
||||
|
||||
internal
|
||||
THREAD_POOL_TASK_FUNC(lnk_assign_comdat_symlinks_task)
|
||||
{
|
||||
|
||||
@@ -143,6 +143,7 @@ internal struct LNK_Lib * lnk_obj_get_lib(LNK_Obj *obj);
|
||||
internal String8 lnk_obj_get_lib_path(LNK_Obj *obj);
|
||||
internal U32 lnk_obj_get_removed_section_number(LNK_Obj *obj);
|
||||
internal B32 lnk_obj_get_comdat_symlink(LNK_Obj *obj, U64 section_number, LNK_ObjSymbolRef *symlink_out);
|
||||
internal U32List lnk_obj_collect_associated_sections(Arena *arena, LNK_Obj *obj, U32 root_section, COFF_SectionFlags skip_flags);
|
||||
|
||||
// --- Symbol & Section Helpers ------------------------------------------------
|
||||
|
||||
|
||||
@@ -8083,6 +8083,123 @@ TEST(icf_fold_two_funcs)
|
||||
T_Ok(str8_match(text_data, str8_array_fixed(expected_text), 0));
|
||||
}
|
||||
|
||||
TEST(icf_associative_child_prevents_fold)
|
||||
{
|
||||
U8 ret_text[] = { 0xc3 };
|
||||
U8 handler_a[] = { 1, 2, 3, 4 };
|
||||
U8 handler_b[] = { 4, 3, 2, 1 };
|
||||
U8 addresses[2 * sizeof(U64)] = {0};
|
||||
|
||||
T_Ok(t_write_def_obj("icf_associative_child.obj", (T_COFF_DefObj){
|
||||
.machine = T_COFF_DefSetMachine(X64),
|
||||
.sections = (T_COFF_DefSection[]){
|
||||
{ "entry", ".text", str8_array_fixed(ret_text), .flags = "rx:code@1" },
|
||||
{ "fn_a", ".text$mn", str8_array_fixed(ret_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT },
|
||||
{ "fn_b", ".text$mn", str8_array_fixed(ret_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT },
|
||||
{ "handler_a", ".xdata", str8_array_fixed(handler_a), .flags = "r:data@4", .raw_flags = COFF_SectionFlag_LnkCOMDAT },
|
||||
{ "handler_b", ".xdata", str8_array_fixed(handler_b), .flags = "r:data@4", .raw_flags = COFF_SectionFlag_LnkCOMDAT },
|
||||
{
|
||||
"addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@1",
|
||||
.relocs = (T_COFF_DefReloc[]){
|
||||
T_COFF_DefReloc(X64_Addr64, 0, "fn_a"),
|
||||
T_COFF_DefReloc(X64_Addr64, sizeof(U64), "fn_b"),
|
||||
{0}
|
||||
}
|
||||
},
|
||||
{0}
|
||||
},
|
||||
.symbols = (T_COFF_DefSymbol[]){
|
||||
T_COFF_DefSymbol_Secdef("fn_a", COFF_ComdatSelect_NoDuplicates),
|
||||
T_COFF_DefSymbol_Secdef("fn_b", COFF_ComdatSelect_NoDuplicates),
|
||||
T_COFF_DefSymbol_Associative("handler_a", "fn_a"),
|
||||
T_COFF_DefSymbol_Associative("handler_b", "fn_b"),
|
||||
T_COFF_DefSymbol_ExternFunc("entry", "entry", 0),
|
||||
T_COFF_DefSymbol_ExternFunc("fn_a", "fn_a", 0),
|
||||
T_COFF_DefSymbol_ExternFunc("fn_b", "fn_b", 0),
|
||||
T_COFF_DefSymbol_Extern("addresses", "addresses", 0),
|
||||
{0}
|
||||
}
|
||||
}));
|
||||
|
||||
t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_associative_child.obj");
|
||||
T_Ok(g_last_exit_code == 0);
|
||||
|
||||
U64 vaddrs[ArrayCount(addresses) / sizeof(U64)] = {0};
|
||||
T_Ok(t_read_exe_data_vaddrs(arena, str8_lit("a.exe"), vaddrs, ArrayCount(vaddrs)));
|
||||
T_Ok(vaddrs[0] != 0);
|
||||
T_Ok(vaddrs[1] != 0);
|
||||
T_Ok(vaddrs[0] != vaddrs[1]);
|
||||
}
|
||||
|
||||
TEST(icf_comdat_reloc_targets_fold)
|
||||
{
|
||||
U8 fn_text[] = {
|
||||
0x48, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, // mov rax, shared
|
||||
0xc3, // ret
|
||||
};
|
||||
U8 shared_data[] = { 0 };
|
||||
U8 entry_text[] = { 0xc3 };
|
||||
U8 addresses[2 * sizeof(U64)] = {0};
|
||||
|
||||
T_Ok(t_write_def_obj("icf_comdat_reloc_a.obj", (T_COFF_DefObj){
|
||||
.machine = T_COFF_DefSetMachine(X64),
|
||||
.sections = (T_COFF_DefSection[]){
|
||||
{ "fn_a", ".text$mn", str8_array_fixed(fn_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr64, 2, "shared_local_a"), {0} } },
|
||||
{ "shared", ".rdata", str8_array_fixed(shared_data), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT },
|
||||
{0}
|
||||
},
|
||||
.symbols = (T_COFF_DefSymbol[]){
|
||||
T_COFF_DefSymbol_Secdef("fn_a", COFF_ComdatSelect_NoDuplicates),
|
||||
T_COFF_DefSymbol_Secdef("shared", COFF_ComdatSelect_Any),
|
||||
T_COFF_DefSymbol_ExternFunc("fn_a", "fn_a", 0),
|
||||
T_COFF_DefSymbol_Extern("shared", "shared", 0),
|
||||
T_COFF_DefSymbol_Static("shared_local_a", "shared", 0),
|
||||
{0}
|
||||
}
|
||||
}));
|
||||
|
||||
T_Ok(t_write_def_obj("icf_comdat_reloc_b.obj", (T_COFF_DefObj){
|
||||
.machine = T_COFF_DefSetMachine(X64),
|
||||
.sections = (T_COFF_DefSection[]){
|
||||
{ "fn_b", ".text$mn", str8_array_fixed(fn_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr64, 2, "shared_local_b"), {0} } },
|
||||
{ "shared", ".rdata", str8_array_fixed(shared_data), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT },
|
||||
{0}
|
||||
},
|
||||
.symbols = (T_COFF_DefSymbol[]){
|
||||
T_COFF_DefSymbol_Secdef("fn_b", COFF_ComdatSelect_NoDuplicates),
|
||||
T_COFF_DefSymbol_Secdef("shared", COFF_ComdatSelect_Any),
|
||||
T_COFF_DefSymbol_ExternFunc("fn_b", "fn_b", 0),
|
||||
T_COFF_DefSymbol_Extern("shared", "shared", 0),
|
||||
T_COFF_DefSymbol_Static("shared_local_b", "shared", 0),
|
||||
{0}
|
||||
}
|
||||
}));
|
||||
|
||||
T_Ok(t_write_def_obj("icf_comdat_reloc_entry.obj", (T_COFF_DefObj){
|
||||
.machine = T_COFF_DefSetMachine(X64),
|
||||
.sections = (T_COFF_DefSection[]){
|
||||
{ "entry", ".text", str8_array_fixed(entry_text), .flags = "rx:code@1" },
|
||||
{ "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@1", .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr64, 0, "fn_a"), T_COFF_DefReloc(X64_Addr64, sizeof(U64), "fn_b"), {0} } },
|
||||
{0}
|
||||
},
|
||||
.symbols = (T_COFF_DefSymbol[]){
|
||||
T_COFF_DefSymbol_ExternFunc("entry", "entry", 0),
|
||||
T_COFF_DefSymbol_UndefFunc("fn_a"),
|
||||
T_COFF_DefSymbol_UndefFunc("fn_b"),
|
||||
T_COFF_DefSymbol_Extern("addresses", "addresses", 0),
|
||||
{0}
|
||||
}
|
||||
}));
|
||||
|
||||
t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_comdat_reloc_entry.obj icf_comdat_reloc_a.obj icf_comdat_reloc_b.obj");
|
||||
T_Ok(g_last_exit_code == 0);
|
||||
|
||||
U64 vaddrs[ArrayCount(addresses) / sizeof(U64)] = {0};
|
||||
T_Ok(t_read_exe_data_vaddrs(arena, str8_lit("a.exe"), vaddrs, ArrayCount(vaddrs)));
|
||||
T_Ok(vaddrs[0] != 0);
|
||||
T_Ok(vaddrs[0] == vaddrs[1]);
|
||||
}
|
||||
|
||||
TEST(icf_same_but_different)
|
||||
{
|
||||
U8 text[] = {
|
||||
@@ -9350,4 +9467,3 @@ TEST(lib_member_reloc_apply_off_out_of_bounds)
|
||||
T_Ok(g_last_exit_code != 0);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
Reference in New Issue
Block a user