diff --git a/src/linker/base_ext/base_radix_sort.c b/src/linker/base_ext/base_radix_sort.c new file mode 100644 index 00000000..f4c07fab --- /dev/null +++ b/src/linker/base_ext/base_radix_sort.c @@ -0,0 +1,93 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +#define U64_RADIX_BITS 8 +#define U64_RADIX_SIZE (1 << U64_RADIX_BITS) +#define U64_RADIX_PARALLEL_MIN 140000 + +typedef struct U64RadixTask +{ + Rng1U64 *ranges; + U64 *src; + U64 *dst; + U64 *offsets; + U32 shift; +} U64RadixTask; + +internal +THREAD_POOL_TASK_FUNC(u64_radix_hist_task) +{ + U64RadixTask *task = raw_task; + U64 *counts = task->offsets + task_id * U64_RADIX_SIZE; + MemoryZeroTyped(counts, U64_RADIX_SIZE); + for EachInRange(i, task->ranges[task_id]) { + U64 digit = (task->src[i] >> task->shift) & (U64_RADIX_SIZE - 1); + counts[digit] += 1; + } +} + +internal +THREAD_POOL_TASK_FUNC(u64_radix_scatter_task) +{ + U64RadixTask *task = raw_task; + U64 *offsets = task->offsets + task_id * U64_RADIX_SIZE; + for EachInRange(i, task->ranges[task_id]) { + U64 value = task->src[i]; + U64 digit = (value >> task->shift) & (U64_RADIX_SIZE - 1); + task->dst[offsets[digit]++] = value; + } +} + +internal void +u64_array_sort_radix_parallel(TP_Context *tp, U64 count, U64 *v) +{ + if (count < U64_RADIX_PARALLEL_MIN) { + u64_array_sort(count, v); + return; + } + + U64 max_value = u64_array_max(count, v); + if (max_value == 0) { return; } + + Temp scratch = scratch_begin(0,0); + + U64 task_count = tp->worker_count; + U64 *buffer = push_array_no_zero(scratch.arena, U64, count); + U64 *offsets = push_array_no_zero(scratch.arena, U64, task_count * U64_RADIX_SIZE); + U64RadixTask task = { + .ranges = tp_divide_work(scratch.arena, count, task_count), + .src = v, + .dst = buffer, + .offsets = offsets, + }; + + U64 pass_count = (64 - clz64(max_value) + U64_RADIX_BITS - 1) / U64_RADIX_BITS; + for EachIndex(pass_idx, pass_count) { + task.shift = safe_cast_u32(pass_idx * U64_RADIX_BITS); + tp_for_parallel(tp, 0, task_count, u64_radix_hist_task, &task); + + U64 cursor = 0; + for EachIndex(digit, U64_RADIX_SIZE) { + for EachIndex(task_idx, task_count) { + U64 *slot = &offsets[task_idx * U64_RADIX_SIZE + digit]; + U64 slot_count = *slot; + *slot = cursor; + cursor += slot_count; + } + } + Assert(cursor == count); + + tp_for_parallel(tp, 0, task_count, u64_radix_scatter_task, &task); + Swap(U64 *, task.src, task.dst); + } + + if (task.src != v) { + MemoryCopyTyped(v, task.src, count); + } + + scratch_end(scratch); +} + +#undef U64_RADIX_BITS +#undef U64_RADIX_SIZE +#undef U64_RADIX_PARALLEL_MIN diff --git a/src/linker/base_ext/base_radix_sort.h b/src/linker/base_ext/base_radix_sort.h new file mode 100644 index 00000000..b89c4547 --- /dev/null +++ b/src/linker/base_ext/base_radix_sort.h @@ -0,0 +1,6 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +#pragma once + +internal void u64_array_sort_radix_parallel(TP_Context *tp, U64 count, U64 *v); diff --git a/src/linker/lnk.c b/src/linker/lnk.c index e43ef673..0da81a8f 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -63,11 +63,13 @@ #include "base_ext/base_inc.h" #include "thread_pool/thread_pool.h" +#include "base_ext/base_radix_sort.h" #include "codeview_ext/codeview.h" #include "pdb_ext/msf_builder.h" #include "base_ext/base_inc.c" #include "thread_pool/thread_pool.c" +#include "base_ext/base_radix_sort.c" #include "codeview_ext/codeview.c" // --- PDB Extensions- --------------------------------------------------------- diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 23e22e11..213c8f76 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -1193,6 +1193,26 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, return input; } +internal force_inline LNK_LeafRef +lnk_leaf_ref_make(U64 obj_idx, U64 leaf_idx) +{ + LNK_LeafRef result = Compose64Bit(safe_cast_u32(obj_idx), safe_cast_u32(leaf_idx)); + Assert(result != LNK_LEAF_REF_NULL); + return result; +} + +internal force_inline U32 +lnk_leaf_ref_obj_idx(LNK_LeafRef ref) +{ + return (U32)(ref >> 32); +} + +internal force_inline U32 +lnk_leaf_ref_leaf_idx(LNK_LeafRef ref) +{ + return (U32)ref; +} + internal LNK_LeafRef lnk_leaf_ref_from_ti(LNK_CodeViewInput *input, U32 obj_idx, CV_TypeIndexSource source, CV_TypeIndex ti) { @@ -1221,45 +1241,34 @@ lnk_leaf_ref_from_ti(LNK_CodeViewInput *input, U32 obj_idx, CV_TypeIndexSource s // map original leaf index to RRT type index CV_TypeIndex final_ti = obj->ti_map[leaf_idx_og]; - return (LNK_LeafRef){ ts_debug_t_idx, cv_leaf_idx_from_ti(ts_debug_t, source, final_ti) }; + return lnk_leaf_ref_make(ts_debug_t_idx, cv_leaf_idx_from_ti(ts_debug_t, source, final_ti)); } - return (LNK_LeafRef){ ts_debug_t_idx, cv_leaf_idx_from_ti(ts_debug_t, source, ti) }; + return lnk_leaf_ref_make(ts_debug_t_idx, cv_leaf_idx_from_ti(ts_debug_t, source, ti)); } CV_DebugT *debug_t = input->debug_t_arr + obj_idx; // ti_range: PCH if (contains_1u64(debug_t->pch_ti_range[source], ti)) { - return (LNK_LeafRef){ debug_t->pch_obj_idx, - cv_leaf_idx_from_ti(&input->debug_t_arr[debug_t->pch_obj_idx], source, ti) }; + return lnk_leaf_ref_make(debug_t->pch_obj_idx, + cv_leaf_idx_from_ti(&input->debug_t_arr[debug_t->pch_obj_idx], source, ti)); } // ti range: internal type server - return (LNK_LeafRef){ obj_idx, cv_leaf_idx_from_ti(debug_t, source, ti) }; + return lnk_leaf_ref_make(obj_idx, cv_leaf_idx_from_ti(debug_t, source, ti)); } internal U64 lnk_hash_from_leaf_ref(LNK_CodeViewInput *input, LNK_LeafRef leaf_ref) { - return input->debug_h_arr[leaf_ref.obj_idx].v[leaf_ref.leaf_idx]; + return input->debug_h_arr[lnk_leaf_ref_obj_idx(leaf_ref)].v[lnk_leaf_ref_leaf_idx(leaf_ref)]; } internal int lnk_leaf_ref_compare(LNK_LeafRef a, LNK_LeafRef b) { - if (a.obj_idx == b.obj_idx) { - return a.leaf_idx < b.leaf_idx ? -1 : - a.leaf_idx > b.leaf_idx ? +1 : 0; - } - return a.obj_idx < b.obj_idx ? -1 : - a.obj_idx > b.obj_idx ? +1 : 0; -} - -internal int -lnk_leaf_ref_is_before(void *raw_a, void *raw_b) -{ - return lnk_leaf_ref_compare(**(LNK_LeafRef **)raw_a, **(LNK_LeafRef **)raw_b) < 0; + return a < b ? -1 : a > b ? +1 : 0; } internal B32 @@ -1273,10 +1282,12 @@ lnk_match_leaf_ref(LNK_CodeViewInput *input, LNK_LeafRef a, LNK_LeafRef b) internal U64 lnk_hash_cv_leaf(LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInfoList ti_info_list, B32 discard_cycles) { - CV_DebugT *debug_t = &input->debug_t_arr[leaf_ref.obj_idx]; - CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_ref.leaf_idx); + U32 obj_idx = lnk_leaf_ref_obj_idx(leaf_ref); + U32 leaf_idx = lnk_leaf_ref_leaf_idx(leaf_ref); + CV_DebugT *debug_t = &input->debug_t_arr[obj_idx]; + CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); CV_TypeIndexSource curr_ti_source = cv_type_index_source_from_leaf_kind(leaf.kind); - CV_TypeIndex curr_ti = cv_ti_from_leaf_idx(debug_t, curr_ti_source, leaf_ref.leaf_idx); + CV_TypeIndex curr_ti = cv_ti_from_leaf_idx(debug_t, curr_ti_source, leaf_idx); // init hasher LNK_Hasher hasher; @@ -1323,7 +1334,7 @@ lnk_hash_cv_leaf(LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInf Temp scratch = scratch_begin(0,0); String8 leaf_kind_str = cv_string_from_leaf_kind(leaf.kind); String8 error_msg = push_str8f(scratch.arena, "LF_%S(type_index: 0x%x) out of bounds type index 0x%x (leaf struct offset: 0x%llx)", leaf_kind_str, curr_ti, sub_ti, sub_ti_n->offset); - lnk_error_obj(LNK_Error_InvalidTypeIndex, input->obj_arr[leaf_ref.obj_idx], "%S", error_msg); + lnk_error_obj(LNK_Error_InvalidTypeIndex, input->obj_arr[obj_idx], "%S", error_msg); scratch_end(scratch); break; @@ -1345,31 +1356,31 @@ lnk_hash_cv_leaf(LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInf Temp scratch = scratch_begin(0,0); String8 leaf_kind_str = cv_string_from_leaf_kind(leaf.kind); String8 error_msg = push_str8f(scratch.arena, "LF_%S(type_index: 0x%x) forward refs member type index 0x%x (leaf struct offset: 0x%llx)", leaf_kind_str, curr_ti, sub_ti, sub_ti_n->offset); - lnk_error_obj(LNK_Error_InvalidTypeIndex, input->obj_arr[leaf_ref.obj_idx], "%S", error_msg); + lnk_error_obj(LNK_Error_InvalidTypeIndex, input->obj_arr[obj_idx], "%S", error_msg); scratch_end(scratch); break; } // type index -> hash - LNK_LeafRef sub_ref = lnk_leaf_ref_from_ti(input, leaf_ref.obj_idx, sub_ti_n->source, sub_ti); - U64 sub_hash = input->debug_h_arr[sub_ref.obj_idx].v[sub_ref.leaf_idx]; + LNK_LeafRef sub_ref = lnk_leaf_ref_from_ti(input, obj_idx, sub_ti_n->source, sub_ti); + U64 sub_hash = input->debug_h_arr[lnk_leaf_ref_obj_idx(sub_ref)].v[lnk_leaf_ref_leaf_idx(sub_ref)]; // mix-in sub-type hash lnk_hasher_update_struct(&hasher, &sub_hash); } // hash leaf header - CV_LeafHeader *leaf_header_ptr = cv_debug_t_get_leaf_header(debug_t, leaf_ref.leaf_idx); + CV_LeafHeader *leaf_header_ptr = cv_debug_t_get_leaf_header(debug_t, leaf_idx); lnk_hasher_update_struct(&hasher, leaf_header_ptr); // finalize the type hash U64 hash = lnk_hasher_digest64(&hasher); Assert(hash != 0); - Assert(input->debug_h_arr[leaf_ref.obj_idx].v[leaf_ref.leaf_idx] == 0 || - input->debug_h_arr[leaf_ref.obj_idx].v[leaf_ref.leaf_idx] == 1); - input->debug_h_arr[leaf_ref.obj_idx].v[leaf_ref.leaf_idx] = hash; + Assert(input->debug_h_arr[obj_idx].v[leaf_idx] == 0 || + input->debug_h_arr[obj_idx].v[leaf_idx] == 1); + input->debug_h_arr[obj_idx].v[leaf_idx] = hash; return hash; } @@ -1393,14 +1404,15 @@ lnk_hash_cv_leaf_deep(Arena *arena, } HashStack; // set up root frame - CV_DebugT *root_debug_t = &input->debug_t_arr[root_leaf_ref.obj_idx]; + U32 root_obj_idx = lnk_leaf_ref_obj_idx(root_leaf_ref); + CV_DebugT *root_debug_t = &input->debug_t_arr[root_obj_idx]; HashStack *root_frame = push_array(temp.arena, HashStack, 1); root_frame->leaf_ref = root_leaf_ref; root_frame->ti_info_list = root_ti_info_list; root_frame->ti_info = root_ti_info_list.first; - root_frame->leaf = cv_debug_t_get_leaf(root_debug_t, root_leaf_ref.leaf_idx); + root_frame->leaf = cv_debug_t_get_leaf(root_debug_t, lnk_leaf_ref_leaf_idx(root_leaf_ref)); root_frame->ti_source = cv_type_index_source_from_leaf_kind(root_frame->leaf.kind); - root_frame->ti = cv_ti_from_leaf_idx(root_debug_t, root_frame->ti_source, root_leaf_ref.leaf_idx); + root_frame->ti = cv_ti_from_leaf_idx(root_debug_t, root_frame->ti_source, lnk_leaf_ref_leaf_idx(root_leaf_ref)); HashStack *stack = root_frame; while (stack) { @@ -1415,17 +1427,19 @@ lnk_hash_cv_leaf_deep(Arena *arena, CV_TypeIndex ti = memory_read32(ti_ptr); // skip out of bounds indices - if ( ! contains_1u64(input->debug_t_arr[root_leaf_ref.obj_idx].ti_ranges[ti_info->source], ti)) { continue; } + if ( ! contains_1u64(input->debug_t_arr[root_obj_idx].ti_ranges[ti_info->source], ti)) { continue; } // skip hashed types - LNK_LeafRef leaf_ref = lnk_leaf_ref_from_ti(input, root_leaf_ref.obj_idx, ti_info->source, ti); - if (input->debug_h_arr[leaf_ref.obj_idx].v[leaf_ref.leaf_idx] != 0) { continue; } - input->debug_h_arr[leaf_ref.obj_idx].v[leaf_ref.leaf_idx] = 1; + LNK_LeafRef leaf_ref = lnk_leaf_ref_from_ti(input, root_obj_idx, ti_info->source, ti); + U32 obj_idx = lnk_leaf_ref_obj_idx(leaf_ref); + U32 leaf_idx = lnk_leaf_ref_leaf_idx(leaf_ref); + if (input->debug_h_arr[obj_idx].v[leaf_idx] != 0) { continue; } + input->debug_h_arr[obj_idx].v[leaf_idx] = 1; // recurse down to sub types HashStack *frame = push_array(temp.arena, HashStack, 1); frame->leaf_ref = leaf_ref; - frame->leaf = cv_debug_t_get_leaf(&input->debug_t_arr[leaf_ref.obj_idx], leaf_ref.leaf_idx); + frame->leaf = cv_debug_t_get_leaf(&input->debug_t_arr[obj_idx], leaf_idx); frame->ti_info_list = cv_get_leaf_type_index_offsets(temp.arena, frame->leaf.kind, frame->leaf.data); frame->ti_info = frame->ti_info_list.first; frame->ti = ti; @@ -1447,8 +1461,8 @@ lnk_hash_cv_leaf_deep(Arena *arena, internal CV_TypeIndex lnk_assigned_ti_hash_search(LNK_AssignedTiHash *ht, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref) { - CV_DebugH *debug_h = &input->debug_h_arr[leaf_ref.obj_idx]; - U64 hash = debug_h->v[leaf_ref.leaf_idx]; + CV_DebugH *debug_h = &input->debug_h_arr[lnk_leaf_ref_obj_idx(leaf_ref)]; + U64 hash = debug_h->v[lnk_leaf_ref_leaf_idx(leaf_ref)]; U64 best_idx = hash % ht->cap; U64 idx = best_idx; do { @@ -1472,7 +1486,7 @@ THREAD_POOL_TASK_FUNC(lnk_hash_debug_t_task) Temp temp = temp_begin(task->fixed_arenas[worker_id]); CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); CV_TypeIndexInfoList ti_list = cv_get_leaf_type_index_offsets(temp.arena, leaf.kind, leaf.data); - lnk_hash_cv_leaf(task->input, (LNK_LeafRef){ obj_idx, leaf_idx }, ti_list, 1); + lnk_hash_cv_leaf(task->input, lnk_leaf_ref_make(obj_idx, leaf_idx), ti_list, 1); temp_end(temp); } ProfEnd(); @@ -1490,7 +1504,7 @@ THREAD_POOL_TASK_FUNC(lnk_hash_debug_t_deep_task) Temp temp = temp_begin(task->fixed_arenas[worker_id]); CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); CV_TypeIndexInfoList ti_list = cv_get_leaf_type_index_offsets(temp.arena, leaf.kind, leaf.data); - lnk_hash_cv_leaf_deep(temp.arena, task->input, (LNK_LeafRef){ obj_idx, leaf_idx }, ti_list); + lnk_hash_cv_leaf_deep(temp.arena, task->input, lnk_leaf_ref_make(obj_idx, leaf_idx), ti_list); temp_end(temp); } ProfEnd(); @@ -1506,49 +1520,29 @@ THREAD_POOL_TASK_FUNC(lnk_populate_leaf_ht) CV_DebugH *debug_h = &task->input->debug_h_arr[task->pop_obj_idx]; for EachInRange(leaf_idx, task->pop_range[task_id]) { - LNK_LeafRef *bucket = 0; - - // alloc new bucket and assign type ref - if (bucket == 0) { bucket = push_array_no_zero(arena, LNK_LeafRef, 1); } - - // fill bucket - *bucket = (LNK_LeafRef){ .obj_idx = obj_idx, .leaf_idx = leaf_idx }; - - B32 is_inserted_or_updated = 1; - + + LNK_LeafRef leaf_ref = lnk_leaf_ref_make(obj_idx, leaf_idx); CV_LeafHeader *header = cv_debug_t_get_leaf_header(debug_t, leaf_idx); // leaf index -> leaf header CV_LeafKind kind = memory_read16(MemberFromPtr(CV_LeafHeader, header, kind)); // leaf header -> leaf kind CV_TypeIndexSource leaf_source = cv_type_index_source_from_leaf_kind(kind); // leaf kind -> type stream LNK_LeafHashTable *leaf_ht = &task->leaf_ht_arr[leaf_source]; // type stream -> hash table U64 best_idx = debug_h->v[leaf_idx] % leaf_ht->cap; // leaf ref -> hash -> bucket index U64 idx = best_idx; + do { - // load leaf ref - LNK_LeafRef *curr = ins_atomic_ptr_eval(&leaf_ht->bucket_arr[idx]); - - while (curr == 0) { - // exit if leaf ref is not recent - if (curr != 0 && lnk_leaf_ref_compare(*bucket, *curr) >= 0) { - goto exit; - } - - // try to update the bucket - LNK_LeafRef *cmp = ins_atomic_ptr_eval_cond_assign(&leaf_ht->bucket_arr[idx], bucket, curr); + LNK_LeafRef curr = ins_atomic_u64_eval(&leaf_ht->bucket_arr[idx]); + if (curr == LNK_LEAF_REF_NULL) { + LNK_LeafRef cmp = ins_atomic_u64_eval_cond_assign(&leaf_ht->bucket_arr[idx], leaf_ref, curr); if (cmp == curr) { - bucket = 0; goto exit; } - - // another thread updated the bucket -- retry - curr = cmp; } // advance to next bucket idx = ((idx + 1) == leaf_ht->cap ? 0 : (idx + 1)); } while (idx != best_idx); - is_inserted_or_updated = 0; + InvalidPath; exit:; - Assert(is_inserted_or_updated); } } @@ -1559,38 +1553,34 @@ THREAD_POOL_TASK_FUNC(lnk_leaf_dedup_task) U64 obj_idx = task->indices.v[task_id]; CV_DebugT *debug_t = &task->input->debug_t_arr[obj_idx]; CV_DebugH *debug_h = &task->input->debug_h_arr[obj_idx]; + ProfBeginDynamic("dedup in obj 0x%llx (%.*s) leaf count %llu", obj_idx, str8_varg(task->input->obj_arr[obj_idx]->path), debug_t->count); - LNK_LeafRef *bucket = 0; for EachIndex(leaf_idx, debug_t->count) { - // alloc new bucket and assign type ref - if (bucket == 0) { bucket = push_array_no_zero(arena, LNK_LeafRef, 1); } - - // fill bucket - *bucket = (LNK_LeafRef){ .obj_idx = obj_idx, .leaf_idx = leaf_idx }; B32 is_inserted_or_updated = 1; + LNK_LeafRef leaf_ref = lnk_leaf_ref_make(obj_idx, leaf_idx); CV_LeafHeader *header = cv_debug_t_get_leaf_header(debug_t, leaf_idx); // leaf index -> leaf header CV_LeafKind kind = memory_read16(MemberFromPtr(CV_LeafHeader, header, kind)); // leaf header -> leaf kind CV_TypeIndexSource leaf_source = cv_type_index_source_from_leaf_kind(kind); // leaf kind -> type stream LNK_LeafHashTable *leaf_ht = &task->leaf_ht_arr[leaf_source]; // type stream -> hash table U64 best_idx = debug_h->v[leaf_idx] % leaf_ht->cap; // leaf ref -> hash -> bucket index U64 idx = best_idx; + do { // load leaf ref - LNK_LeafRef *curr = ins_atomic_ptr_eval(&leaf_ht->bucket_arr[idx]); + LNK_LeafRef curr = ins_atomic_u64_eval(&leaf_ht->bucket_arr[idx]); - while (curr == 0 || lnk_match_leaf_ref(task->input, *curr, *bucket)) { + while (curr == LNK_LEAF_REF_NULL || lnk_match_leaf_ref(task->input, curr, leaf_ref)) { // exit if leaf ref is not recent - if (curr != 0 && lnk_leaf_ref_compare(*bucket, *curr) >= 0) { + if (curr != LNK_LEAF_REF_NULL && lnk_leaf_ref_compare(leaf_ref, curr) >= 0) { goto exit; } // try to update the bucket - LNK_LeafRef *cmp = ins_atomic_ptr_eval_cond_assign(&leaf_ht->bucket_arr[idx], bucket, curr); + LNK_LeafRef cmp = ins_atomic_u64_eval_cond_assign(&leaf_ht->bucket_arr[idx], leaf_ref, curr); if (cmp == curr) { - bucket = 0; goto exit; } @@ -1601,6 +1591,7 @@ THREAD_POOL_TASK_FUNC(lnk_leaf_dedup_task) // advance to next bucket idx = ((idx + 1) == leaf_ht->cap ? 0 : (idx + 1)); } while (idx != best_idx); + is_inserted_or_updated = 0; exit:; Assert(is_inserted_or_updated); @@ -1618,7 +1609,7 @@ THREAD_POOL_TASK_FUNC(lnk_count_present_buckets_task) LNK_LeafHashTable *ht = &task->leaf_ht_arr[task->ti_source]; for EachInRange(bucket_idx, task->ranges[task_id]) { - if (ht->bucket_arr[bucket_idx] != 0) { + if (ht->bucket_arr[bucket_idx] != LNK_LEAF_REF_NULL) { task->counts[task->ti_source][task_id] += 1; } } @@ -1638,7 +1629,7 @@ THREAD_POOL_TASK_FUNC(lnk_get_present_buckets_task) LNK_LeafRefArray unique_leaf_refs = task->unique_leaf_refs_arr[task->ti_source]; for EachInRange(bucket_idx, task->ranges[task_id]) { - if (ht->bucket_arr[bucket_idx]) { + if (ht->bucket_arr[bucket_idx] != LNK_LEAF_REF_NULL) { unique_leaf_refs.v[cursor++] = ht->bucket_arr[bucket_idx]; } } @@ -1646,162 +1637,6 @@ THREAD_POOL_TASK_FUNC(lnk_get_present_buckets_task) ProfEnd(); } -internal -THREAD_POOL_TASK_FUNC(lnk_leaf_ref_histo_task) -{ - ProfBeginFunction(); - - LNK_MergeTypes *task = raw_task; - Rng1U64 range = task->ranges[task_id]; - U32 *counts_ptr = task->counts_arr[task_id]; - - U32 obj_idx_bit_count_0 = task->obj_idx_bit_count_0; - U32 obj_idx_bit_count_1 = task->obj_idx_bit_count_1; - U32 obj_idx_bit_count_2 = task->obj_idx_bit_count_2; - - MemoryZeroTyped(task->counts_arr[task_id], task->counts_max); - - switch (task->pass_idx) { - case 0: { - for EachInRange(i, range) { - LNK_LeafRef *bucket = task->src[i]; - U64 leaf_digit0 = BitExtract(bucket->leaf_idx, 10, 0); - counts_ptr[leaf_digit0] += 1; - } - } break; - case 1: { - for EachInRange(i, range) { - LNK_LeafRef *bucket = task->src[i]; - U64 leaf_digit1 = BitExtract(bucket->leaf_idx, 11, 10); - counts_ptr[leaf_digit1] += 1; - } - } break; - case 2: { - for EachInRange(i, range) { - LNK_LeafRef *bucket = task->src[i]; - U64 leaf_digit2 = BitExtract(bucket->leaf_idx, 11, 21); - counts_ptr[leaf_digit2] += 1; - } - } break; - - case 3: { - for EachInRange(i, range) { - LNK_LeafRef *bucket = task->src[i]; - U64 digit0 = BitExtract(bucket->obj_idx, obj_idx_bit_count_0, 0); - counts_ptr[digit0] += 1; - } - } break; - case 4: { - for EachInRange(i, range) { - LNK_LeafRef *bucket = task->src[i]; - U64 digit1 = BitExtract(bucket->obj_idx, obj_idx_bit_count_1, obj_idx_bit_count_0); - counts_ptr[digit1] += 1; - } - } break; - case 5: { - for EachInRange(i, range) { - LNK_LeafRef *bucket = task->src[i]; - U64 digit2 = BitExtract(bucket->obj_idx, obj_idx_bit_count_2, obj_idx_bit_count_0 + obj_idx_bit_count_1); - counts_ptr[digit2] += 1; - } - } break; - default: InvalidPath; - } - - ProfEnd(); -} - -internal -THREAD_POOL_TASK_FUNC(lnk_loc_idx_radix_sort_task) -{ - ProfBeginFunction(); - - LNK_MergeTypes *task = raw_task; - Rng1U64 range = task->ranges[task_id]; - U32 *counts_ptr = task->counts_arr[task_id]; - U32 obj_idx_bit_count_0 = task->obj_idx_bit_count_0; - U32 obj_idx_bit_count_1 = task->obj_idx_bit_count_1; - U32 obj_idx_bit_count_2 = task->obj_idx_bit_count_2; - - switch (task->pass_idx) { - // - // Sort items on leaf index - // - case 0: { - ProfBegin("Leaf Sort Low"); - for EachInRange(i, range) { - LNK_LeafRef *bucket = task->src[i]; - U64 leaf_digit0 = BitExtract(bucket->leaf_idx, 10, 0); - task->dst[counts_ptr[leaf_digit0]++] = bucket; - } - ProfEnd(); - } break; - case 1: { - ProfBegin("Leaf Sort Mid"); - for EachInRange(i, range) { - LNK_LeafRef *bucket = task->src[i]; - U64 leaf_digit1 = BitExtract(bucket->leaf_idx, 11, 10); - task->dst[counts_ptr[leaf_digit1]++] = bucket; - } - ProfEnd(); - } break; - case 2: { - ProfBegin("Leaf Sort High"); - for EachInRange(i, range) { - LNK_LeafRef *bucket = task->src[i]; - U64 leaf_digit2 = BitExtract(bucket->leaf_idx, 11, 21); - task->dst[counts_ptr[leaf_digit2]++] = bucket; - } - ProfEnd(); - } break; - - // - // Sort items on obj and type server index - // - case 3: { - ProfBegin("Obj Sort Low"); - for EachInRange(i, range) { - LNK_LeafRef *bucket = task->src[i]; - U64 digit0 = BitExtract(bucket->obj_idx, obj_idx_bit_count_0, 0); - task->dst[counts_ptr[digit0]++] = bucket; - } - ProfEnd(); - } break; - case 4: { - ProfBegin("Obj Sort Mid"); - for EachInRange(i, range) { - LNK_LeafRef *bucket = task->src[i]; - U64 digit1 = BitExtract(bucket->obj_idx, obj_idx_bit_count_1, obj_idx_bit_count_0); - task->dst[counts_ptr[digit1]++] = bucket; - } - ProfEnd(); - } break; - case 5: { - ProfBegin("Obj Sort High"); - for EachInRange(i, range) { - LNK_LeafRef *bucket = task->src[i]; - U64 digit2 = BitExtract(bucket->obj_idx, obj_idx_bit_count_2, obj_idx_bit_count_0 + obj_idx_bit_count_1); - Assert(counts_ptr[digit2] != max_U32); - task->dst[counts_ptr[digit2]++] = bucket; - } - ProfEnd(); - } break; - - default: InvalidPath; - } - - ProfEnd(); -} - -internal void -lnk_leaf_ref_array_sort(TP_Context *tp, LNK_CodeViewInput *input, LNK_LeafRefArray arr, U64 debug_t_count) -{ - Temp scratch = scratch_begin(0,0); - ProfBeginDynamic("Leaf Sort [Leaf Count: %.*s]", str8_varg(str8_from_count(scratch.arena, arr.count))); - scratch_end(scratch); - ProfEnd(); -} - internal THREAD_POOL_TASK_FUNC(lnk_assign_type_indices_task) { @@ -1814,10 +1649,10 @@ THREAD_POOL_TASK_FUNC(lnk_assign_type_indices_task) CV_DebugH *debug_h_arr = task->input->debug_h_arr; for EachInRange(i, task->ranges[task_id]) { - LNK_LeafRef *leaf_ref = unique_leaf_refs.v[i]; + LNK_LeafRef leaf_ref = unique_leaf_refs.v[i]; CV_TypeIndex type_index = min_type_index + i; - U64 hash = debug_h_arr[leaf_ref->obj_idx].v[leaf_ref->leaf_idx]; + U64 hash = debug_h_arr[lnk_leaf_ref_obj_idx(leaf_ref)].v[lnk_leaf_ref_leaf_idx(leaf_ref)]; U64 best_idx = hash % assigned->cap; U64 idx = best_idx; @@ -1910,11 +1745,12 @@ THREAD_POOL_TASK_FUNC(lnk_cv_patcher_leaves_task) Arena *fixed_arena = task->fixed_arenas[task_id]; for EachInRange(leaf_ref_idx, range) { Temp temp = temp_begin(fixed_arena); - LNK_LeafRef *patch = task->unique_leaf_refs_arr[task->ti_source].v[leaf_ref_idx]; - CV_DebugT *debug_t = &task->input->debug_t_arr[patch->obj_idx]; - CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, patch->leaf_idx); + LNK_LeafRef patch = task->unique_leaf_refs_arr[task->ti_source].v[leaf_ref_idx]; + U32 obj_idx = lnk_leaf_ref_obj_idx(patch); + CV_DebugT *debug_t = &task->input->debug_t_arr[obj_idx]; + CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, lnk_leaf_ref_leaf_idx(patch)); CV_TypeIndexInfoList ti_info_list = cv_get_leaf_type_index_offsets(temp.arena, leaf.kind, leaf.data); - lnk_fixup_cv_type_indices(task, patch->obj_idx, leaf.data, ti_info_list); + lnk_fixup_cv_type_indices(task, obj_idx, leaf.data, ti_info_list); temp_end(temp); } ProfEnd(); @@ -1926,9 +1762,9 @@ THREAD_POOL_TASK_FUNC(lnk_unbucket_raw_leaves_task) LNK_MergeTypes *task = raw_task; Rng1U64 range = task->ranges[task_id]; for EachInRange(i, range) { - LNK_LeafRef leaf_ref = *task->unique_leaf_refs_arr[task->ti_source].v[i]; - CV_DebugT *debug_t = &task->input->debug_t_arr[leaf_ref.obj_idx]; - String8 raw_leaf = cv_debug_t_get_raw_leaf(debug_t, leaf_ref.leaf_idx); + LNK_LeafRef leaf_ref = task->unique_leaf_refs_arr[task->ti_source].v[i]; + CV_DebugT *debug_t = &task->input->debug_t_arr[lnk_leaf_ref_obj_idx(leaf_ref)]; + String8 raw_leaf = cv_debug_t_get_raw_leaf(debug_t, lnk_leaf_ref_leaf_idx(leaf_ref)); task->result.v[task->ti_source][i] = raw_leaf.str; } } @@ -1939,10 +1775,12 @@ THREAD_POOL_TASK_FUNC(lnk_unbucket_hashes_task) LNK_MergeTypes *task = raw_task; Rng1U64 range = task->ranges[task_id]; for EachInRange(i, range) { - LNK_LeafRef leaf_ref = *task->unique_leaf_refs_arr[task->ti_source].v[i]; - CV_DebugT *debug_t = &task->input->debug_t_arr[leaf_ref.obj_idx]; - String8 raw_leaf = cv_debug_t_get_raw_leaf(debug_t, leaf_ref.leaf_idx); - task->result.hashes[task->ti_source][i] = task->input->debug_h_arr[leaf_ref.obj_idx].v[leaf_ref.leaf_idx]; + LNK_LeafRef leaf_ref = task->unique_leaf_refs_arr[task->ti_source].v[i]; + U32 obj_idx = lnk_leaf_ref_obj_idx(leaf_ref); + U32 leaf_idx = lnk_leaf_ref_leaf_idx(leaf_ref); + CV_DebugT *debug_t = &task->input->debug_t_arr[obj_idx]; + String8 raw_leaf = cv_debug_t_get_raw_leaf(debug_t, leaf_idx); + task->result.hashes[task->ti_source][i] = task->input->debug_h_arr[obj_idx].v[leaf_idx]; } } @@ -2021,7 +1859,7 @@ THREAD_POOL_TASK_FUNC(lnk_build_obj_ti_map) for EachIndex(leaf_idx, debug_t->count) { CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); CV_TypeIndexSource source = cv_type_index_source_from_leaf_kind(leaf.kind); - LNK_LeafRef leaf_ref = { obj_idx, leaf_idx }; + LNK_LeafRef leaf_ref = lnk_leaf_ref_make(obj_idx, leaf_idx); LNK_AssignedTiHash *assigned = &task->assigned_ti_arr[source]; obj_ti_map[leaf_idx] = lnk_assigned_ti_hash_search(assigned, input, leaf_ref); } @@ -2110,7 +1948,8 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK task.leaf_ht_arr[ti_source].cap = total_count; task.leaf_ht_arr[ti_source].cap = 1 + ((task.leaf_ht_arr[ti_source].cap * 13) / 10); // * 1.3 - task.leaf_ht_arr[ti_source].bucket_arr = push_array(scratch.arena, LNK_LeafRef *, task.leaf_ht_arr[ti_source].cap); + task.leaf_ht_arr[ti_source].bucket_arr = push_array_no_zero(scratch.arena, LNK_LeafRef, task.leaf_ht_arr[ti_source].cap); + MemorySet(task.leaf_ht_arr[ti_source].bucket_arr, 0xff, sizeof(LNK_LeafRef) * task.leaf_ht_arr[ti_source].cap); #if PROFILE_TELEMETRY tmMessage(0, TMMF_ICON_NOTE, "%.*s Bucket Count: %.*s", str8_varg(cv_string_from_type_index_source(ti_source)), str8_varg(str8_from_count(scratch.arena, task.leaf_ht_arr[ti_source].cap))); @@ -2166,90 +2005,13 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK tp_for_parallel_prof(tp, 0, tp->worker_count, lnk_count_present_buckets_task, &task, "Count present buckets"); task.unique_leaf_refs_arr[ti_source].count = sum_array_u64(tp->worker_count, task.counts[ti_source]); - task.unique_leaf_refs_arr[ti_source].v = push_array_no_zero(scratch.arena, LNK_LeafRef *, task.unique_leaf_refs_arr[ti_source].count); + task.unique_leaf_refs_arr[ti_source].v = push_array_no_zero(scratch.arena, LNK_LeafRef, task.unique_leaf_refs_arr[ti_source].count); task.offsets[ti_source] = offsets_from_counts_array_u64(scratch.arena, task.counts[ti_source], tp->worker_count); tp_for_parallel_prof(tp, 0, tp->worker_count, lnk_get_present_buckets_task, &task, "Copy present buckets"); // sort output leaves based on { location index, leaf index } to guarantee determinism - { - LNK_LeafRefArray arr = task.unique_leaf_refs_arr[ti_source]; - if (arr.count > 140000) { - ProfBegin("Radix"); - - U32 obj_idx_max_bits = 32 - clz32(input->count); - - task.obj_idx_bit_count_0 = Clamp(0, (S32)obj_idx_max_bits - 21, 11); - task.obj_idx_bit_count_1 = Clamp(0, (S32)obj_idx_max_bits - 10, 11); - task.obj_idx_bit_count_2 = Clamp(0, (S32)obj_idx_max_bits, 10); - task.counts_max = (1 << 11); - task.ranges = tp_divide_work(scratch.arena, arr.count, tp->worker_count); - task.dst = push_array_no_zero(scratch.arena, LNK_LeafRef *, arr.count); - task.src = arr.v; - - ProfBegin("Push Counts"); - task.counts_arr = push_array_no_zero(scratch.arena, U32 *, tp->worker_count); - for (U64 i = 0; i < tp->worker_count; ++i) { - // zero-out happens in histogram step - task.counts_arr[i] = push_array_no_zero(scratch.arena, U32, task.counts_max); - } - ProfEnd(); - - for (task.pass_idx = 0; task.pass_idx < 6; ++task.pass_idx) { - ProfBeginDynamic("Pass: %u", task.pass_idx); - - ProfBegin("Histo"); - tp_for_parallel(tp, 0, tp->worker_count, lnk_leaf_ref_histo_task, &task); - ProfEnd(); - - B32 is_range_not_empty = 0; - for (U64 task_id = 0; task_id < tp->worker_count; ++task_id) { - is_range_not_empty = task.counts_arr[task_id][0] != dim_1u64(task.ranges[task_id]); - if (is_range_not_empty) { - break; - } - } - - ProfBegin("Counts -> Offsets"); - { - U64 digit_cursor = 0; - for EachIndex(digit_idx, task.counts_max) { - for EachIndex(task_id, tp->worker_count) { - U64 count = task.counts_arr[task_id][digit_idx]; - task.counts_arr[task_id][digit_idx] = digit_cursor; - digit_cursor += count; - } - } - Assert(digit_cursor == arr.count); - } - ProfEnd(); - - ProfBegin("Sort"); - tp_for_parallel(tp, 0, tp->worker_count, lnk_loc_idx_radix_sort_task, &task); - Swap(LNK_LeafRef **, task.src, task.dst); - ProfEnd(); - - ProfEnd(); - } - - if (task.src != arr.v) { - MemoryCopyTyped(arr.v, task.dst, arr.count); - } - - ProfEnd(); - } else { - ProfBegin("Radsort"); - radsort(arr.v, arr.count, lnk_leaf_ref_is_before); - ProfEnd(); - } - -#if 0 - for (U64 i = 1; i < arr.count; ++i) { - AssertAlways(arr.v[i-1]->obj_idx <= arr.v[i]->obj_idx); - if (arr.v[i-1]->obj_idx == arr.v[i]->obj_idx) { - AssertAlways(arr.v[i-1]->obj_idx <= arr.v[i]->obj_idx); - } - } -#endif + ProfScope("Radix Sort") { + u64_array_sort_radix_parallel(tp, task.unique_leaf_refs_arr[ti_source].count, task.unique_leaf_refs_arr[ti_source].v); } } @@ -2271,7 +2033,7 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK // unique extraction is complete, so the dedup bucket slots can back the // direct hash table without increasing peak memory Assert(task.assigned_ti_arr[ti_source].cap <= task.leaf_ht_arr[ti_source].cap); - task.assigned_ti_arr[ti_source].hash_arr = (U64 *)task.leaf_ht_arr[ti_source].bucket_arr; + task.assigned_ti_arr[ti_source].hash_arr = task.leaf_ht_arr[ti_source].bucket_arr; task.min_type_indices[ti_source] = CV_MinComplexTypeIndex; task.ranges = tp_divide_work(scratch.arena, task.unique_leaf_refs_arr[ti_source].count, tp->worker_count); diff --git a/src/linker/lnk_debug_info.h b/src/linker/lnk_debug_info.h index 29412a5b..573fc3b5 100644 --- a/src/linker/lnk_debug_info.h +++ b/src/linker/lnk_debug_info.h @@ -120,13 +120,15 @@ typedef struct //////////////////////////////// // Type Merging -typedef struct { U32 obj_idx; U32 leaf_idx; } LNK_LeafRef; -typedef struct { U64 count; LNK_LeafRef **v; } LNK_LeafRefArray; +typedef U64 LNK_LeafRef; +typedef struct { U64 count; LNK_LeafRef *v; } LNK_LeafRefArray; + +#define LNK_LEAF_REF_NULL max_U64 typedef struct { - U64 cap; - LNK_LeafRef **bucket_arr; + U64 cap; + LNK_LeafRef *bucket_arr; } LNK_LeafHashTable; typedef struct @@ -181,16 +183,6 @@ typedef struct U64 *counts [CV_TypeIndexSource_COUNT]; U64 *offsets[CV_TypeIndexSource_COUNT]; - // leaf ref radix sort - U64 obj_idx_bit_count_0; - U64 obj_idx_bit_count_1; - U64 obj_idx_bit_count_2; - U64 counts_max; - U32 **counts_arr; - LNK_LeafRef **dst; - LNK_LeafRef **src; - U64 pass_idx; - CV_TypeIndex min_type_indices [CV_TypeIndexSource_COUNT]; LNK_LeafRefArray unique_leaf_refs_arr[CV_TypeIndexSource_COUNT]; @@ -270,11 +262,9 @@ internal B32 lnk_rrt_from_string (Arena *arena, String8 rrt_data, St internal LNK_CodeViewInput lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, U64 objs_count, LNK_Obj **objs, LNK_RRT_Array rrt_input); internal int lnk_leaf_ref_compare (LNK_LeafRef a, LNK_LeafRef b); -internal int lnk_leaf_ref_is_before (void *raw_a, void *raw_b); internal B32 lnk_match_leaf_ref (LNK_CodeViewInput *input, LNK_LeafRef a, LNK_LeafRef b); internal U64 lnk_hash_cv_leaf (LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInfoList ti_info_list, B32 discard_cycles); internal void lnk_hash_cv_leaf_deep (Arena *arena, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInfoList ti_info_list); -internal LNK_LeafRef * lnk_leaf_hash_table_insert_or_update(LNK_LeafHashTable *leaf_ht, LNK_CodeViewInput *input, CV_DebugH *hashes, U64 hash, LNK_LeafRef *new_bucket); internal CV_TypeIndex lnk_assigned_ti_hash_search (LNK_AssignedTiHash *ht, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref); internal LNK_MergedTypes lnk_merge_types (TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK_MergeTypeFlags merge_flags); internal void lnk_replace_type_names_with_hashes (TP_Context *tp, TP_Arena *arena, U64 leaf_count, U8 **leaf_arr, LNK_TypeNameHashMode mode, U64 hash_length, String8 map_name); diff --git a/src/linker/tests/linker_tests.c b/src/linker/tests/linker_tests.c index 99b0b0ad..1c6cad10 100644 --- a/src/linker/tests/linker_tests.c +++ b/src/linker/tests/linker_tests.c @@ -7146,6 +7146,48 @@ TEST(psi_addr_map_radix_sort) tp_release(tp); } +TEST(u64_array_radix_sort_parallel) +{ + TP_Context *tp = tp_alloc(arena, 1, 1, str8_zero()); + + U64 small_values[] = { max_U64, 0, 7, 1, 7, 0x100000000ull, 2 }; + u64_array_sort_radix_parallel(tp, ArrayCount(small_values), small_values); + for (U64 i = 1; i < ArrayCount(small_values); i += 1) { + T_Ok(small_values[i-1] <= small_values[i]); + } + + U64 count = 200003; + U64 *values = push_array_no_zero(arena, U64, count); + U64 state = 0x9e3779b97f4a7c15ull; + U64 sum_before = 0; + U64 xor_before = 0; + for EachIndex(i, count) { + state ^= state >> 12; + state ^= state << 25; + state ^= state >> 27; + values[i] = state * 0x2545f4914f6cdd1dull; + if ((i % 97) == 0) { values[i] = 0; } + if ((i % 193) == 0) { values[i] = max_U64; } + if ((i % 389) == 0) { values[i] = 0x100000001ull; } + sum_before += values[i]; + xor_before ^= values[i]; + } + + u64_array_sort_radix_parallel(tp, count, values); + U64 sum_after = 0; + U64 xor_after = 0; + B32 is_sorted = 1; + for EachIndex(i, count) { + if (i > 0 && values[i-1] > values[i]) { is_sorted = 0; } + sum_after += values[i]; + xor_after ^= values[i]; + } + T_Ok(is_sorted); + T_Ok(sum_before == sum_after); + T_Ok(xor_before == xor_after); + tp_release(tp); +} + TEST(pdbstripped) { String8 debug_obj; diff --git a/src/torture/torture_main.c b/src/torture/torture_main.c index 9d9e58e8..1b3a6915 100644 --- a/src/torture/torture_main.c +++ b/src/torture/torture_main.c @@ -91,6 +91,7 @@ #include "linker/base_ext/base_arena.h" #include "linker/base_ext/base_arrays.h" #include "linker/thread_pool/thread_pool.h" +#include "linker/base_ext/base_radix_sort.h" #include "linker/codeview_ext/codeview.h" #include "linker/pdb_ext/msf_builder.h" #include "linker/pdb_ext/pdb.h" @@ -173,6 +174,7 @@ #include "linker/base_ext/base_arrays.c" #include "linker/base_ext/base_crc32.c" #include "linker/thread_pool/thread_pool.c" +#include "linker/base_ext/base_radix_sort.c" #include "linker/codeview_ext/codeview.c" #include "linker/pdb_ext/msf_builder.c" #include "linker/pdb_ext/pdb.c"