WIP PDB Writer Thread

This commit is contained in:
Nikita Smith
2026-07-27 14:47:26 -07:00
committed by Ryan Fleury
parent a73e3e0790
commit c96c9620d0
6 changed files with 343 additions and 21 deletions
+4 -5
View File
@@ -6424,10 +6424,9 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config)
config->pdb_hash_type_name_length,
config->pdb_hash_type_name_map);
}
pdb_data = lnk_build_pdb(tp, arena, image_ctx.image_data, config, symtab, &cv, cv_types, LNK_PDB_BuilderFlag_All);
if (config->debug_mode == LNK_DebugMode_Full) {
lnk_write_data_list_to_file_path(config->pdb_name, config->temp_pdb_name, pdb_data);
}
String8 pdb_output_path = config->debug_mode == LNK_DebugMode_Full ? config->pdb_name : str8_zero();
String8 pdb_temp_output_path = config->debug_mode == LNK_DebugMode_Full ? config->temp_pdb_name : str8_zero();
pdb_data = lnk_build_pdb(tp, arena, image_ctx.image_data, config, symtab, &cv, cv_types, pdb_output_path, pdb_temp_output_path, LNK_PDB_BuilderFlag_All);
lnk_timer_end(LNK_Timer_Pdb);
}
@@ -6511,7 +6510,7 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config)
stripped_cv.debug_s_arr = debug_s_arr;
stripped_cv.symbol_input_ranges = push_array(scratch.arena, Rng1U64, tp->worker_count);
String8List pdb_data = lnk_build_pdb(tp, arena, image_ctx.image_data, config, symtab, &stripped_cv, (LNK_MergedTypes){0}, LNK_PDB_BuilderFlag_All);
String8List pdb_data = lnk_build_pdb(tp, arena, image_ctx.image_data, config, symtab, &stripped_cv, (LNK_MergedTypes){0}, str8_zero(), str8_zero(), LNK_PDB_BuilderFlag_All);
lnk_write_data_list_to_file_path(config->pdb_stripped_name, str8f(scratch.arena, "%S.tmp", config->pdb_stripped_name), pdb_data);
}
+289 -7
View File
@@ -3042,7 +3042,7 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules)
String8 name = str8_cstring_capped(string_table.str + header.name_off, string_table.str + string_table.size);
CV_StringBucket *bucket = cv_string_hash_table_lookup(task->string_ht, name);
U64 name_off = task->pdb->info->strtab.size + bucket->u.offset;
U64 name_off = task->string_table_base_offset + bucket->u.offset;
// update name offset
{
@@ -3138,8 +3138,257 @@ THREAD_POOL_TASK_FUNC(lnk_push_dbi_sec_contrib_task)
}
}
typedef struct LNK_PdbWriteJob
{
U64 file_off;
String8 data;
} LNK_PdbWriteJob;
typedef struct LNK_PdbWriter
{
Arena *queue_arena;
String8 path;
String8 temp_path;
String8 open_path;
File file;
GuardedRing *queue;
Thread thread;
MSF_StreamNumber *sealed_streams;
U64 sealed_stream_count;
U64 sealed_stream_cap;
U64 bytes_written;
B32 open_with_rename;
B32 is_open;
B32 write_failed;
} LNK_PdbWriter;
typedef struct LNK_MsfPageCursor
{
MSF_PageDataNode *node;
U64 node_idx;
U64 pages_per_node;
} LNK_MsfPageCursor;
internal U8 *
lnk_msf_data_from_pn(LNK_MsfPageCursor *cursor, MSF_Context *msf, MSF_PageNumber pn)
{
U64 node_idx = pn / cursor->pages_per_node;
if (node_idx < cursor->node_idx) {
cursor->node = msf->page_data_list.first;
cursor->node_idx = 0;
}
while (cursor->node_idx < node_idx) {
cursor->node = cursor->node->next;
cursor->node_idx += 1;
}
Assert(cursor->node != 0);
return cursor->node->data + (pn % cursor->pages_per_node) * msf->page_size;
}
internal void
lnk_pdb_writer_thread(void *raw_writer)
{
ProfBeginFunction();
LNK_PdbWriter *writer = raw_writer;
for (;;) {
LNK_PdbWriteJob job = {0};
RingGuard guard = guarded_ring_open(writer->queue);
B32 is_read = guarded_ring_read_struct_or_wait(&guard, &job, max_U64);
guarded_ring_close(&guard);
Assert(is_read);
if (job.data.str == 0) { break; }
U64 write_size = lnk_write_file(&writer->file, job.file_off, job.data.str, job.data.size);
if (write_size != job.data.size) {
writer->write_failed = 1;
}
writer->bytes_written += write_size;
}
ProfEnd();
}
internal B32
lnk_pdb_writer_begin(Arena *arena, LNK_PdbWriter *writer, String8 path, String8 temp_path, U64 stream_cap)
{
ProfBegin("PDB Writer Begin");
B32 is_ok = 1;
writer->path = path;
writer->temp_path = temp_path;
writer->open_with_rename = (temp_path.size > 0);
writer->sealed_stream_cap = stream_cap;
writer->sealed_streams = push_array_no_zero(arena, MSF_StreamNumber, stream_cap);
if (writer->open_with_rename) {
writer->file = lnk_file_open_with_rename_permissions(temp_path);
writer->open_path = temp_path;
} else {
lnk_open_file_write((char *)path.str, path.size, &writer->file, sizeof(writer->file));
writer->open_path = path;
}
writer->is_open = !file_match(writer->file, file_zero());
if (!writer->is_open) {
lnk_error(LNK_Error_NoAccess, "don't have access to write to %S", path);
goto exit;
}
if (writer->open_with_rename && !lnk_file_set_delete_on_close(writer->file, 1)) {
lnk_error(LNK_Error_IO, "failed to update file disposition on %S", writer->open_path);
}
writer->queue_arena = arena_alloc(.reserve_size = MB(2), .commit_size = MB(2), .name = "PDB_WRITE_QUEUE");
writer->queue = guarded_ring_alloc(writer->queue_arena, MB(1));
writer->thread = thread_launch(lnk_pdb_writer_thread, writer);
is_ok = 1;
exit:;
ProfEnd();
return is_ok;
}
internal void
lnk_pdb_writer_enqueue_job(LNK_PdbWriter *writer, U64 file_off, U8 *data, U64 size)
{
ProfBegin("PDB Writer Enqueue Job");
if (writer->is_open && size > 0) {
LNK_PdbWriteJob job = { .file_off = file_off, .data = str8(data, size) };
RingGuard guard = guarded_ring_open(writer->queue);
B32 is_written = guarded_ring_write_struct_or_wait(&guard, &job, max_U64);
guarded_ring_close(&guard);
Assert(is_written);
}
ProfEnd();
}
internal void
lnk_pdb_writer_enqueue_stream(LNK_PdbWriter *writer, MSF_Context *msf, MSF_StreamNumber sn)
{
if (!writer->is_open || sn == MSF_INVALID_STREAM_NUMBER) { return; }
Assert(writer->sealed_stream_count < writer->sealed_stream_cap);
writer->sealed_streams[writer->sealed_stream_count++] = sn;
MSF_Stream *stream = msf_find_stream(msf, sn);
if (stream == 0 || stream->page_list.count == 0) { return; }
LNK_MsfPageCursor cursor = {
.node = msf->page_data_list.first,
.pages_per_node = msf_get_data_node_size(msf->page_size) / msf->page_size,
};
MSF_PageNumber run_first_pn = 0;
MSF_PageNumber run_last_pn = 0;
U8 *run_data = 0;
U64 run_size = 0;
for EachNode(page, MSF_PageNode, stream->page_list.first) {
U8 *page_data = lnk_msf_data_from_pn(&cursor, msf, page->pn);
if (run_data != 0 && page->pn == run_last_pn + 1 && page_data == run_data + run_size) {
run_last_pn = page->pn;
run_size += msf->page_size;
} else {
if (run_data != 0) {
lnk_pdb_writer_enqueue_job(writer, (U64)run_first_pn * msf->page_size, run_data, run_size);
}
run_first_pn = run_last_pn = page->pn;
run_data = page_data;
run_size = msf->page_size;
}
}
if (run_data != 0) {
lnk_pdb_writer_enqueue_job(writer, (U64)run_first_pn * msf->page_size, run_data, run_size);
}
}
internal void
lnk_pdb_writer_finalize_stream(void *user_data, MSF_Context *msf, MSF_StreamNumber sn)
{
lnk_pdb_writer_enqueue_stream(user_data, msf, sn);
}
internal B32
lnk_pdb_writer_finish(LNK_PdbWriter *writer, MSF_Context *msf)
{
if (!writer->is_open) { return 0; }
Temp scratch = scratch_begin(0,0);
U64 save_size = msf_get_save_size(msf);
U64 page_count = CeilIntegerDiv(save_size, msf->page_size);
U8 *is_written = push_array(scratch.arena, U8, page_count);
for EachIndex(i, writer->sealed_stream_count) {
MSF_Stream *stream = msf_find_stream(msf, writer->sealed_streams[i]);
if (stream == 0) { continue; }
for EachNode(page, MSF_PageNode, stream->page_list.first) {
Assert(page->pn < page_count);
is_written[page->pn] = 1;
}
}
LNK_MsfPageCursor cursor = {
.node = msf->page_data_list.first,
.pages_per_node = msf_get_data_node_size(msf->page_size) / msf->page_size,
};
U64 run_first_pn = 0;
U8 *run_data = 0;
U64 run_size = 0;
for EachIndex(pn, page_count) {
U8 *page_data = lnk_msf_data_from_pn(&cursor, msf, pn);
U64 page_size = Min(msf->page_size, save_size - pn * msf->page_size);
if (!is_written[pn]) {
if (run_data != 0 && page_data == run_data + run_size) {
run_size += page_size;
} else {
if (run_data != 0) {
lnk_pdb_writer_enqueue_job(writer, run_first_pn * msf->page_size, run_data, run_size);
}
run_first_pn = pn;
run_data = page_data;
run_size = page_size;
}
} else if (run_data != 0) {
lnk_pdb_writer_enqueue_job(writer, run_first_pn * msf->page_size, run_data, run_size);
run_data = 0;
run_size = 0;
}
}
if (run_data != 0) {
lnk_pdb_writer_enqueue_job(writer, run_first_pn * msf->page_size, run_data, run_size);
}
LNK_PdbWriteJob stop = {0};
RingGuard guard = guarded_ring_open(writer->queue);
B32 is_written_stop = guarded_ring_write_struct_or_wait(&guard, &stop, max_U64);
guarded_ring_close(&guard);
Assert(is_written_stop);
thread_join(writer->thread, -1);
B32 is_complete = !writer->write_failed && writer->bytes_written == save_size;
if (is_complete && writer->open_with_rename) {
if (!lnk_file_set_delete_on_close(writer->file, 0)) {
lnk_error(LNK_Error_IO, "failed to update file disposition on %S", writer->open_path);
is_complete = 0;
} else if (!lnk_file_rename(writer->file, writer->path)) {
lnk_error(LNK_Error_IO, "failed to rename %S -> %S", writer->temp_path, writer->path);
is_complete = 0;
}
}
lnk_close_file(&writer->file);
guarded_ring_release(writer->queue);
arena_release(writer->queue_arena);
if (!is_complete) {
lnk_error(LNK_Error_IO, "incomplete PDB write, %M written, expected %M, file %S", writer->bytes_written, save_size, writer->path);
} else if (lnk_get_log_status(LNK_Log_IO_Write)) {
lnk_log(LNK_Log_IO_Write, "File \"%S\" %M written", writer->path, save_size);
}
scratch_end(scratch);
return is_complete;
}
internal String8List
lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config *config, LNK_SymbolTable *symtab, LNK_CodeViewInput *cv, LNK_MergedTypes cv_types, LNK_PDB_BuilderFlags builder_flags)
lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config *config, LNK_SymbolTable *symtab, LNK_CodeViewInput *cv, LNK_MergedTypes cv_types, String8 output_path, String8 temp_output_path, LNK_PDB_BuilderFlags builder_flags)
{
ProfBeginFunction();
Temp scratch = scratch_begin(tp_arena->v, tp_arena->count);
@@ -3163,6 +3412,15 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config
.image_section_file_section_numbers = push_array(scratch.arena, U64, task.image_section_table_count),
};
LNK_PdbWriter writer = {0};
if (output_path.size > 0) {
lnk_pdb_writer_begin(scratch.arena, &writer, output_path, temp_output_path, cv->obj_count + 128);
}
PDB_BuildHooks build_hooks = {
.stream_finalize = lnk_pdb_writer_finalize_stream,
.user_data = &writer,
};
// set min type indices
for EachElement(ti_source, cv_types.min_type_indices) { task.pdb->type_servers[ti_source]->ti_lo = cv_types.min_type_indices[ti_source]; }
@@ -3190,20 +3448,36 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config
cv_string_hash_table_assign_buffer_offsets(tp, task.string_ht);
ProfEnd();
task.string_table_base_offset = task.pdb->info->strtab.size;
if (writer.is_open) {
ProfBegin("Add string tables");
pdb_strtab_add_cv_string_hash_table(&task.pdb->info->strtab, task.string_ht);
ProfEnd();
pdb_build_types(tp, task.pdb, &build_hooks);
}
if (builder_flags & LNK_PDB_BuilderFlag_Modules) {
ProfScope ("Alloc Modules")
for EachIndex(obj_idx, cv->obj_count) {
task.mod_arr[obj_idx] = dbi_push_module(task.pdb->dbi, cv->obj_arr[obj_idx]->path, lnk_obj_get_lib_path(cv->obj_arr[obj_idx]));
}
ProfScope("Write Modules") tp_for_parallel(tp, 0, tp->worker_count, lnk_write_pdb_modules, &task);
for EachIndex(obj_idx, cv->obj_count) {
lnk_pdb_writer_enqueue_stream(&writer, task.pdb->msf, task.mod_arr[obj_idx]->sn);
}
ProfScope("Move Global Symbols") tp_for_parallel(tp, 0, tp->worker_count, lnk_move_global_symbols_to_gsi, &task);
ProfScope("Build GSI and PSI") pdb_build_gsi_psi(tp, task.pdb);
ProfScope("Write Modules") tp_for_parallel(tp, 0, tp->worker_count, lnk_write_pdb_modules, &task);
lnk_pdb_writer_enqueue_stream(&writer, task.pdb->msf, task.pdb->dbi->publics_sn);
lnk_pdb_writer_enqueue_stream(&writer, task.pdb->msf, task.pdb->dbi->globals_sn);
lnk_pdb_writer_enqueue_stream(&writer, task.pdb->msf, task.pdb->dbi->symbols_sn);
}
ProfBegin("Add string tables");
pdb_strtab_add_cv_string_hash_table(&task.pdb->info->strtab, task.string_ht);
ProfEnd();
if (!writer.is_open) {
ProfBegin("Add string tables");
pdb_strtab_add_cv_string_hash_table(&task.pdb->info->strtab, task.string_ht);
ProfEnd();
}
if (builder_flags & LNK_PDB_BuilderFlag_SC) {
ProfBegin("Build Section Contrib Map");
@@ -3265,7 +3539,11 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config
ProfEnd();
}
pdb_build(tp, tp_arena, task.pdb, task.string_ht, 0, cv->is_stripped);
if (writer.is_open) {
pdb_build_dbi_info(tp, task.pdb, task.string_ht, 0, cv->is_stripped, &build_hooks);
} else {
pdb_build(tp, tp_arena, task.pdb, task.string_ht, 0, cv->is_stripped, 0);
}
MSF_Error msf_err = msf_build(task.pdb->msf);
if (msf_err != MSF_Error_OK) {
@@ -3275,6 +3553,10 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config
ProfBegin("Get Page Nodes");
String8List page_data_list = msf_get_page_data_nodes(tp_arena->v[0], task.pdb->msf);
ProfEnd();
if (writer.is_open) {
lnk_pdb_writer_finish(&writer, task.pdb->msf);
}
// NOTE: linker is about to exit so we can skip memory release
// and let windows free memory since it does this faster
+2 -1
View File
@@ -223,6 +223,7 @@ typedef struct
LNK_CodeViewInput *cv;
PDB_Context *pdb;
CV_StringHashTable string_ht;
U64 string_table_base_offset;
PDB_DbiModule **mod_arr; // [obj_count]
U32Array *obj_indices; // [obj_count]
@@ -273,4 +274,4 @@ internal void lnk_replace_type_names_with_hashes (TP_Context *tp, TP
////////////////////////////////
// PDB
internal String8List lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config *config, LNK_SymbolTable *symtab, LNK_CodeViewInput *cv, LNK_MergedTypes cv_types, LNK_PDB_BuilderFlags builder_flags);
internal String8List lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config *config, LNK_SymbolTable *symtab, LNK_CodeViewInput *cv, LNK_MergedTypes cv_types, String8 output_path, String8 temp_output_path, LNK_PDB_BuilderFlags builder_flags);
+37 -4
View File
@@ -3418,31 +3418,64 @@ pdb_build_gsi_psi(TP_Context *tp, PDB_Context *pdb)
}
internal void
pdb_build(TP_Context *tp, TP_Arena *pool_temp, PDB_Context *pdb, CV_StringHashTable string_ht, B32 build_gsi, B32 is_stripped)
pdb_build_types(TP_Context *tp, PDB_Context *pdb, PDB_BuildHooks *hooks)
{
ProfBeginFunction();
PDB_InfoContext *info = pdb->info;
PDB_StringTable *strtab = &info->strtab;
PDB_DbiContext *dbi = pdb->dbi;
PDB_TypeServer *tpi = pdb->type_servers[CV_TypeIndexSource_TPI];
PDB_TypeServer *ipi = pdb->type_servers[CV_TypeIndexSource_IPI];
pdb_type_server_build(tp, tpi, strtab, pdb->msf, PDB_FixedStream_Tpi);
if (hooks && hooks->stream_finalize) {
hooks->stream_finalize(hooks->user_data, pdb->msf, PDB_FixedStream_Tpi);
hooks->stream_finalize(hooks->user_data, pdb->msf, tpi->hash_sn);
}
if (info->flags & PDB_FeatureFlag_HAS_ID_STREAM) {
pdb_type_server_build(tp, ipi, strtab, pdb->msf, PDB_FixedStream_Ipi);
if (hooks && hooks->stream_finalize) {
hooks->stream_finalize(hooks->user_data, pdb->msf, PDB_FixedStream_Ipi);
hooks->stream_finalize(hooks->user_data, pdb->msf, ipi->hash_sn);
}
}
ProfEnd();
}
internal void
pdb_build_dbi_info(TP_Context *tp, PDB_Context *pdb, CV_StringHashTable string_ht, B32 build_gsi, B32 is_stripped, PDB_BuildHooks *hooks)
{
ProfBeginFunction();
if (build_gsi) {
pdb_build_gsi_psi(tp, pdb);
}
dbi_build(tp, pdb->dbi, pdb->msf, PDB_FixedStream_Dbi, string_ht, is_stripped);
if (hooks && hooks->stream_finalize) {
hooks->stream_finalize(hooks->user_data, pdb->msf, PDB_FixedStream_Dbi);
for EachElement(i, pdb->dbi->dbg_streams) {
hooks->stream_finalize(hooks->user_data, pdb->msf, pdb->dbi->dbg_streams[i]);
}
}
pdb_info_build(pdb->info, pdb->msf, PDB_FixedStream_Info);
if (hooks && hooks->stream_finalize) {
hooks->stream_finalize(hooks->user_data, pdb->msf, PDB_FixedStream_Info);
}
ProfEnd();
}
internal void
pdb_build(TP_Context *tp, TP_Arena *pool_temp, PDB_Context *pdb, CV_StringHashTable string_ht, B32 build_gsi, B32 is_stripped, PDB_BuildHooks *hooks)
{
ProfBeginFunction();
pdb_build_types(tp, pdb, hooks);
pdb_build_dbi_info(tp, pdb, string_ht, build_gsi, is_stripped, hooks);
ProfEnd();
}
////////////////////////////////
internal String8
+10 -3
View File
@@ -310,6 +310,13 @@ typedef struct PDB_Context
PDB_TypeServer *type_servers[CV_TypeIndexSource_COUNT];
} PDB_Context;
typedef void PDB_StreamFinalizeFunc(void *user_data, MSF_Context *msf, MSF_StreamNumber sn);
typedef struct PDB_BuildHooks
{
PDB_StreamFinalizeFunc *stream_finalize;
void *user_data;
} PDB_BuildHooks;
////////////////////////////////
typedef struct
@@ -334,7 +341,9 @@ typedef struct
internal PDB_Context * pdb_alloc(U64 page_size, COFF_MachineType machine, COFF_TimeStamp time_stamp, U32 age, Guid guid);
internal void pdb_release(PDB_Context *pdb);
internal void pdb_build(TP_Context *tp, TP_Arena *pool_temp, PDB_Context *pdb, CV_StringHashTable string_ht, B32 build_gsi, B32 is_stripped);
internal void pdb_build_types(TP_Context *tp, PDB_Context *pdb, PDB_BuildHooks *hooks);
internal void pdb_build_dbi_info(TP_Context *tp, PDB_Context *pdb, CV_StringHashTable string_ht, B32 build_gsi, B32 is_stripped, PDB_BuildHooks *hooks);
internal void pdb_build(TP_Context *tp, TP_Arena *pool_temp, PDB_Context *pdb, CV_StringHashTable string_ht, B32 build_gsi, B32 is_stripped, PDB_BuildHooks *hooks);
internal void pdb_set_machine(PDB_Context *pdb, COFF_MachineType machine);
internal void pdb_set_guid(PDB_Context *pdb, Guid guid);
internal void pdb_set_time_stamp(PDB_Context *pdb, COFF_TimeStamp time_stamp);
@@ -461,5 +470,3 @@ internal PDB_TypeHashStreamInfo pdb_type_hash_stream_build(TP_Context *tp, PDB_
// Enum -> String
internal String8 pdb_string_from_src_error(PDB_SrcError error);
+1 -1
View File
@@ -6843,7 +6843,7 @@ data_from_pdb(Arena *arena, PDB_Context *pdb)
{
TP_Context *tp = tp_alloc(arena, 1, 1, str8_lit("foo"));
TP_Arena *tp_arena = tp_arena_alloc(tp);
pdb_build(tp, tp_arena, pdb, (CV_StringHashTable){0}, 1, 0);
pdb_build(tp, tp_arena, pdb, (CV_StringHashTable){0}, 1, 0, 0);
AssertAlways(msf_build(pdb->msf) == MSF_Error_OK);
String8List raw_msf_list = msf_get_page_data_nodes(arena, pdb->msf);