refactor PDB background file output

Add a shared multi-file background writer with lazy worker startup.
Represent PDB output as a file artifact, keep PDB output state internal
to the builder, and overlap deferred PDB writers with the file maps release.

src/base/base_arena.c

@@ -28,6 +28,8 @@ global ArenaTableNode *free_arena_table_node = 0;
 internal Arena *
 arena_alloc_(ArenaParams *params)
 {
+  ProfBeginFunction();
+
   U64 reserve_size = params->reserve_size;
   U64 commit_size  = params->commit_size;
   
@@ -133,12 +135,15 @@ arena_alloc_(ArenaParams *params)
   }
 #endif
   
+  ProfEnd();
   return arena;
 }
 
 internal void
 arena_release(Arena *arena)
 {
+  ProfBeginFunction();
+
 #if PROFILE_TELEMETRY
   {
     Arena *base_arena = arena;
@@ -171,6 +176,8 @@ arena_release(Arena *arena)
     AsanUnpoisonMemoryRegion(n, n->cmt);
     release_memory(n, n->res);
   }
+
+  ProfEnd();
 }
 
 //- rjf: arena push/pop core functions

src/base/base_ring.c

@@ -46,18 +46,22 @@ ring_try_read(Ring *ring, U64 size, void *ptr)
 internal GuardedRing *
 guarded_ring_alloc(Arena *arena, U64 size)
 {
+  ProfBeginFunction();
   GuardedRing *gr = push_array(arena, GuardedRing, 1);
   gr->ring = make_ring(arena, size);
   gr->mutex = mutex_alloc();
   gr->cv = cond_var_alloc();
+  ProfEnd();
   return gr;
 }
 
 internal void
 guarded_ring_release(GuardedRing *ring)
 {
+  ProfBeginFunction();
   mutex_release(ring->mutex);
   cond_var_release(ring->cv);
+  ProfEnd();
 }
 
 internal RingGuard

src/linker/lnk.c

@@ -182,6 +182,9 @@ lnk_make_default_cmd_line(Arena *arena, LNK_CmdLine user_cmd_line)
 
     // Use LLVM significant addresses hints for the /OPT:ICF.
     "/LLVM_ADDRSIG",
+
+    // By default keep full type names, override when TPI/IPI streams overflow.
+    "/RAD_PDB_HASH_TYPE_NAMES:NONE",
   };
 
   char *push_opts[] = {
@@ -6366,6 +6369,11 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config)
   image_write_ctx->data      = image_ctx.image_data;
   Thread image_write_thread = thread_launch(lnk_write_thread, image_write_ctx);
 
+  LNK_BackgroundFileWriter background_file_writer = {0};
+  LNK_PdbWriter            pdb_writer = { .file_writer = &background_file_writer };
+  Temp                     pdb_huge_temp = temp_begin(lnk_get_huge_arena());
+  lnk_background_file_writer_begin(pdb_writer.file_writer);
+
   //
   // RAD Map
   //
@@ -6410,12 +6418,11 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config)
     // TODO: Parallel debug info builds are currently blocked by the patch
     // strings in $$FILE_CHECKSUM step in `lnk_process_c13_data_task`.
     if (config->debug_mode == LNK_DebugMode_Full || config->rad_debug == LNK_SwitchState_Yes) {
-      Temp huge_arena_temp = temp_begin(lnk_get_huge_arena());
+      LNK_FileArtifact pdb_artifact = {0};
-
-      String8List pdb_data = {0};
       {
         lnk_timer_begin(LNK_Timer_Pdb);
-        if (config->pdb_hash_type_names != LNK_TypeNameHashMode_Null && config->pdb_hash_type_names != LNK_TypeNameHashMode_None) {
+
+        if (config->pdb_hash_type_names != LNK_TypeNameHashMode_None) {
           lnk_replace_type_names_with_hashes(tp,
                                              arena,
                                              cv_types.count[CV_TypeIndexSource_TPI],
@@ -6424,16 +6431,18 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config)
                                              config->pdb_hash_type_name_length,
                                              config->pdb_hash_type_name_map);
         }
-        String8 pdb_output_path      = config->debug_mode == LNK_DebugMode_Full ? config->pdb_name      : str8_zero();
+
-        String8 pdb_temp_output_path = config->debug_mode == LNK_DebugMode_Full ? config->temp_pdb_name : str8_zero();
+        pdb_writer.output_path      = config->debug_mode == LNK_DebugMode_Full ? config->pdb_name      : str8_zero();
-        pdb_data = lnk_build_pdb(tp, arena, image_ctx.image_data, config, symtab, &cv, cv_types, pdb_output_path, pdb_temp_output_path, LNK_PDB_BuilderFlag_All);
+        pdb_writer.temp_output_path = config->debug_mode == LNK_DebugMode_Full ? config->temp_pdb_name : str8_zero();
+        pdb_artifact                = lnk_build_pdb(tp, arena, image_ctx.image_data, config, symtab, &cv, cv_types, pdb_writer, LNK_PDB_BuilderFlag_All);
+
         lnk_timer_end(LNK_Timer_Pdb);
       }
 
       if (config->rad_debug == LNK_SwitchState_Yes) {
         lnk_timer_begin(LNK_Timer_Rdi);
 
-        LNK_P2R p2r = { .config = config, .pdb_data = str8_list_join(lnk_get_huge_arena(), &pdb_data, 0), .image_data = image_ctx.image_data };
+        LNK_P2R p2r = { .config = config, .pdb_data = lnk_data_from_file_artifact(lnk_get_huge_arena(), &pdb_artifact), .image_data = image_ctx.image_data };
         tp_for_parallel(tp, arena, tp->worker_count, lnk_p2r_worker, &p2r);
 
         String8List rdi_blobs = rdim_file_blobs_from_section_bundle(scratch.arena, &p2r.bake_results.section_bundle);
@@ -6441,8 +6450,6 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config)
 
         lnk_timer_end(LNK_Timer_Rdi);
       }
-
-      temp_end(huge_arena_temp);
     }
 
     //
@@ -6510,8 +6517,8 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config)
       stripped_cv.debug_s_arr         = debug_s_arr;
       stripped_cv.symbol_input_ranges = push_array(scratch.arena, Rng1U64, tp->worker_count);
 
-      String8List pdb_data = lnk_build_pdb(tp, arena, image_ctx.image_data, config, symtab, &stripped_cv, (LNK_MergedTypes){0}, str8_zero(), str8_zero(), LNK_PDB_BuilderFlag_All);
+      LNK_FileArtifact pdb_artifact = lnk_build_pdb(tp, arena, image_ctx.image_data, config, symtab, &stripped_cv, (LNK_MergedTypes){0}, (LNK_PdbWriter){0}, LNK_PDB_BuilderFlag_All);
-      lnk_write_data_list_to_file_path(config->pdb_stripped_name, str8f(scratch.arena, "%S.tmp", config->pdb_stripped_name), pdb_data);
+      lnk_write_data_list_to_file_path(config->pdb_stripped_name, str8f(scratch.arena, "%S.tmp", config->pdb_stripped_name), pdb_artifact.data);
     }
 
     lnk_timer_end(LNK_Timer_Debug);
@@ -6525,6 +6532,10 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config)
   ProfEnd();
 #endif
 
+  // PDB output borrows pages from the huge arena, so drain after map release
+  lnk_background_file_writer_end(pdb_writer.file_writer);
+  temp_end(pdb_huge_temp);
+
   // wait for the thread to finish writing image to disk
   thread_join(image_write_thread, -1);
 

src/linker/lnk_debug_info.c

@@ -3138,29 +3138,14 @@ THREAD_POOL_TASK_FUNC(lnk_push_dbi_sec_contrib_task)
   }
 }
 
-typedef struct LNK_PdbWriteJob
+typedef struct
 {
-  U64     file_off;
+  LNK_BackgroundFileWriter *writer;
-  String8 data;
+  LNK_BackgroundFile       *file;
-} LNK_PdbWriteJob;
+  MSF_StreamNumber         *sealed_streams;
-
+  U64                       sealed_stream_count;
-typedef struct LNK_PdbWriter
+  U64                       sealed_stream_cap;
-{
+} LNK_PdbOutput;
-  Arena            *queue_arena;
-  String8           path;
-  String8           temp_path;
-  String8           open_path;
-  File              file;
-  GuardedRing      *queue;
-  Thread            thread;
-  MSF_StreamNumber *sealed_streams;
-  U64               sealed_stream_count;
-  U64               sealed_stream_cap;
-  U64               bytes_written;
-  B32               open_with_rename;
-  B32               is_open;
-  B32               write_failed;
-} LNK_PdbWriter;
 
 typedef struct LNK_MsfPageCursor
 {
@@ -3186,94 +3171,15 @@ lnk_msf_data_from_pn(LNK_MsfPageCursor *cursor, MSF_Context *msf, MSF_PageNumber
 }
 
 internal void
-lnk_pdb_writer_thread(void *raw_writer)
+lnk_pdb_output_enqueue_stream(LNK_PdbOutput *output, MSF_Context *msf, MSF_StreamNumber sn)
 {
-  ProfBeginFunction();
+  if (sn == MSF_INVALID_STREAM_NUMBER) { return; }
-  LNK_PdbWriter *writer = raw_writer;
+  Assert(output->sealed_stream_count < output->sealed_stream_cap);
-  for (;;) {
+  output->sealed_streams[output->sealed_stream_count++] = sn;
-    LNK_PdbWriteJob job = {0};
-    RingGuard guard = guarded_ring_open(writer->queue);
-    B32 is_read = guarded_ring_read_struct_or_wait(&guard, &job, max_U64);
-    guarded_ring_close(&guard);
-    Assert(is_read);
-
-    if (job.data.str == 0) { break; }
-
-    U64 write_size = lnk_write_file(&writer->file, job.file_off, job.data.str, job.data.size);
-    if (write_size != job.data.size) {
-      writer->write_failed = 1;
-    }
-    writer->bytes_written += write_size;
-  }
-  ProfEnd();
-}
-
-internal B32
-lnk_pdb_writer_begin(Arena *arena, LNK_PdbWriter *writer, String8 path, String8 temp_path, U64 stream_cap)
-{
-  ProfBegin("PDB Writer Begin");
-
-  B32 is_ok = 1;
-
-  writer->path              = path;
-  writer->temp_path         = temp_path;
-  writer->open_with_rename  = (temp_path.size > 0);
-  writer->sealed_stream_cap = stream_cap;
-  writer->sealed_streams    = push_array_no_zero(arena, MSF_StreamNumber, stream_cap);
-
-  if (writer->open_with_rename) {
-    writer->file      = lnk_file_open_with_rename_permissions(temp_path);
-    writer->open_path = temp_path;
-  } else {
-    lnk_open_file_write((char *)path.str, path.size, &writer->file, sizeof(writer->file));
-    writer->open_path = path;
-  }
-
-  writer->is_open = !file_match(writer->file, file_zero());
-  if (!writer->is_open) {
-    lnk_error(LNK_Error_NoAccess, "don't have access to write to %S", path);
-    goto exit;
-  }
-
-  if (writer->open_with_rename && !lnk_file_set_delete_on_close(writer->file, 1)) {
-    lnk_error(LNK_Error_IO, "failed to update file disposition on %S", writer->open_path);
-  }
-
-  writer->queue_arena = arena_alloc(.reserve_size = MB(2), .commit_size = MB(2), .name = "PDB_WRITE_QUEUE");
-  writer->queue  = guarded_ring_alloc(writer->queue_arena, MB(1));
-  writer->thread = thread_launch(lnk_pdb_writer_thread, writer);
-
-  is_ok = 1;
-  exit:;
-  ProfEnd();
-  return is_ok;
-}
-
-internal void
-lnk_pdb_writer_enqueue_job(LNK_PdbWriter *writer, U64 file_off, U8 *data, U64 size)
-{
-  ProfBegin("PDB Writer Enqueue Job");
-
-  if (writer->is_open && size > 0) {
-    LNK_PdbWriteJob job = { .file_off = file_off, .data = str8(data, size) };
-    RingGuard guard = guarded_ring_open(writer->queue);
-    B32 is_written = guarded_ring_write_struct_or_wait(&guard, &job, max_U64);
-    guarded_ring_close(&guard);
-    Assert(is_written);
-  }
-
-  ProfEnd();
-}
-
-internal void
-lnk_pdb_writer_enqueue_stream(LNK_PdbWriter *writer, MSF_Context *msf, MSF_StreamNumber sn)
-{
-  if (!writer->is_open || sn == MSF_INVALID_STREAM_NUMBER) { return; }
-  Assert(writer->sealed_stream_count < writer->sealed_stream_cap);
-  writer->sealed_streams[writer->sealed_stream_count++] = sn;
 
   MSF_Stream *stream = msf_find_stream(msf, sn);
-  if (stream == 0 || stream->page_list.count == 0) { return; }
+  Assert(stream != 0);
+  if (stream->page_list.count == 0) { return; }
 
   LNK_MsfPageCursor cursor = {
     .node           = msf->page_data_list.first,
@@ -3290,7 +3196,7 @@ lnk_pdb_writer_enqueue_stream(LNK_PdbWriter *writer, MSF_Context *msf, MSF_Strea
       run_size += msf->page_size;
     } else {
       if (run_data != 0) {
-        lnk_pdb_writer_enqueue_job(writer, (U64)run_first_pn * msf->page_size, run_data, run_size);
+        lnk_background_file_writer_enqueue(output->writer, output->file, (U64)run_first_pn * msf->page_size, str8(run_data, run_size));
       }
       run_first_pn = run_last_pn = page->pn;
       run_data = page_data;
@@ -3298,28 +3204,27 @@ lnk_pdb_writer_enqueue_stream(LNK_PdbWriter *writer, MSF_Context *msf, MSF_Strea
     }
   }
   if (run_data != 0) {
-    lnk_pdb_writer_enqueue_job(writer, (U64)run_first_pn * msf->page_size, run_data, run_size);
+    lnk_background_file_writer_enqueue(output->writer, output->file, (U64)run_first_pn * msf->page_size, str8(run_data, run_size));
   }
 }
 
 internal void
-lnk_pdb_writer_finalize_stream(void *user_data, MSF_Context *msf, MSF_StreamNumber sn)
+lnk_pdb_output_finalize_stream(void *user_data, MSF_Context *msf, MSF_StreamNumber sn)
 {
-  lnk_pdb_writer_enqueue_stream(user_data, msf, sn);
+  lnk_pdb_output_enqueue_stream(user_data, msf, sn);
 }
 
-internal B32
+internal void
-lnk_pdb_writer_finish(LNK_PdbWriter *writer, MSF_Context *msf)
+lnk_pdb_output_enqueue_remaining(LNK_PdbOutput *output, MSF_Context *msf)
 {
-  if (!writer->is_open) { return 0; }
   Temp scratch = scratch_begin(0,0);
   U64 save_size  = msf_get_save_size(msf);
   U64 page_count = CeilIntegerDiv(save_size, msf->page_size);
   U8 *is_written = push_array(scratch.arena, U8, page_count);
 
-  for EachIndex(i, writer->sealed_stream_count) {
+  for EachIndex(i, output->sealed_stream_count) {
-    MSF_Stream *stream = msf_find_stream(msf, writer->sealed_streams[i]);
+    MSF_Stream *stream = msf_find_stream(msf, output->sealed_streams[i]);
-    if (stream == 0) { continue; }
+    Assert(stream != 0);
     for EachNode(page, MSF_PageNode, stream->page_list.first) {
       Assert(page->pn < page_count);
       is_written[page->pn] = 1;
@@ -3341,54 +3246,26 @@ lnk_pdb_writer_finish(LNK_PdbWriter *writer, MSF_Context *msf)
         run_size += page_size;
       } else {
         if (run_data != 0) {
-          lnk_pdb_writer_enqueue_job(writer, run_first_pn * msf->page_size, run_data, run_size);
+          lnk_background_file_writer_enqueue(output->writer, output->file, run_first_pn * msf->page_size, str8(run_data, run_size));
         }
         run_first_pn = pn;
         run_data = page_data;
         run_size = page_size;
       }
     } else if (run_data != 0) {
-      lnk_pdb_writer_enqueue_job(writer, run_first_pn * msf->page_size, run_data, run_size);
+      lnk_background_file_writer_enqueue(output->writer, output->file, run_first_pn * msf->page_size, str8(run_data, run_size));
       run_data = 0;
       run_size = 0;
     }
   }
   if (run_data != 0) {
-    lnk_pdb_writer_enqueue_job(writer, run_first_pn * msf->page_size, run_data, run_size);
+    lnk_background_file_writer_enqueue(output->writer, output->file, run_first_pn * msf->page_size, str8(run_data, run_size));
-  }
-
-  LNK_PdbWriteJob stop = {0};
-  RingGuard guard = guarded_ring_open(writer->queue);
-  B32 is_written_stop = guarded_ring_write_struct_or_wait(&guard, &stop, max_U64);
-  guarded_ring_close(&guard);
-  Assert(is_written_stop);
-  thread_join(writer->thread, -1);
-
-  B32 is_complete = !writer->write_failed && writer->bytes_written == save_size;
-  if (is_complete && writer->open_with_rename) {
-    if (!lnk_file_set_delete_on_close(writer->file, 0)) {
-      lnk_error(LNK_Error_IO, "failed to update file disposition on %S", writer->open_path);
-      is_complete = 0;
-    } else if (!lnk_file_rename(writer->file, writer->path)) {
-      lnk_error(LNK_Error_IO, "failed to rename %S -> %S", writer->temp_path, writer->path);
-      is_complete = 0;
-    }
-  }
-
-  lnk_close_file(&writer->file);
-  guarded_ring_release(writer->queue);
-  arena_release(writer->queue_arena);
-  if (!is_complete) {
-    lnk_error(LNK_Error_IO, "incomplete PDB write, %M written, expected %M, file %S", writer->bytes_written, save_size, writer->path);
-  } else if (lnk_get_log_status(LNK_Log_IO_Write)) {
-    lnk_log(LNK_Log_IO_Write, "File \"%S\" %M written", writer->path, save_size);
   }
   scratch_end(scratch);
-  return is_complete;
 }
 
-internal String8List
+internal LNK_FileArtifact
-lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config *config, LNK_SymbolTable *symtab, LNK_CodeViewInput *cv, LNK_MergedTypes cv_types, String8 output_path, String8 temp_output_path, LNK_PDB_BuilderFlags builder_flags)
+lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config *config, LNK_SymbolTable *symtab, LNK_CodeViewInput *cv, LNK_MergedTypes cv_types, LNK_PdbWriter writer, LNK_PDB_BuilderFlags builder_flags)
 {
   ProfBeginFunction();
   Temp scratch = scratch_begin(tp_arena->v, tp_arena->count);
@@ -3412,14 +3289,23 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config
     .image_section_file_section_numbers = push_array(scratch.arena, U64, task.image_section_table_count),
   };
 
-  LNK_PdbWriter writer = {0};
+  LNK_PdbOutput  output     = {0};
-  if (output_path.size > 0) {
+  LNK_PdbOutput *output_ptr = 0;
-    lnk_pdb_writer_begin(scratch.arena, &writer, output_path, temp_output_path, cv->obj_count + 128);
+  if (writer.output_path.size > 0) {
+    output.writer            = writer.file_writer;
+    output.file              = lnk_background_file_writer_begin_file(output.writer, writer.output_path, writer.temp_output_path);
+    if (output.file != 0) {
+      output.sealed_stream_cap = cv->obj_count + 128;
+      output.sealed_streams    = push_array_no_zero(scratch.arena, MSF_StreamNumber, output.sealed_stream_cap);
+      output_ptr               = &output;
+    }
+  }
+
+  PDB_BuildHooks build_hooks = {0};
+  if (output_ptr != 0) {
+    build_hooks.stream_finalize = lnk_pdb_output_finalize_stream;
+    build_hooks.user_data       = output_ptr;
   }
-  PDB_BuildHooks build_hooks = {
-    .stream_finalize = lnk_pdb_writer_finalize_stream,
-    .user_data       = &writer,
-  };
 
   // set min type indices
   for EachElement(ti_source, cv_types.min_type_indices) { task.pdb->type_servers[ti_source]->ti_lo = cv_types.min_type_indices[ti_source]; }
@@ -3449,12 +3335,10 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config
   ProfEnd();
 
   task.string_table_base_offset = task.pdb->info->strtab.size;
-  if (writer.is_open) {
+  ProfBegin("Add string tables");
-    ProfBegin("Add string tables");
+  pdb_strtab_add_cv_string_hash_table(&task.pdb->info->strtab, task.string_ht);
-    pdb_strtab_add_cv_string_hash_table(&task.pdb->info->strtab, task.string_ht);
+  ProfEnd();
-    ProfEnd();
+  pdb_build_types(tp, task.pdb, &build_hooks);
-    pdb_build_types(tp, task.pdb, &build_hooks);
-  }
 
   if (builder_flags & LNK_PDB_BuilderFlag_Modules) {
     ProfScope ("Alloc Modules")
@@ -3463,20 +3347,18 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config
       }
 
     ProfScope("Write Modules")       tp_for_parallel(tp, 0, tp->worker_count, lnk_write_pdb_modules, &task);
-    for EachIndex(obj_idx, cv->obj_count) {
+    if (output_ptr != 0) {
-      lnk_pdb_writer_enqueue_stream(&writer, task.pdb->msf, task.mod_arr[obj_idx]->sn);
+      for EachIndex(obj_idx, cv->obj_count) {
+        lnk_pdb_output_enqueue_stream(output_ptr, task.pdb->msf, task.mod_arr[obj_idx]->sn);
+      }
     }
     ProfScope("Move Global Symbols") tp_for_parallel(tp, 0, tp->worker_count, lnk_move_global_symbols_to_gsi, &task);
     ProfScope("Build GSI and PSI")   pdb_build_gsi_psi(tp, task.pdb);
-    lnk_pdb_writer_enqueue_stream(&writer, task.pdb->msf, task.pdb->dbi->publics_sn);
+    if (output_ptr != 0) {
-    lnk_pdb_writer_enqueue_stream(&writer, task.pdb->msf, task.pdb->dbi->globals_sn);
+      lnk_pdb_output_enqueue_stream(output_ptr, task.pdb->msf, task.pdb->dbi->publics_sn);
-    lnk_pdb_writer_enqueue_stream(&writer, task.pdb->msf, task.pdb->dbi->symbols_sn);
+      lnk_pdb_output_enqueue_stream(output_ptr, task.pdb->msf, task.pdb->dbi->globals_sn);
-  }
+      lnk_pdb_output_enqueue_stream(output_ptr, task.pdb->msf, task.pdb->dbi->symbols_sn);
-
+    }
-  if (!writer.is_open) {
-    ProfBegin("Add string tables");
-    pdb_strtab_add_cv_string_hash_table(&task.pdb->info->strtab, task.string_ht);
-    ProfEnd();
   }
   
   if (builder_flags & LNK_PDB_BuilderFlag_SC) {
@@ -3539,23 +3421,23 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config
     ProfEnd();
   }
 
-  if (writer.is_open) {
+  pdb_build_dbi_info(tp, task.pdb, task.string_ht, 0, cv->is_stripped, &build_hooks);
-    pdb_build_dbi_info(tp, task.pdb, task.string_ht, 0, cv->is_stripped, &build_hooks);
-  } else {
-    pdb_build(tp, tp_arena, task.pdb, task.string_ht, 0, cv->is_stripped, 0);
-  }
 
   MSF_Error msf_err = msf_build(task.pdb->msf);
   if (msf_err != MSF_Error_OK) {
     lnk_error(LNK_Error_UnableToSerializeMsf, "unable to serialize MSF: %s", msf_error_to_string(msf_err));
   }
 
+  if (output_ptr != 0) {
+    lnk_pdb_output_enqueue_remaining(output_ptr, task.pdb->msf);
+  }
+
   ProfBegin("Get Page Nodes");
-  String8List page_data_list = msf_get_page_data_nodes(tp_arena->v[0], task.pdb->msf);
+  LNK_FileArtifact artifact = { .data = msf_get_page_data_nodes(tp_arena->v[0], task.pdb->msf) };
   ProfEnd();
 
-  if (writer.is_open) {
+  if (output_ptr != 0) {
-    lnk_pdb_writer_finish(&writer, task.pdb->msf);
+    lnk_background_file_writer_end_file(output_ptr->writer, output_ptr->file, artifact.data.total_size);
   }
 
   // NOTE: linker is about to exit so we can skip memory release
@@ -3568,5 +3450,5 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config
 
   scratch_end(scratch);
   ProfEnd();
-  return page_data_list;
+  return artifact;
 }

src/linker/lnk_debug_info.h

@@ -239,6 +239,13 @@ typedef struct
   PDB_DbiSCList       *sc_list; // [obj_count]
 } LNK_BuildPdb;
 
+typedef struct
+{
+  LNK_BackgroundFileWriter *file_writer;
+  String8                   output_path;
+  String8                   temp_output_path;
+} LNK_PdbWriter;
+
 typedef struct
 {
   U64          leaf_count;
@@ -274,4 +281,4 @@ internal void            lnk_replace_type_names_with_hashes  (TP_Context *tp, TP
 ////////////////////////////////
 // PDB
 
-internal String8List lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config *config, LNK_SymbolTable *symtab, LNK_CodeViewInput *cv, LNK_MergedTypes cv_types, String8 output_path, String8 temp_output_path, LNK_PDB_BuilderFlags builder_flags);
+internal LNK_FileArtifact lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config *config, LNK_SymbolTable *symtab, LNK_CodeViewInput *cv, LNK_MergedTypes cv_types, LNK_PdbWriter writer, LNK_PDB_BuilderFlags builder_flags);

src/linker/lnk_io.c

@@ -10,9 +10,11 @@ lnk_open_file_read(char *path, uint64_t path_size, void *handle_buffer, uint64_t
 shared_function int
 lnk_open_file_write(char *path, uint64_t path_size, void *handle_buffer, uint64_t handle_buffer_max)
 {
+  ProfBeginFunction();
   File handle = file_open(AccessFlag_Write, str8((U8*)path, path_size));
   Assert(sizeof(handle) <= handle_buffer_max);
   MemoryCopy(handle_buffer, &handle, sizeof(handle));
+  ProfEnd();
   return !file_match(handle, file_zero());
 }
 
@@ -84,6 +86,7 @@ lnk_find_first_file(Arena *arena, String8List dir_list, String8 path)
 internal File
 lnk_file_open_with_rename_permissions(String8 path)
 {
+  ProfBeginFunction();
   File file_handle = file_zero();
 #if OS_WINDOWS
   Temp scratch = scratch_begin(0,0);
@@ -106,6 +109,7 @@ lnk_file_open_with_rename_permissions(String8 path)
 #else
   file_handle = file_open(AccessFlag_Read|AccessFlag_Write, path);
 #endif
+  ProfEnd();
   return file_handle;
 }
 
@@ -401,3 +405,230 @@ lnk_write_data_to_file_path(String8 path, String8 temp_path, String8 data)
   lnk_write_data_list_to_file_path(path, temp_path, data_list);
   scratch_end(scratch);
 }
+
+internal String8
+lnk_data_from_file_artifact(Arena *arena, LNK_FileArtifact *artifact)
+{
+  return str8_list_join(arena, &artifact->data, 0);
+}
+
+// --- Background Writer -------------------------------------------------------
+
+struct LNK_BackgroundFile
+{
+  LNK_BackgroundFile *next;
+  String8             path;
+  String8             temp_path;
+  String8             open_path;
+  File                file;
+  U64                 bytes_written;
+  B32                 open_with_rename;
+  B32                 is_open;
+  B32                 is_finished;
+  B32                 is_complete;
+  B32                 write_failed;
+};
+
+typedef enum
+{
+  LNK_BackgroundFileJobKind_Write,
+  LNK_BackgroundFileJobKind_EndFile,
+  LNK_BackgroundFileJobKind_EndWriter,
+} LNK_BackgroundFileJobKind;
+
+typedef struct
+{
+  LNK_BackgroundFileJobKind kind;
+  LNK_BackgroundFile       *file;
+  U64                       file_off;
+  U64                       expected_byte_count;
+  String8                   data;
+} LNK_BackgroundFileWriteJob;
+
+internal void
+lnk_background_file_writer_end_file_on_thread(LNK_BackgroundFile *file, U64 expected_byte_count)
+{
+  B32 is_complete = !file->write_failed && file->bytes_written == expected_byte_count;
+  if (is_complete && file->open_with_rename) {
+    if (!lnk_file_set_delete_on_close(file->file, 0)) {
+      lnk_error(LNK_Error_IO, "failed to update file disposition on %S", file->open_path);
+      is_complete = 0;
+    } else if (!lnk_file_rename(file->file, file->path)) {
+      lnk_error(LNK_Error_IO, "failed to rename %S -> %S", file->temp_path, file->path);
+      is_complete = 0;
+    }
+  }
+
+  lnk_close_file(&file->file);
+  file->is_open     = 0;
+  file->is_complete = is_complete;
+
+  if (!is_complete) {
+    lnk_error(LNK_Error_IO, "incomplete write, %M written, expected %M, file %S", file->bytes_written, expected_byte_count, file->path);
+  } else if (lnk_get_log_status(LNK_Log_IO_Write)) {
+    lnk_log(LNK_Log_IO_Write, "File \"%S\" %M written", file->path, expected_byte_count);
+  }
+}
+
+internal void
+lnk_background_file_writer_thread(void *raw_writer)
+{
+  ProfBeginFunction();
+  set_thread_namef("Background File Writer");
+
+  LNK_BackgroundFileWriter *writer = raw_writer;
+  for (;;) {
+    LNK_BackgroundFileWriteJob job = {0};
+    RingGuard guard = guarded_ring_open(writer->queue);
+    B32 is_read = guarded_ring_read_struct_or_wait(&guard, &job, max_U64);
+    guarded_ring_close(&guard);
+    Assert(is_read);
+
+    if (job.kind == LNK_BackgroundFileJobKind_EndWriter) { break; }
+
+    if (job.kind == LNK_BackgroundFileJobKind_Write) {
+      U64 write_size = lnk_write_file(&job.file->file, job.file_off, job.data.str, job.data.size);
+      if (write_size != job.data.size) {
+        job.file->write_failed = 1;
+      }
+      job.file->bytes_written += write_size;
+    } else if (job.kind == LNK_BackgroundFileJobKind_EndFile) {
+      lnk_background_file_writer_end_file_on_thread(job.file, job.expected_byte_count);
+    }
+  }
+  ProfEnd();
+}
+
+internal void
+lnk_background_file_writer_begin(LNK_BackgroundFileWriter *writer)
+{
+  ProfBegin("Background File Writer Begin");
+
+  writer->queue_arena = arena_alloc(.reserve_size = MB(2), .commit_size = MB(2), .name = "BACKGROUND_FILE_WRITE_QUEUE");
+  writer->queue       = guarded_ring_alloc(writer->queue_arena, MB(1));
+  ProfEnd();
+}
+
+internal LNK_BackgroundFile *
+lnk_background_file_writer_begin_file(LNK_BackgroundFileWriter *writer, String8 path, String8 temp_path)
+{
+  ProfBegin("Background File Writer Begin File");
+
+  if (!writer->is_running) {
+    writer->thread     = thread_launch(lnk_background_file_writer_thread, writer);
+    writer->is_running = writer->thread.u64[0] != 0;
+    if (!writer->is_running) {
+      lnk_error(LNK_Error_IO, "failed to start background file writer");
+      ProfEnd();
+      return 0;
+    }
+  }
+
+  LNK_BackgroundFile *file = push_array(writer->queue_arena, LNK_BackgroundFile, 1);
+
+  file->path             = path;
+  file->temp_path        = temp_path;
+  file->open_with_rename = (temp_path.size > 0);
+
+  if (file->open_with_rename) {
+    file->file      = lnk_file_open_with_rename_permissions(temp_path);
+    file->open_path = temp_path;
+  } else {
+    lnk_open_file_write((char *)path.str, path.size, &file->file, sizeof(file->file));
+    file->open_path = path;
+  }
+
+  file->is_open = !file_match(file->file, file_zero());
+  if (!file->is_open) {
+    lnk_error(LNK_Error_NoAccess, "don't have access to write to %S", path);
+    goto exit;
+  }
+
+  if (file->open_with_rename && !lnk_file_set_delete_on_close(file->file, 1)) {
+    lnk_error(LNK_Error_IO, "failed to update file disposition on %S", file->open_path);
+    lnk_close_file(&file->file);
+    file->is_open = 0;
+    goto exit;
+  }
+
+  SLLQueuePush(writer->file_first, writer->file_last, file);
+
+  exit:;
+  ProfEnd();
+  return file->is_open ? file : 0;
+}
+
+internal void
+lnk_background_file_writer_enqueue(LNK_BackgroundFileWriter *writer, LNK_BackgroundFile *file, U64 file_off, String8 data)
+{
+  ProfBegin("Background File Writer Enqueue");
+
+  Assert(writer->is_running);
+  Assert(file->is_open && !file->is_finished);
+
+  if (data.size > 0) {
+    LNK_BackgroundFileWriteJob job = {
+      .kind     = LNK_BackgroundFileJobKind_Write,
+      .file     = file,
+      .file_off = file_off,
+      .data     = data,
+    };
+    RingGuard guard      = guarded_ring_open(writer->queue);
+    B32       is_written = guarded_ring_write_struct_or_wait(&guard, &job, max_U64);
+    guarded_ring_close(&guard);
+    Assert(is_written);
+  }
+
+  ProfEnd();
+}
+
+internal void
+lnk_background_file_writer_end_file(LNK_BackgroundFileWriter *writer, LNK_BackgroundFile *file, U64 expected_byte_count)
+{
+  ProfBegin("Background File Writer End File");
+
+  Assert(writer->is_running);
+  Assert(file->is_open && !file->is_finished);
+  file->is_finished = 1;
+
+  LNK_BackgroundFileWriteJob job = {
+    .kind                = LNK_BackgroundFileJobKind_EndFile,
+    .file                = file,
+    .expected_byte_count = expected_byte_count,
+  };
+  RingGuard guard      = guarded_ring_open(writer->queue);
+  B32       is_written = guarded_ring_write_struct_or_wait(&guard, &job, max_U64);
+  guarded_ring_close(&guard);
+  Assert(is_written);
+
+  ProfEnd();
+}
+
+internal void
+lnk_background_file_writer_end(LNK_BackgroundFileWriter *writer)
+{
+  ProfBegin("Background File Writer End");
+
+  if (writer->is_running) {
+    LNK_BackgroundFileWriteJob job        = { .kind = LNK_BackgroundFileJobKind_EndWriter };
+    RingGuard                  guard      = guarded_ring_open(writer->queue);
+    B32                        is_written = guarded_ring_write_struct_or_wait(&guard, &job, max_U64);
+    guarded_ring_close(&guard);
+    Assert(is_written);
+    thread_join(writer->thread, -1);
+  }
+
+  for EachNode(file, LNK_BackgroundFile, writer->file_first) {
+    if (file->is_open) {
+      lnk_error(LNK_Error_IO, "unfinished background write, file %S", file->path);
+      lnk_close_file(&file->file);
+      file->is_open = 0;
+    }
+  }
+
+  guarded_ring_release(writer->queue);
+  arena_release(writer->queue_arena);
+  writer->is_running = 0;
+
+  ProfEnd();
+}

src/linker/lnk_io.h

@@ -18,6 +18,23 @@ typedef struct
   U8          *buffer;
 } LNK_DiskReader;
 
+typedef struct
+{
+  String8List data;
+} LNK_FileArtifact;
+
+typedef struct LNK_BackgroundFile LNK_BackgroundFile;
+
+typedef struct
+{
+  Arena              *queue_arena;
+  GuardedRing        *queue;
+  Thread              thread;
+  LNK_BackgroundFile *file_first;
+  LNK_BackgroundFile *file_last;
+  B32                 is_running;
+} LNK_BackgroundFileWriter;
+
 // --- Shared File API ---------------------------------------------------------
 
 shared_function int      lnk_open_file_read(char *path, uint64_t path_size, void *handle_buffer, uint64_t handle_buffer_max);
@@ -38,4 +55,12 @@ internal String8Array lnk_read_data_from_file_path_parallel(TP_Context *tp, Aren
 
 internal void lnk_write_data_list_to_file_path(String8 path, String8 temp_path, String8List list);
 internal void lnk_write_data_to_file_path(String8 path, String8 temp_path, String8 data);
+internal String8 lnk_data_from_file_artifact(Arena *arena, LNK_FileArtifact *artifact);
 
+// --- Background Writer -------------------------------------------------------
+
+internal void lnk_background_file_writer_begin      (LNK_BackgroundFileWriter *writer);
+internal LNK_BackgroundFile *lnk_background_file_writer_begin_file(LNK_BackgroundFileWriter *writer, String8 path, String8 temp_path);
+internal void lnk_background_file_writer_enqueue    (LNK_BackgroundFileWriter *writer, LNK_BackgroundFile *file, U64 file_off, String8 data);
+internal void lnk_background_file_writer_end_file   (LNK_BackgroundFileWriter *writer, LNK_BackgroundFile *file, U64 expected_byte_count);
+internal void lnk_background_file_writer_end        (LNK_BackgroundFileWriter *writer);

src/linux/base/linux_base.c

@@ -364,6 +364,7 @@ set_platform_thread_name(String8 name)
 internal Thread
 thread_launch(ThreadEntryPointFunctionType *f, void *p)
 {
+  ProfBeginFunction();
   LNX_Entity *entity = lnx_entity_alloc(LNX_EntityKind_Thread);
   entity->thread.func = f;
   entity->thread.ptr = p;
@@ -376,6 +377,7 @@ thread_launch(ThreadEntryPointFunctionType *f, void *p)
     }
   }
   Thread handle = {(U64)entity};
+  ProfEnd();
   return handle;
 }
 

src/win32/base/win32_base.c

@@ -446,11 +446,13 @@ set_platform_thread_name(String8 name)
 internal Thread
 thread_launch(ThreadEntryPointFunctionType *f, void *p)
 {
+  ProfBeginFunction();
   W32_Entity *entity = w32_entity_alloc(W32_EntityKind_Thread);
   entity->thread.func = f;
   entity->thread.ptr = p;
   entity->thread.handle = CreateThread(0, 0, w32_thread_entry_point, entity, 0, &entity->thread.tid);
   Thread result = {IntFromPtr(entity)};
+  ProfEnd();
   return result;
 }