ed/raddebugger
1
0 Fork 0
mirror of https://github.com/Ed94/raddebugger.git synced 2026-11-14 02:03:27 -08:00
Code Issues Packages Projects Releases Wiki Activity

radlink: refactor MSF writer to allow allocation of contiguous stream backing

Browse Source 
* remove MSF reader API
* remove PDB reader API
This commit is contained in:
Nikita Smith committed 2026-09-21 14:00:07 -07:00
1 parent 98c1191506
commit 4add147a09
8 files changed
+653 -2400
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/linker/lnk_debug_info.c
+20 -60
View File
@@ -2697,17 +2697,18 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules)
if (mod->sn == MSF_INVALID_STREAM_NUMBER) { continue; }
CV_DebugS debug_s = task->cv->debug_s_arr[obj_idx];
String8List mod_data = msf_data_from_sn(temp.arena, task->pdb->msf, mod->sn);
String8 mod_data = msf_stream_data(task->pdb->msf, mod->sn);
if (mod_data.node_count) {
String8Node buf = *mod_data.first;
if (mod_data.size) {
String8Node buf = { .string = mod_data };
U64 pos = 0;
lnk_write_debug_s_to_pdb_module(mod, debug_s, &buf, &pos);
// sub range symbol data pages and patch symbol tree offsets
// Patch the contiguous symbol buffer before handing it to output.
if (mod->sym_data_size) {
Rng1U64 sym_data_range = r1u64(sizeof(CV_Signature), mod->sym_data_size);
String8List mod_symbols = str8_list_substr(temp.arena, mod_data, sym_data_range);
String8List mod_symbols = {0};
str8_list_push(temp.arena, &mod_symbols, str8_substr(mod_data, sym_data_range));
Assert(mod_symbols.total_size == dim_1u64(sym_data_range));
cv_patch_symbol_tree_offsets(mod_symbols, sizeof(CV_Signature), PDB_SYMBOL_ALIGN);
}
@@ -2754,50 +2755,38 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules)
if (mod->sn == MSF_INVALID_STREAM_NUMBER) { continue; }
String8List mod_data = msf_data_from_sn(temp.arena, task->pdb->msf, mod->sn);
String8 mod_data = msf_stream_data(task->pdb->msf, mod->sn);
Rng1U64 c13_data_range = r1u64(mod->sym_data_size + mod->c11_data_size, mod->sym_data_size + mod->c11_data_size + mod->c13_data_size);
String8List c13_data = str8_list_substr(temp.arena, mod_data, c13_data_range);
String8 c13_data = str8_substr(mod_data, c13_data_range);
CV_DebugS debug_s = task->cv->debug_s_arr[obj_idx];
String8 string_table = cv_string_table_from_debug_s(debug_s);
// checksum is always at the head of C13 data
String8List file_chksms_raw = {0};
String8 file_chksms_raw = {0};
{
String8Node buf = *c13_data.first;
U64 buf_pos = 0;
CV_C13SubSectionHeader header = {0};
str8_buffer_read(&buf, &buf_pos, sizeof(header), &header);
str8_deserial_read_struct(c13_data, 0, &header);
if (header.kind == CV_C13SubSectionKind_FileChksms) {
Rng1U64 file_chksms_range = r1u64(sizeof(CV_C13SubSectionHeader), sizeof(CV_C13SubSectionHeader) + header.size);
file_chksms_raw = str8_list_substr(temp.arena, c13_data, file_chksms_range);
file_chksms_raw = str8_substr(c13_data, file_chksms_range);
}
}
// fixup file name offsets in checksum headers
if (file_chksms_raw.total_size) {
String8Node buf = *file_chksms_raw.first;
U64 buf_pos = 0;
if (file_chksms_raw.size) {
U64 cursor = 0;
for (;;) {
CV_C13Checksum header = {0};
if (str8_buffer_peek(&buf, &buf_pos, sizeof(header), &header) != sizeof(header)) { break; }
if (str8_deserial_read_struct(file_chksms_raw, cursor, &header) != sizeof(header)) { break; }
String8 name = str8_cstring_capped(string_table.str + header.name_off, string_table.str + string_table.size);
CV_StringBucket *bucket = cv_string_hash_table_lookup(task->string_ht, name);
U64 name_off = task->string_table_base_offset + bucket->u.offset;
// update name offset
{
String8Node buf_copy = buf;
U64 buf_pos_copy = buf_pos;
str8_buffer_skip(&buf_copy, &buf_pos_copy, OffsetOf(CV_C13Checksum, name_off));
str8_buffer_write_u32(&buf_copy, &buf_pos_copy, safe_cast_u32(name_off));
}
str8_buffer_skip(&buf, &buf_pos, AlignPow2(sizeof(header) + header.len, CV_FileCheckSumsAlign));
memory_write32(file_chksms_raw.str + cursor + OffsetOf(CV_C13Checksum, name_off), safe_cast_u32(name_off));
cursor += AlignPow2(sizeof(header) + header.len, CV_FileCheckSumsAlign);
}
}
@@ -2889,29 +2878,6 @@ typedef struct
U64 sealed_stream_cap;
} LNK_PdbOutput;
typedef struct LNK_MsfPageCursor
{
MSF_PageDataNode *node;
U64 node_idx;
U64 pages_per_node;
} LNK_MsfPageCursor;
internal U8 *
lnk_msf_data_from_pn(LNK_MsfPageCursor *cursor, MSF_Context *msf, MSF_PageNumber pn)
{
U64 node_idx = pn / cursor->pages_per_node;
if (node_idx < cursor->node_idx) {
cursor->node = msf->page_data_list.first;
cursor->node_idx = 0;
}
while (cursor->node_idx < node_idx) {
cursor->node = cursor->node->next;
cursor->node_idx += 1;
}
Assert(cursor->node != 0);
return cursor->node->data + (pn % cursor->pages_per_node) * msf->page_size;
}
internal void
lnk_pdb_output_enqueue_stream(LNK_PdbOutput *output, MSF_Context *msf, MSF_StreamNumber sn)
{
@@ -2923,16 +2889,14 @@ lnk_pdb_output_enqueue_stream(LNK_PdbOutput *output, MSF_Context *msf, MSF_Strea
Assert(stream != 0);
if (stream->page_list.count == 0) { return; }
LNK_MsfPageCursor cursor = {
.node = msf->page_data_list.first,
.pages_per_node = msf_get_data_node_size(msf->page_size) / msf->page_size,
};
U64 stream_offset = 0;
MSF_PageNumber run_first_pn = 0;
MSF_PageNumber run_last_pn = 0;
U8 *run_data = 0;
U64 run_size = 0;
for EachNode(page, MSF_PageNode, stream->page_list.first) {
U8 *page_data = lnk_msf_data_from_pn(&cursor, msf, page->pn);
U8 *page_data = stream->data + stream_offset;
stream_offset += msf->page_size;
if (run_data != 0 && page->pn == run_last_pn + 1 && page_data == run_data + run_size) {
run_last_pn = page->pn;
run_size += msf->page_size;
@@ -2973,15 +2937,11 @@ lnk_pdb_output_enqueue_remaining(LNK_PdbOutput *output, MSF_Context *msf)
}
}
LNK_MsfPageCursor cursor = {
.node = msf->page_data_list.first,
.pages_per_node = msf_get_data_node_size(msf->page_size) / msf->page_size,
};
U64 run_first_pn = 0;
U8 *run_data = 0;
U64 run_size = 0;
for EachIndex(pn, page_count) {
U8 *page_data = lnk_msf_data_from_pn(&cursor, msf, pn);
U8 *page_data = msf_data_from_pn(msf, pn).str;
U64 page_size = Min(msf->page_size, save_size - pn * msf->page_size);
if (!is_written[pn]) {
if (run_data != 0 && page_data == run_data + run_size) {
@@ -3183,7 +3143,7 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config
pdb_build_dbi_info(tp, task.pdb, task.string_ht, 0, cv->is_stripped, &build_hooks);
MSF_Error msf_err = msf_build(task.pdb->msf);
if (msf_err != MSF_Error_OK) {
if (msf_err != MSF_Error_Ok) {
lnk_error(LNK_Error_UnableToSerializeMsf, "unable to serialize MSF: %s", msf_error_to_string(msf_err));
}
src/linker/pdb_ext/msf_builder.c
+373 -1783
View File
File diff suppressed because it is too large. Load diff
src/linker/pdb_ext/msf_builder.h
+74 -131
View File
@@ -5,25 +5,18 @@
#define MSF_PAGE_STATE_FREE 1
#define MSF_PAGE_STATE_ALLOC 0
#define MSF_FPM0 1
#define MSF_FPM1 2
#define MSF_DEFAULT_PAGE_SIZE 4096
#define MSF_DEFAULT_FPM MSF_FPM0
typedef struct MSF_PageNumberArray
typedef struct MSF_PageNode MSF_PageNode;
struct MSF_PageNode
{
U64 count;
MSF_PageNumber *v;
} MSF_PageNumberArray;
typedef struct MSF_PageNode
{
struct MSF_PageNode *next;
struct MSF_PageNode *prev;
MSF_PageNumber pn;
} MSF_PageNode;
MSF_PageNode *next;
MSF_PageNode *prev;
MSF_PageNumber pn;
};
typedef struct MSF_PageList
{
@@ -34,19 +27,21 @@ typedef struct MSF_PageList
typedef struct MSF_Stream
{
MSF_StreamNumber sn;
MSF_UInt size;
MSF_UInt pos;
MSF_PageNode *pos_page;
MSF_PageList page_list;
MSF_StreamNumber sn;
MSF_UInt size;
MSF_UInt pos;
U64 capacity;
U8 *data;
MSF_PageList page_list;
} MSF_Stream;
typedef struct MSF_StreamNode
typedef struct MSF_StreamNode MSF_StreamNode;
struct MSF_StreamNode
{
struct MSF_StreamNode *next;
struct MSF_StreamNode *prev;
MSF_Stream data;
} MSF_StreamNode;
MSF_StreamNode *next;
MSF_StreamNode *prev;
MSF_Stream data;
};
typedef struct MSF_StreamList
{
@@ -55,125 +50,84 @@ typedef struct MSF_StreamList
MSF_StreamNode *last;
} MSF_StreamList;
typedef struct MSF_PageDataNode
typedef struct MSF_PageBlock MSF_PageBlock;
struct MSF_PageBlock
{
struct MSF_PageDataNode *next;
struct MSF_PageDataNode *prev;
U8 *data;
} MSF_PageDataNode;
typedef struct MSF_PageDataList
{
MSF_PageDataNode *first;
MSF_PageDataNode *last;
MSF_UInt count;
} MSF_PageDataList;
MSF_PageBlock *next;
U32 *free_bits;
U8 *fpm_data;
U8 **data;
};
typedef struct MSF_Context
{
Arena *arena;
MSF_UInt page_size;
MSF_UInt active_fpm;
MSF_UInt fpm_rover;
MSF_PageNumber page_count;
MSF_PageDataList page_data_list;
MSF_PageDataList page_data_pool;
MSF_PageList header_page_list;
MSF_PageList root_page_list;
MSF_PageList st_page_list;
MSF_PageList page_pool;
MSF_StreamList sectab;
Arena *arena;
MSF_UInt page_size;
MSF_UInt active_fpm;
MSF_PageNumber fpm_rover;
MSF_UInt block_count;
MSF_PageBlock *first_block;
MSF_PageBlock *last_block;
U8 *zero_page;
U8 *empty_fpm;
MSF_PageList page_pool;
MSF_Stream header;
MSF_Stream root;
MSF_Stream stream_table;
MSF_StreamList sectab;
} MSF_Context;
typedef enum MSF_Error
{
MSF_Error_OK,
// if you get this error this means stream table was divided into too many
// pages, and to fix this you need to bump up the page size
MSF_Error_STREAM_TABLE_HAS_TOO_MANY_PAGES,
MSF_OpenError_NOT_ENOUGH_BYTES_TO_READ_HEADER,
MSF_OpenError_INVALID_MAGIC,
MSF_OpenError_PAGE_SIZE_IS_NOT_POW2,
MSF_OpenError_INVALID_PAGE_SIZE,
MSF_OpenError_NOT_ENOUGH_PAGES_TO_INIT,
MSF_OpenError_INVALID_ROOT_STREAM_PAGE_NUMBER,
MSF_OpenError_UNABLE_TO_READ_STREAM_TABLE_PAGE_NUMBERS,
MSF_OpenError_STREAM_COUNT_OVERFLOW,
MSF_OpenError_UNABLE_TO_READ_STREAM_SIZES,
MSF_OpenError_INVALID_STREAM_TABLE,
MSF_OpenError_INVALID_ACTIVE_FPM,
MSF_OpenError_PAGE_COUNT_DOESNT_MATCH_DATA_SIZE,
MSF_BuildError_UNABLE_TO_WRITE_STREAM_TABLE,
MSF_BuildError_UNABLE_TO_WRITE_STREAM_TABLE_PAGE_NUMBER_DIRECTORY,
MSF_BuildError_UNABLE_TO_WRITE_ROOT_DIRECTORY,
MSF_BuildError_UNABLE_TO_WRITE_HEADER,
MSF_Error_Ok,
MSF_Error_StreamTableHasTooManyPages,
MSF_Error_FailedToWriteStreamTable,
MSF_Error_FailedToWriteHeader,
} MSF_Error;
////////////////////////////////
typedef struct
typedef struct MSF_WriteTask
{
MSF_UInt page_size;
MSF_PageDataList page_data_list;
MSF_PageList page_list;
MSF_UInt stream_pos;
String8 data;
Rng1U64 *range_arr;
U8 *dst;
U8 *src;
Rng1U64 *ranges;
} MSF_WriteTask;
////////////////////////////////
internal MSF_Context *msf_alloc(MSF_UInt page_size, MSF_UInt active_fpm);
internal void msf_release(MSF_Context *msf);
internal MSF_Error msf_build(MSF_Context *msf);
internal U64 msf_get_save_size(MSF_Context *msf);
internal MSF_Context * msf_alloc(MSF_UInt page_size, MSF_UInt active_fpm);
internal void msf_release(MSF_Context *msf_ptr);
internal MSF_Error msf_build(MSF_Context *msf);
internal U64 msf_get_save_size(MSF_Context *msf);
internal String8List msf_get_page_data_nodes(Arena *arena, MSF_Context *msf);
internal B32 msf_save(MSF_Context *msf, void *buffer, U64 buffer_size);
internal MSF_Error msf_save_arena(Arena *arena, MSF_Context *msf, String8 *data_out);
internal MSF_StreamNode * msf_find_stream_node(MSF_Context *msf, MSF_StreamNumber sn);
internal MSF_Stream * msf_find_stream(MSF_Context *msf, MSF_StreamNumber sn);
internal B32 msf_grow(MSF_Context *msf, MSF_PageNumber page_count);
internal MSF_PageNumber * msf_alloc_pn_arr(Arena *arena, MSF_Context *msf, MSF_UInt alloc_count);
internal void msf_free_pn_arr(MSF_Context *msf, MSF_PageNumber *pn_arr, MSF_UInt pn_count);
internal MSF_PageList msf_alloc_pages(MSF_Context *msf, MSF_UInt alloc_count);
internal void msf_free_pages(MSF_Context *msf, MSF_PageList *page_list);
// File-order spans borrow stream/metadata storage and remain valid until it
// is modified or released. No second copy of the stream payloads is made.
internal String8List msf_get_page_data_nodes(Arena *arena, MSF_Context *msf);
internal String8 msf_data_from_pn(MSF_Context *msf, MSF_PageNumber pn);
internal B32 msf_save(MSF_Context *msf, void *buffer, U64 buffer_size);
internal MSF_Error msf_save_arena(Arena *arena, MSF_Context *msf, String8 *data_out);
internal MSF_StreamNode *msf_find_stream_node(MSF_Context *msf, MSF_StreamNumber sn);
internal MSF_Stream *msf_find_stream(MSF_Context *msf, MSF_StreamNumber sn);
internal MSF_PageList msf_alloc_pages(MSF_Context *msf, MSF_UInt count);
internal void msf_free_pages(MSF_Context *msf, MSF_PageList *pages);
internal MSF_StreamNumber msf_stream_alloc_ex(MSF_Context *msf, MSF_UInt size);
internal MSF_StreamNumber msf_stream_alloc(MSF_Context *msf);
internal B32 msf_stream_free(MSF_Context *msf, MSF_StreamNumber sn);
internal B32 msf_stream_resize(MSF_Context *msf, MSF_StreamNumber sn, MSF_UInt new_size);
internal B32 msf_stream_resize(MSF_Context *msf, MSF_StreamNumber sn, MSF_UInt size);
internal B32 msf_stream_resize_ex(MSF_Context *msf, MSF_Stream *stream, MSF_UInt size);
internal void msf_stream_set_size(MSF_Context *msf, MSF_StreamNumber sn, MSF_UInt size);
internal MSF_UInt msf_stream_get_size(MSF_Context *msf, MSF_StreamNumber sn);
internal MSF_UInt msf_stream_get_cap(MSF_Context *msf, MSF_StreamNumber);
internal MSF_UInt msf_stream_get_pos(MSF_Context *msf, MSF_StreamNumber sn);
internal void msf_stream_align(MSF_Context *msf, MSF_StreamNumber sn, MSF_UInt align);
internal B32 msf_stream_reserve(MSF_Context *msf, MSF_StreamNumber sn, MSF_UInt size);
internal B32 msf_stream_seek(MSF_Context *msf, MSF_StreamNumber sn, MSF_UInt new_pos);
internal B32 msf_stream_seek(MSF_Context *msf, MSF_StreamNumber sn, MSF_UInt pos);
internal B32 msf_stream_seek_start(MSF_Context *msf, MSF_StreamNumber sn);
internal B32 msf_stream_seek_end(MSF_Context *msf, MSF_StreamNumber sn);
internal void msf_stream_align(MSF_Context *msf, MSF_StreamNumber sn, MSF_UInt align);
internal MSF_UInt msf_stream_read(MSF_Context *msf, MSF_StreamNumber sn, void *dst, MSF_UInt dst_len);
internal String8 msf_stream_read_block(Arena *arena, MSF_Context *msf, MSF_StreamNumber sn, U64 block_size);
internal String8 msf_stream_read_string(Arena *arena, MSF_Context *msf, MSF_StreamNumber sn);
internal S8 msf_stream_read_s8(MSF_Context *msf, MSF_StreamNumber sn);
internal S16 msf_stream_read_s16(MSF_Context *msf, MSF_StreamNumber sn);
internal S32 msf_stream_read_s32(MSF_Context *msf, MSF_StreamNumber sn);
internal S64 msf_stream_read_s64(MSF_Context *msf, MSF_StreamNumber sn);
internal U8 msf_stream_read_u8(MSF_Context *msf, MSF_StreamNumber sn);
internal U16 msf_stream_read_u16(MSF_Context *msf, MSF_StreamNumber sn);
internal U32 msf_stream_read_u32(MSF_Context *msf, MSF_StreamNumber sn);
internal U64 msf_stream_read_u64(MSF_Context *msf, MSF_StreamNumber sn);
#define msf_stream_read_array(msf, sn, ptr, count) msf_stream_read(msf, sn, ptr, sizeof(*ptr) * (count))
#define msf_stream_read_struct(msf, sn, ptr) msf_stream_read_array(msf, sn, ptr, 1)
internal String8 msf_stream_data(MSF_Context *msf, MSF_StreamNumber sn);
// Writable, contiguous construction storage. Reserve/resize before taking a
// view: growth beyond capacity invalidates views, but preserves page numbers.
// Finalized streams must not be changed while asynchronous output borrows them.
internal B32 msf_stream_write(MSF_Context *msf, MSF_StreamNumber sn, void *buffer, MSF_UInt buffer_size);
internal B32 msf_stream_write(MSF_Context *msf, MSF_StreamNumber sn, void *buffer, MSF_UInt size);
internal B32 msf_stream_write_string(MSF_Context *msf, MSF_StreamNumber sn, String8 string);
internal B32 msf_stream_write_list(MSF_Context *msf, MSF_StreamNumber sn, String8List list);
internal B32 msf_stream_write_uint(MSF_Context *msf, MSF_StreamNumber sn, MSF_UInt value);
internal B32 msf_stream_write_cstr(MSF_Context *msf, MSF_StreamNumber sn, String8 string);
internal B32 msf_stream_write_u8(MSF_Context *msf, MSF_StreamNumber sn, U8 value);
internal B32 msf_stream_write_u16(MSF_Context *msf, MSF_StreamNumber sn, U16 value);
@@ -183,21 +137,10 @@ internal B32 msf_stream_write_s8(MSF_Context *msf, MSF_StreamNumber sn, S8 value
internal B32 msf_stream_write_s16(MSF_Context *msf, MSF_StreamNumber sn, S16 value);
internal B32 msf_stream_write_s32(MSF_Context *msf, MSF_StreamNumber sn, S32 value);
internal B32 msf_stream_write_s64(MSF_Context *msf, MSF_StreamNumber sn, S64 value);
internal B32 msf_stream_write_parallel(TP_Context *tp, MSF_Context *msf, MSF_StreamNumber sn, void *buffer, MSF_UInt buffer_size);
#define msf_stream_write_array(m, s, v, c) msf_stream_write(m, s, (void*)(v), sizeof(*(v)) * (c))
#define msf_stream_write_struct(m, s, v ) msf_stream_write_array(m, s, v, 1)
internal B32 msf_stream_write_parallel(TP_Context *tp, MSF_Context *msf, MSF_StreamNumber sn, void *buffer, MSF_UInt size);
#define msf_stream_write_array(m, s, v, c) msf_stream_write(m, s, (void *)(v), sizeof(*(v)) * (c))
#define msf_stream_write_struct(m, s, v) msf_stream_write_array(m, s, v, 1)
internal String8List msf_data_from_sn(Arena *arena, MSF_Context *msf, MSF_StreamNumber sn);
internal MSF_UInt msf_count_pages(MSF_UInt page_size, U64 data_size);
internal MSF_PageNumber msf_get_page_count_cap(MSF_PageDataList page_data_list, MSF_UInt page_size);
internal MSF_UInt msf_get_fpm_interval_correct(MSF_UInt page_size);
internal MSF_UInt msf_get_fpm_interval_wrong(MSF_UInt page_size);
internal MSF_UInt msf_get_fpm_idx_from_pn(MSF_UInt page_size, MSF_PageNumber pn);
internal MSF_UInt msf_get_fpm_page_bit_state(MSF_PageDataList page_data_list, MSF_UInt page_size, MSF_PageNumber active_fpm, MSF_PageNumber pn);
internal void msf_set_fpm_bit(MSF_PageDataList page_data_list, MSF_UInt page_size, MSF_PageNumber active_fpm, MSF_PageNumber pn, B32 state);
internal B32 msf_write(MSF_PageDataList page_data_list, MSF_UInt page_size, MSF_PageList page_list, MSF_UInt offset, void *buffer, MSF_UInt buffer_size);
internal MSF_UInt msf_read(MSF_PageDataList page_data_list, MSF_UInt page_size, MSF_PageList page_list, MSF_UInt offset, void *buffer, MSF_UInt buffer_size);
internal char * msf_error_to_string(MSF_Error code);
internal MSF_UInt msf_count_pages(MSF_UInt page_size, U64 data_size);
internal char *msf_error_to_string(MSF_Error code);
src/linker/pdb_ext/pdb_builder.c
+65 -360
View File
@@ -594,6 +594,7 @@ pdb_strtab_build(PDB_StringTable *strtab, MSF_Context *msf, MSF_StreamNumber sn)
MSF_UInt reserve_size = sizeof(header)
+ sizeof(strtab->size)
+ strtab->size
+ sizeof(strtab->bucket_max)
+ sizeof(bucket_offset_arr[0]) * strtab->bucket_max
+ sizeof(strtab->bucket_count);
msf_stream_reserve(msf, sn, reserve_size);
@@ -890,11 +891,32 @@ pdb_type_hash_stream_build(TP_Context *tp,
ProfBeginFunction();
Temp scratch = scratch_begin(0,0);
String8 hash_adj_data = {0};
if (ts->hash_adj.count) {
hash_adj_data = pdb_data_from_hash_adj_hash_table(scratch.arena, &ts->hash_adj, strtab);
}
PDB_OffsetSize hash_vals = { .off = 0, .size = sizeof(U32) * ts->leaf_list.node_count };
PDB_OffsetSize hint_offs = { .off = hash_vals.size, .size = sizeof(hint_arr[0]) * hint_count };
PDB_OffsetSize hash_adj = {0};
if (ts->hash_adj.count) {
hash_adj.off = hint_offs.off + hint_offs.size;
hash_adj.size = hash_adj_data.size;
}
U64 stream_size = hash_vals.size + hint_offs.size + hash_adj_data.size;
if (ts->hash_sn == MSF_INVALID_STREAM_NUMBER) { ts->hash_sn = msf_stream_alloc(msf); }
AssertAlways(msf_stream_resize(msf, ts->hash_sn, safe_cast_u32(stream_size)));
String8 stream_data = msf_stream_data(msf, ts->hash_sn);
// write (type index -> bucket index) map
//
// zero-out entire map so non-UDTs type indices, that are NOT in the hash table,
// map to zero offset
U32 *type_to_bucket_map = push_array(scratch.arena, U32, ts->leaf_list.node_count);
U32 *type_to_bucket_map = (U32 *)stream_data.str;
if (hash_vals.size != 0) { MemoryZero(type_to_bucket_map, hash_vals.size); }
{
ProfBegin("Bucket Map");
PDB_WriteTypeToBucketMap type_to_bucket_task;
@@ -904,33 +926,11 @@ pdb_type_hash_stream_build(TP_Context *tp,
ProfEnd();
}
ProfBegin("MSF Write");
// write data to stream
if (ts->hash_sn == MSF_INVALID_STREAM_NUMBER) {
ts->hash_sn = msf_stream_alloc(msf);
}
msf_stream_seek_start(msf, ts->hash_sn);
PDB_OffsetSize hash_vals;
hash_vals.off = msf_stream_get_pos(msf, ts->hash_sn);
hash_vals.size = sizeof(type_to_bucket_map[0]) * ts->leaf_list.node_count;
msf_stream_write(msf, ts->hash_sn, &type_to_bucket_map[0], hash_vals.size);
PDB_OffsetSize hint_offs;
hint_offs.off = msf_stream_get_pos(msf, ts->hash_sn);
hint_offs.size = sizeof(hint_arr[0]) * hint_count;
msf_stream_seek(msf, ts->hash_sn, hint_offs.off);
msf_stream_write(msf, ts->hash_sn, &hint_arr[0], hint_offs.size);
PDB_OffsetSize hash_adj = {0};
if (ts->hash_adj.count) {
// write bucket adjust info
String8 hash_adj_data = pdb_data_from_hash_adj_hash_table(scratch.arena, &ts->hash_adj, strtab);
hash_adj.off = msf_stream_get_pos(msf, ts->hash_sn);
hash_adj.size = hash_adj_data.size;
msf_stream_write_string(msf, ts->hash_sn, hash_adj_data);
}
msf_stream_write_string(msf, ts->hash_sn, hash_adj_data);
ProfEnd();
@@ -990,7 +990,7 @@ pdb_type_server_build(TP_Context *tp, PDB_TypeServer *ts, PDB_StringTable *strta
U64 lf_buf_size = 0;
U64 lf_node_idx = 0;
U64 lf_arr_idx = 0;
for (String8Node *lf = ts->leaf_list.first; lf != 0; lf = lf->next) {
for EachNode(lf, String8Node, ts->leaf_list.first) {
if (lf_node_idx == lf_range_arr[lf_arr_idx].min) { // :thread_pool_dummy_range
lf_cursor_arr[lf_arr_idx] = lf_buf_size;
lf_arr[lf_arr_idx] = lf;
@@ -1012,7 +1012,8 @@ pdb_type_server_build(TP_Context *tp, PDB_TypeServer *ts, PDB_StringTable *strta
write_types_task.lf_arr = lf_arr;
write_types_task.lf_range_arr = lf_range_arr;
write_types_task.lf_cursor_arr = lf_cursor_arr;
write_types_task.lf_buf = push_array_no_zero(scratch.arena, U8, lf_buf_size);
AssertAlways(msf_stream_resize(msf, sn, safe_cast_u32(sizeof(PDB_TpiHeader) + lf_buf_size)));
write_types_task.lf_buf = msf_stream_data(msf, sn).str + sizeof(PDB_TpiHeader);
write_types_task.lf_buf_size = lf_buf_size;
tp_for_parallel(tp, 0, tp->worker_count, pdb_write_types_task, &write_types_task);
@@ -1040,7 +1041,7 @@ pdb_type_server_build(TP_Context *tp, PDB_TypeServer *ts, PDB_StringTable *strta
ProfBegin("MSF Commit");
msf_stream_seek_start(msf, sn);
msf_stream_write_struct(msf, sn, &header);
msf_stream_write_parallel(tp, msf, sn, write_types_task.lf_buf, lf_buf_size);
msf_stream_seek(msf, sn, sizeof(header) + lf_buf_size);
ProfEnd();
scratch_end(scratch);
@@ -1278,135 +1279,6 @@ pdb_type_server_push_parallel(TP_Context *tp, PDB_TypeServer *type_server, U64 l
ProfEnd();
}
#if 0
internal CV_LeafNode *
pdb_type_server_leaf_from_string(PDB_TypeServer *ts, String8 string)
{
ProfBeginFunction();
U32 hash = pdb_hash_v1(string);
U32 bucket_idx = hash % ts->bucket_count;
PDB_TypeBucket *head_bucket = ts->bucket_table[bucket_idx];
CV_LeafNode *result = 0;
for (PDB_TypeBucket *i = head_bucket; i != 0; i = i->next) {
CV_LeafNode *leaf = i->leaf_node;
String8 leaf_name = cv_get_leaf_name(leaf->data.kind, leaf->data.data);
if (str8_match(leaf_name, string, 0)) {
result = leaf;
break;
}
}
ProfEnd();
return result;
}
#endif
////////////////////////////////
#if 0
internal PDB_TypeIndexMap *
pdb_load_types_from_leaf_list(PDB_TypeServer **type_server_arr, CV_LeafList leaf_list)
{
ProfBeginFunction();
Temp scratch = scratch_begin(0, 0);
// 1. redistribute leaves in parallel
CV_LeafList leaf_list_arr[CV_TypeIndexSource_COUNT] = {0};
for (CV_LeafNode *curr = leaf_list.first, *next = 0; curr != 0; curr = next) {
next = curr->next;
curr->next = 0;
CV_TypeIndexSource ti_source = cv_type_index_source_from_leaf_kind(curr->data.kind);
CV_LeafList *list = &leaf_list_arr[ti_source];
SLLQueuePush(list->first, list->last, curr);
list->count += 1;
}
// 2. reserve type leafs on main thread
PDB_TypeLeaf *leaf_arr_arr[CV_TypeIndexSource_COUNT];
for (U64 source_idx = 0; source_idx < ArrayCount(leaf_list_arr); source_idx += 1) {
PDB_TypeServer *type_server = type_server_arr[source_idx];
CV_LeafList input_leaf_list = leaf_list_arr[source_idx];
PDB_TypeLeaf *leaf_arr = pdb_type_server_reserve(type_server, input_leaf_list.count);
leaf_arr_arr[source_idx] = leaf_arr;
}
// 3. populate type index map in parallel
PDB_TypeIndexMap *ti_map = pdb_type_index_map_alloc();
for (U64 source_idx = 0; source_idx < ArrayCount(leaf_list_arr); source_idx += 1) {
CV_LeafList input_leaf_list = leaf_list_arr[source_idx];
PDB_TypeLeaf *leaf_arr = leaf_arr_arr[source_idx];
for (U64 leaf_idx = 0; leaf_idx < input_leaf_list.count; leaf_idx += 1) {
CV_TypeIndex external_ti = ti_map->min_itype[source_idx] + leaf_idx;
CV_TypeIndex internal_ti = leaf_arr[leaf_idx].type_index;
pdb_type_index_map_add(ti_map, (CV_TypeIndexSource)source_idx, external_ti, internal_ti);
}
}
// 4. patch type indices in parallel
for (U64 source_idx = 0; source_idx < ArrayCount(leaf_list_arr); source_idx += 1) {
CV_LeafList list = leaf_list_arr[source_idx];
for (CV_LeafNode *node = list.first; node != 0; node = node->next) {
Temp temp = temp_begin(scratch.arena);
// get offsets for type indices in data blob
CV_Leaf *leaf = &node->data;
CV_TypeIndexInfoList ti_info_list = cv_get_leaf_type_index_offsets(temp.arena, leaf->kind, leaf->data);
for (CV_TypeIndexInfo *ti_info = ti_info_list.first; ti_info != 0; ti_info = ti_info->next) {
Assert(ti_info->offset + sizeof(CV_TypeIndex) <= leaf->data.size);
CV_TypeIndex *ti_ptr = (CV_TypeIndex *)(leaf->data.str + ti_info->offset);
CV_TypeIndex external_ti = *ti_ptr;
B32 is_complex_type = external_ti >= ti_map->min_itype[ti_info->source];
if (is_complex_type) {
// search external type index
CV_TypeIndex internal_tpi_idx = pdb_type_index_map_search(ti_map, CV_TypeIndexSource_TPI, external_ti);
CV_TypeIndex internal_ipi_idx = pdb_type_index_map_search(ti_map, CV_TypeIndexSource_IPI, external_ti);
// error checks
if (internal_tpi_idx == 0 && internal_ipi_idx == 0) {
lnk_invalid_path("unable to find match for external type index 0x%X", external_ti);
continue;
}
if (internal_tpi_idx != 0 && internal_ipi_idx != 0) {
lnk_invalid_path("both TPI and IPI matched for external type index 0x%X", external_ti);
continue;
}
// rewrite index
CV_TypeIndex internal_ti = internal_tpi_idx ? internal_tpi_idx : internal_ipi_idx;
*ti_ptr = internal_ti;
}
}
temp_end(temp);
}
}
// 5. push types to hash table on main thread
for (U64 source_idx = 0; source_idx < ArrayCount(leaf_list_arr); source_idx += 1) {
PDB_TypeServer *type_server = type_server_arr[source_idx];
CV_LeafList list = leaf_list_arr[source_idx];
PDB_TypeLeaf *leaf_arr = leaf_arr_arr[source_idx];
U64 leaf_idx = 0;
for (CV_LeafNode *node = list.first; node != 0; node = node->next, leaf_idx += 1) {
CV_Leaf *external_leaf = &node->data;
// move patched type data
PDB_TypeLeaf *internal_leaf = leaf_arr + leaf_idx;
internal_leaf->kind = external_leaf->kind;
internal_leaf->data = push_str8_copy(type_server->arena, external_leaf->data);
// push leaf to type server
pdb_type_server_push_(type_server, internal_leaf);
}
}
scratch_end(scratch);
ProfEnd();
return ti_map;
}
#endif
////////////////////////////////
internal PDB_InfoContext *
@@ -1491,6 +1363,7 @@ pdb_info_build_src_header_block(PDB_InfoContext *info, MSF_Context *msf)
MemoryZeroStruct(&src_header.pad);
// write to stream
msf_stream_reserve(msf, src_header_block_sn, safe_cast_u32(src_header_stream_size));
B32 is_header_written = msf_stream_write_struct(msf, src_header_block_sn, &src_header);
B32 is_hash_table_written = msf_stream_write_string(msf, src_header_block_sn, hash_table_data);
AssertAlways(is_header_written);
@@ -1710,8 +1583,7 @@ internal void
gsi_write_build_result(TP_Context *tp,
PDB_GsiBuildResult build,
MSF_Context *msf,
MSF_StreamNumber gsi_sn,
MSF_StreamNumber symbols_sn)
MSF_StreamNumber gsi_sn)
{
ProfBeginFunction();
@@ -1724,10 +1596,6 @@ gsi_write_build_result(TP_Context *tp,
msf_stream_reserve(msf, gsi_sn, gsi_size);
ProfEnd();
ProfBeginV("Reserve %M for symbols", build.symbol_data.size);
msf_stream_reserve(msf, symbols_sn, build.symbol_data.size);
ProfEnd();
ProfBegin("Write GSI header");
msf_stream_write_struct(msf, gsi_sn, &build.header);
ProfEnd();
@@ -1744,10 +1612,6 @@ gsi_write_build_result(TP_Context *tp,
msf_stream_write(msf, gsi_sn, &build.compressed_bucket_arr[0], compressed_bucket_arr_size);
ProfEnd();
ProfBegin("Write symbols [%M]", build.symbol_data.size);
msf_stream_write_string_parallel(tp, msf, symbols_sn, build.symbol_data);
ProfEnd();
ProfEnd();
}
@@ -2242,7 +2106,7 @@ THREAD_POOL_TASK_FUNC(gsi_serialize_symbols_task)
}
internal PDB_GsiBuildResult
gsi_build_ex(TP_Context *tp, Arena *arena, PDB_GsiContext *gsi, U64 symbol_data_base, B32 is_pub32, U64 msf_page_size)
gsi_build_ex(TP_Context *tp, Arena *arena, PDB_GsiContext *gsi, MSF_Context *msf, MSF_StreamNumber symbols_sn, B32 is_pub32)
{
ProfBeginFunction();
Temp scratch = scratch_begin(&arena,1);
@@ -2250,6 +2114,7 @@ gsi_build_ex(TP_Context *tp, Arena *arena, PDB_GsiContext *gsi, U64 symbol_data_
ProfBegin("Serialize & Sort Symbols");
PDB_GsiSerializeSymbolsTask serial_task = {0};
U64 symbol_data_base = msf_stream_get_pos(msf, symbols_sn);
serial_task.symbol_data_base = symbol_data_base;
serial_task.symbol_align = gsi->symbol_align;
serial_task.bucket_arr = gsi->bucket_arr;
@@ -2260,7 +2125,11 @@ gsi_build_ex(TP_Context *tp, Arena *arena, PDB_GsiContext *gsi, U64 symbol_data_
// prepare serial buffer
U64 buffer_size = sum_array_u64(gsi->bucket_count, serial_task.bucket_size_arr);
serial_task.buffer = push_array_no_zero(arena, U8, buffer_size);
U64 symbol_data_end = symbol_data_base + buffer_size;
if (symbol_data_end > msf_stream_get_size(msf, symbols_sn)) {
AssertAlways(msf_stream_resize(msf, symbols_sn, safe_cast_u32(symbol_data_end)));
}
serial_task.buffer = msf_stream_data(msf, symbols_sn).str + symbol_data_base;
serial_task.bucket_off_arr = push_array_copy_u64(scratch.arena, serial_task.bucket_size_arr, gsi->bucket_count);
u64_array_counts_to_offsets(gsi->bucket_count, serial_task.bucket_off_arr);
@@ -2275,6 +2144,7 @@ gsi_build_ex(TP_Context *tp, Arena *arena, PDB_GsiContext *gsi, U64 symbol_data_
// fill out sort records & serialize symbols
TP_TaskFunc *serial_func = is_pub32 ? gsi_serialize_pub32 : gsi_serialize_symbols_task;
tp_for_parallel(tp, 0, gsi->bucket_count, serial_func, &serial_task);
msf_stream_seek(msf, symbols_sn, safe_cast_u32(symbol_data_end));
ProfEnd();
@@ -2325,7 +2195,6 @@ gsi_build_ex(TP_Context *tp, Arena *arena, PDB_GsiContext *gsi, U64 symbol_data_
result.compressed_bucket_count = compressed_offset_count;
result.compressed_bucket_arr = compressed_offset_arr;
result.total_hash_size = sizeof(header) + header.hash_record_arr_size + header.bucket_data_size;
result.symbol_data = str8(serial_task.buffer, buffer_size);
scratch_end(scratch);
ProfEnd();
@@ -2338,9 +2207,8 @@ gsi_build(TP_Context *tp, PDB_GsiContext *gsi, MSF_Context *msf, MSF_StreamNumbe
ProfBeginFunction();
Temp scratch = scratch_begin(0,0);
U64 symbol_data_base = msf_stream_get_pos(msf, symbols_sn);
PDB_GsiBuildResult build = gsi_build_ex(tp, scratch.arena, gsi, symbol_data_base, /* is_pub32: */ 0, msf->page_size);
gsi_write_build_result(tp, build, msf, sn, symbols_sn);
PDB_GsiBuildResult build = gsi_build_ex(tp, scratch.arena, gsi, msf, symbols_sn, /* is_pub32: */ 0);
gsi_write_build_result(tp, build, msf, sn);
scratch_end(scratch);
ProfEnd();
@@ -2471,8 +2339,7 @@ psi_build(TP_Context *tp, PDB_PsiContext *psi, MSF_Context *msf, MSF_StreamNumbe
ProfBeginFunction();
Temp scratch = scratch_begin(0,0);
U64 symbol_data_base = msf_stream_get_pos(msf, symbols_sn);
PDB_GsiBuildResult gsi_build = gsi_build_ex(tp, scratch.arena, psi->gsi, symbol_data_base, /* is_pub32: */ 1, msf->page_size);
PDB_GsiBuildResult gsi_build = gsi_build_ex(tp, scratch.arena, psi->gsi, msf, symbols_sn, /* is_pub32: */ 1);
ProfBegin("Address Map");
@@ -2493,8 +2360,9 @@ psi_build(TP_Context *tp, PDB_PsiContext *psi, MSF_Context *msf, MSF_StreamNumbe
header.sec_count = 0;
ProfBegin("MSF Write");
msf_stream_reserve(msf, sn, safe_cast_u32(sizeof(header) + gsi_build.total_hash_size + addr_map_size));
msf_stream_write_struct(msf, sn, &header);
gsi_write_build_result(tp, gsi_build, msf, sn, symbols_sn);
gsi_write_build_result(tp, gsi_build, msf, sn);
msf_stream_write_array(msf, sn, &addr_map[0], addr_map_count);
ProfEnd();
@@ -2554,149 +2422,6 @@ dbi_alloc(COFF_MachineType machine, U32 age)
return dbi;
}
internal String8List *
dbi_open_file_info(Arena *arena, MSF_Context *msf, MSF_StreamNumber sn, PDB_DbiHeader *dbi_header)
{
ProfBeginFunction();
Temp scratch = scratch_begin(&arena, 1);
MSF_UInt file_info_pos = sizeof(PDB_DbiHeader) +
dbi_header->module_info_size +
dbi_header->sec_con_size +
dbi_header->sec_map_size;
msf_stream_seek(msf, sn, file_info_pos);
U16 mod_count = msf_stream_read_u16(msf, sn);
U16 total_file_count16 = msf_stream_read_u16(msf, sn);
CV_ModIndex *imod_array = push_array(scratch.arena, CV_ModIndex, mod_count);
msf_stream_read_array(msf, sn, &imod_array[0], mod_count);
U16 *mod_file_count = push_array(scratch.arena, U16, mod_count);
msf_stream_read_array(msf, sn, &mod_file_count[0], mod_count);
U64 total_file_count = 0;
for (U16 imod = 0; imod < mod_count; imod += 1) {
total_file_count += mod_file_count[imod];
}
U32 *file_name_offset_array = push_array(scratch.arena, U32, total_file_count);
msf_stream_read_array(msf, sn, &file_name_offset_array[0], total_file_count);
U64 file_name_buffer_offset = sizeof(mod_count) +
sizeof(total_file_count16) +
sizeof(imod_array[0]) * mod_count +
sizeof(mod_file_count[0]) * mod_count +
sizeof(file_name_offset_array[0]) * total_file_count;
Assert(dbi_header->file_info_size >= file_name_buffer_offset);
U64 file_name_buffer_size = dbi_header->file_info_size - file_name_buffer_offset;
char *file_name_buffer = push_array(arena, char, file_name_buffer_size + 1);
msf_stream_read_array(msf, sn, &file_name_buffer[0], file_name_buffer_size);
String8List *file_info = push_array(arena, String8List, mod_count + 1);
U32 *file_name_offset_ptr = &file_name_offset_array[0];
for (U64 mod_idx = 0; mod_idx < mod_count; ++mod_idx) {
String8List *file_list = &file_info[mod_idx];
U16 file_count = mod_file_count[mod_idx];
for (U16 ifile = 0; ifile < file_count; ifile += 1, file_name_offset_ptr += 1) {
Assert(*file_name_offset_ptr <= file_name_buffer_size);
String8 file_path = str8_cstring(file_name_buffer + *file_name_offset_ptr);
str8_list_push(arena, file_list, file_path);
}
}
scratch_end(scratch);
ProfEnd();
return file_info;
}
internal PDB_DbiModuleList
dbi_open_module_info(Arena *arena, MSF_Context *msf, MSF_StreamNumber sn, PDB_DbiHeader *dbi_header, String8List *file_info)
{
ProfBeginFunction();
PDB_DbiModuleList list = {0};
MSF_UInt module_info_pos = sizeof(PDB_DbiHeader);
msf_stream_seek(msf, sn, module_info_pos);
MSF_UInt module_info_opl = module_info_pos + dbi_header->module_info_size;
while (msf_stream_get_pos(msf, sn) < module_info_opl) {
PDB_DbiCompUnitHeader header = {0};
msf_stream_read_struct(msf, sn, &header);
String8 obj_path = msf_stream_read_string(arena, msf, sn);
String8 lib_path = msf_stream_read_string(arena, msf, sn);
msf_stream_align(msf, sn, PDB_MODULE_ALIGN);
String8List source_file_list = {0};
if (header.contribution.base.mod != CV_ModIndex_Invalid) {
source_file_list = file_info[header.contribution.base.mod];
}
PDB_DbiModule *mod = push_array(arena, PDB_DbiModule, 1);
mod->next = 0;
mod->sn = header.sn;
mod->imod = header.contribution.base.mod;
mod->sym_data_size = header.symbols_size;
mod->c11_data_size = header.c11_lines_size;
mod->c13_data_size = header.c13_lines_size;
mod->source_file_list = source_file_list;
mod->obj_path = obj_path;
mod->lib_path = lib_path;
mod->first_sc = header.contribution;
SLLQueuePush(list.first, list.last, mod);
list.count += 1;
}
ProfEnd();
return list;
}
internal PDB_DbiSCArray
dbi_open_sec_contrib(Arena *arena, MSF_Context *msf, MSF_StreamNumber sn, PDB_DbiHeader *dbi_header)
{
ProfBeginFunction();
PDB_DbiSCArray sec_contribs = {0};
if (dbi_header->sec_con_size > sizeof(PDB_DbiSC)) {
Temp scratch = scratch_begin(&arena, 1);
// seek to start of section contrib info
MSF_UInt sec_con_pos = sizeof(PDB_DbiHeader) + dbi_header->module_info_size;
msf_stream_seek(msf, sn, sec_con_pos);
// read header
PDB_DbiSCVersion version = 0;
msf_stream_read_struct(msf, sn, &version);
// parse contrib items
switch (version) {
case PDB_DbiSCVersion_1: {
U64 contrib_count = dbi_header->sec_con_size / sizeof(PDB_DbiSC);
sec_contribs.v = push_array_no_zero(arena, PDB_DbiSC, contrib_count);
sec_contribs.count = contrib_count;
sec_contribs.cap = contrib_count;
MSF_UInt sec_con_read = msf_stream_read_array(msf, sn, sec_contribs.v, contrib_count);
Assert(sec_con_read == sizeof(sec_contribs.v[0]) * contrib_count);
} break;
case PDB_DbiSCVersion_2: {
NotImplemented;
} break;
default: Assert(!"unknown section contrib version"); break;
}
// have we exhausted sec-con bytes?
Assert(sec_con_pos + dbi_header->sec_con_size == msf_stream_get_pos(msf, sn));
scratch_end(scratch);
}
ProfEnd();
return sec_contribs;
}
internal void
dbi_build_section_header_stream(PDB_DbiContext *dbi, MSF_Context *msf, MSF_StreamNumber sn)
{
@@ -3357,47 +3082,6 @@ dbi_module_push_section_contrib(PDB_DbiContext *dbi,
ProfEnd();
}
internal String8
dbi_module_read_symbol_data(Arena *arena, MSF_Context *msf, PDB_DbiModule *mod)
{
String8 symbol_data = str8(0,0);
if (mod->sn != MSF_INVALID_STREAM_NUMBER) {
B32 is_seek_ok = msf_stream_seek(msf, mod->sn, 0);
if (is_seek_ok) {
symbol_data = msf_stream_read_block(arena, msf, mod->sn, mod->sym_data_size);
}
}
return symbol_data;
}
internal String8
dbi_module_read_c11_data(Arena *arena, MSF_Context *msf, PDB_DbiModule *mod)
{
String8 c11_data = str8(0,0);
if (mod->sn != MSF_INVALID_STREAM_NUMBER) {
MSF_UInt c11_data_pos = mod->sym_data_size;
B32 is_seek_ok = msf_stream_seek(msf, mod->sn, c11_data_pos);
if (is_seek_ok) {
c11_data = msf_stream_read_block(arena, msf, mod->sn, mod->c13_data_size);
}
}
return c11_data;
}
internal String8
dbi_module_read_c13_data(Arena *arena, MSF_Context *msf, PDB_DbiModule *mod)
{
String8 c13_data = str8(0,0);
if (mod->sn != MSF_INVALID_STREAM_NUMBER) {
MSF_UInt c13_data_pos = mod->sym_data_size + mod->c11_data_size;
B32 is_seek_ok = msf_stream_seek(msf, mod->sn, c13_data_pos);
if (is_seek_ok) {
c13_data = msf_stream_read_block(arena, msf, mod->sn, mod->c13_data_size);
}
}
return c13_data;
}
internal void
dbi_push_section(PDB_DbiContext *dbi, COFF_SectionHeader *hdr)
{
@@ -3523,6 +3207,27 @@ pdb_build_gsi_psi(TP_Context *tp, PDB_Context *pdb)
{
PDB_DbiContext *dbi = pdb->dbi;
// Publics and globals share one symbol stream. Size both before either
// serializer borrows the backing so the second append does not copy it.
if (pdb->psi->gsi->symbol_count || pdb->gsi->symbol_count) {
Temp scratch = scratch_begin(0, 0);
PDB_GsiContext *contexts[] = { pdb->psi->gsi, pdb->gsi };
U64 size = 0;
for EachElement(i, contexts) {
PDB_GsiSerializeSymbolsTask task = {
.symbol_align = contexts[i]->symbol_align,
.bucket_arr = contexts[i]->bucket_arr,
.bucket_size_arr = push_array(scratch.arena, U64, contexts[i]->bucket_count),
};
tp_for_parallel(tp, 0, contexts[i]->bucket_count, gsi_size_buckets_task, &task);
size += sum_array_u64(contexts[i]->bucket_count, task.bucket_size_arr);
}
if (dbi->symbols_sn == MSF_INVALID_STREAM_NUMBER) { dbi->symbols_sn = msf_stream_alloc(pdb->msf); }
AssertAlways(msf_stream_resize(pdb->msf, dbi->symbols_sn, safe_cast_u32(size)));
msf_stream_seek_start(pdb->msf, dbi->symbols_sn);
scratch_end(scratch);
}
if (pdb->psi->gsi->symbol_count) {
if (dbi->publics_sn == MSF_INVALID_STREAM_NUMBER) { dbi->publics_sn = msf_stream_alloc(pdb->msf); }
if (dbi->symbols_sn == MSF_INVALID_STREAM_NUMBER) { dbi->symbols_sn = msf_stream_alloc(pdb->msf); }
src/linker/pdb_ext/pdb_builder.h
+2 -13
View File
@@ -204,7 +204,6 @@ typedef struct PDB_GsiBuildResult
U64 compressed_bucket_count;
U32 *compressed_bucket_arr;
U64 total_hash_size;
String8 symbol_data;
} PDB_GsiBuildResult;
typedef struct PDB_GsiSerializeSymbolsTask
@@ -364,8 +363,8 @@ internal PDB_SrcError pdb_add_src(PDB_InfoContext *info, MSF_Context *msf,
internal PDB_GsiContext * gsi_alloc(void);
internal void gsi_build(TP_Context *tp, PDB_GsiContext *gsi, MSF_Context *msf, MSF_StreamNumber gsi_sn, MSF_StreamNumber symbols_sn);
internal void gsi_release(PDB_GsiContext *gsi);
internal void gsi_write_build_result(TP_Context *tp, PDB_GsiBuildResult build, MSF_Context *msf, MSF_StreamNumber sn, MSF_StreamNumber symbols_sn);
internal PDB_GsiBuildResult gsi_build_ex(TP_Context *tp, Arena *arena, PDB_GsiContext *gsi, U64 symbol_data_base, B32 export_symbol_ptr_arr, U64 msf_page_size);
internal void gsi_write_build_result(TP_Context *tp, PDB_GsiBuildResult build, MSF_Context *msf, MSF_StreamNumber sn);
internal PDB_GsiBuildResult gsi_build_ex(TP_Context *tp, Arena *arena, PDB_GsiContext *gsi, MSF_Context *msf, MSF_StreamNumber symbols_sn, B32 is_pub32);
internal U32 gsi_hash(PDB_GsiContext *gsi, String8 input);
internal CV_SymbolNode * gsi_push(PDB_GsiContext *gsi, CV_Symbol *symbol);
internal void gsi_push_many_arr(TP_Context *tp, PDB_GsiContext *gsi, U64 count, CV_SymbolNode **symbol_arr);
@@ -391,16 +390,7 @@ internal PDB_DbiContext * dbi_alloc(COFF_MachineType machine, U32 age);
internal void dbi_build(TP_Context *tp, PDB_DbiContext *dbi, MSF_Context *msf, MSF_StreamNumber dbi_sn, CV_StringHashTable string_ht, B32 is_stripped);
internal void dbi_release(PDB_DbiContext *dbi);
internal PDB_DbiModule * dbi_push_module(PDB_DbiContext *dbi, String8 obj_path, String8 lib_path);
internal String8 dbi_module_read_symbol_data(Arena *arena, MSF_Context *msf, PDB_DbiModule *mod);
internal String8 dbi_module_read_c11_data(Arena *arena, MSF_Context *msf, PDB_DbiModule *mod);
internal String8 dbi_module_read_c13_data(Arena *arena, MSF_Context *msf, PDB_DbiModule *mod);
internal void dbi_module_push_section_contrib(PDB_DbiContext *dbi, PDB_DbiModule *mod, ISectOff isect_off, U32 size, U32 data_crc, U32 reloc_crc, COFF_SectionFlags flags);
internal String8List * dbi_open_file_info(Arena *arena, MSF_Context *msf, MSF_StreamNumber sn, PDB_DbiHeader *dbi_header);
internal PDB_DbiModuleList dbi_open_module_info(Arena *arena, MSF_Context *msf, MSF_StreamNumber sn, PDB_DbiHeader *dbi_header, String8List *file_info);
internal PDB_DbiSCArray dbi_open_sec_contrib(Arena *arena, MSF_Context *msf, MSF_StreamNumber sn, PDB_DbiHeader *dbi_header);
internal PDB_StringTable dbi_open_ec_names(Arena *arena, MSF_Context *msf, MSF_StreamNumber sn, PDB_DbiHeader *dbi_header);
internal void dbi_open_dbg_streams(MSF_StreamNumber *dbg_streams, MSF_Context *msf, MSF_StreamNumber sn, PDB_DbiHeader *dbi_header);
internal PDB_DbiSectionList dbi_open_section_headers(Arena *arena, MSF_Context *msf, MSF_StreamNumber sn);
internal void dbi_build_section_header_stream(PDB_DbiContext *dbi, MSF_Context *msf, MSF_StreamNumber sn);
////////////////////////////////
@@ -450,7 +440,6 @@ internal U32 pdb_strtab_hash(PDB_StringTable *strtab, Strin
// Type Server
internal PDB_TypeServer * pdb_type_server_alloc(U64 bucket_count);
internal PDB_TypeServer * pdb_type_server_open_v80(MSF_Context *msf, MSF_StreamNumber sn, PDB_StringTable *strtab);
internal void pdb_type_server_build(TP_Context *tp, PDB_TypeServer *ts, PDB_StringTable *strtab, MSF_Context *msf, MSF_StreamNumber sn);
internal void pdb_type_server_release(PDB_TypeServer **serv_ptr);
internal void pdb_type_server_push(PDB_TypeServer *ts, String8 raw_leaf);
src/linker/pdb_ext/pdb_helpers.c
-20
View File
@@ -25,25 +25,6 @@ pdb_read_bit_vector_string(String8 data, U64 offset, U32Array *bits_out)
return read_size;
}
internal U64
pdb_read_bit_vector_msf(Arena *arena, MSF_Context *msf, MSF_StreamNumber sn, U32Array *bits_out)
{
// peek word count
MSF_UInt pos = msf_stream_get_pos(msf, sn);
U32 word_count = msf_stream_read_u32(msf, sn);
msf_stream_seek(msf, sn, pos);
// read out header + packed words
U64 buffer_size = sizeof(word_count) + word_count * sizeof(U32);
U8 *buffer = push_array(arena, U8, buffer_size);
MSF_UInt read_size = msf_stream_read(msf, sn, buffer, buffer_size);
Assert(read_size == buffer_size);
// parse words
U64 parse_size = pdb_read_bit_vector_string(str8(buffer, buffer_size), 0, bits_out);
return parse_size;
}
internal B32
pdb_write_bit_vector(MSF_Context *msf, MSF_StreamNumber sn, B32 *flag_array, U64 flag_count)
{
@@ -86,4 +67,3 @@ pdb_get_bit_vector_size(U32 bucket_count)
return result;
}
src/linker/pdb_ext/pdb_helpers.h
-2
View File
@@ -7,8 +7,6 @@ internal U32 pdb_hash_v1(String8 data);
internal U32 pdb_hash_udt(CV_UDTInfo udt_info, String8 data);
internal U64 pdb_read_bit_vector_string(String8 data, U64 offset, U32Array *bits_out);
internal U64 pdb_read_bit_vector_msf(Arena *arena, MSF_Context *msf, MSF_StreamNumber sn, U32Array *bits_out);
internal B32 pdb_write_bit_vector(MSF_Context *msf, MSF_StreamNumber sn, B32 *flag_array, U64 flag_count);
internal U64 pdb_get_bit_vector_size(U32 bucket_count);
src/linker/tests/linker_tests.c
+119 -31
View File
@@ -7779,7 +7779,7 @@ TEST(cyclic_type)
T_Ok(is_cycle_detected);
}
TEST(get_msf_stream_pages)
TEST(msf_contiguous_stream)
{
MSF_Context *msf = msf_alloc(MSF_DEFAULT_PAGE_SIZE, MSF_DEFAULT_FPM);
@@ -7792,44 +7792,107 @@ TEST(get_msf_stream_pages)
MemorySet(test, 0xca, stream_size/2);
MemorySet(test + stream_size/2, 0xbe, stream_size/2);
String8List stream_data = msf_data_from_sn(arena, msf, sn);
T_Ok(stream_data.total_size == stream_size);
T_Ok(stream_data.node_count == 12);
String8 stream_data = msf_stream_data(msf, sn);
T_Ok(stream_data.size == stream_size);
MemoryCopy(stream_data.str, test, stream_size);
String8Array a = str8_array_from_list(arena, &stream_data);
T_Ok(a.v[0].size == 0xffd000);
T_Ok(a.v[1].size == 0xffe000);
T_Ok(a.v[2].size == 0xffe000);
T_Ok(a.v[3].size == 0xffe000);
T_Ok(a.v[4].size == 0xffe000);
T_Ok(a.v[5].size == 0xffe000);
T_Ok(a.v[6].size == 0xffe000);
T_Ok(a.v[7].size == 0xffd000);
T_Ok(a.v[8].size == 0x1000);
T_Ok(a.v[9].size == 0xffe000);
T_Ok(a.v[10].size == 0x613000);
T_Ok(a.v[11].size == 1);
String8Node buf = *stream_data.first;
U64 buf_pos = 0;
str8_buffer_write(&buf, &buf_pos, str8(test, stream_size));
String8 cmp = msf_stream_read_block(arena, msf, sn, stream_size);
// Physical pages alias the one contiguous construction allocation, even
// across reserved FPM slots and bitmap intervals.
MSF_Stream *stream = msf_find_stream(msf, sn);
U64 offset = 0;
for EachNode(page, MSF_PageNode, stream->page_list.first) {
T_Ok(msf_data_from_pn(msf, page->pn).str == stream_data.str + offset);
offset += msf->page_size;
}
MSF_PageNumber first_pn = stream->page_list.first->pn;
MSF_PageNumber last_pn = stream->page_list.last->pn;
U64 before_build = arena_pos(msf->arena);
String8 saved = {0};
T_Ok(msf_save_arena(arena, msf, &saved) == MSF_Error_Ok);
// Building adds metadata, not another 150 MiB payload allocation.
T_Ok(arena_pos(msf->arena) - before_build < MB(1));
T_Ok(stream->data == stream_data.str);
T_Ok(stream->page_list.first->pn == first_pn);
T_Ok(stream->page_list.last->pn == last_pn);
MSF_Parsed *parsed = msf_parsed_from_data(arena, saved);
T_Ok(parsed != 0);
String8 cmp = msf_data_from_stream(parsed, sn);
T_Ok(cmp.size == stream_size);
T_Ok(MemoryCompare(cmp.str, test, stream_size) == 0);
}
{
MSF_StreamNumber sn = msf_stream_alloc_ex(msf, 1);
String8List stream_data = msf_data_from_sn(arena, msf, sn);
T_Ok(stream_data.node_count == 1);
T_Ok(stream_data.total_size == 1);
T_Ok(stream_data.first->string.size == 1);
String8 stream_data = msf_stream_data(msf, sn);
T_Ok(stream_data.size == 1);
}
msf_release(msf);
}
TEST(msf_stream_growth_and_page_references)
{
MSF_Context *msf = msf_alloc(512, MSF_FPM1);
U64 P = msf->page_size;
MSF_StreamNumber a = msf_stream_alloc_ex(msf, P + 3);
MSF_StreamNumber b = msf_stream_alloc_ex(msf, P);
MSF_StreamNumber empty = msf_stream_alloc(msf);
MSF_Stream *stream = msf_find_stream(msf, a);
MSF_PageNumber first_pn = stream->page_list.first->pn;
MSF_PageNumber second_pn = stream->page_list.last->pn;
MSF_PageNumber b_pn = msf_find_stream(msf, b)->page_list.first->pn;
MemorySet(stream->data, 0x5a, stream->size);
T_Ok(msf_stream_resize(msf, a, 3 * P + 7));
T_Ok(stream->page_list.first->pn == first_pn);
T_Ok(stream->page_list.first->next->pn == second_pn);
T_Ok(stream->page_list.first->next->next->pn != second_pn + 1);
String8 data = msf_stream_data(msf, a);
for EachIndex(i, P + 3) { T_Ok(data.str[i] == 0x5a); }
for (U64 i = P + 3; i < data.size; i += 1) { T_Ok(data.str[i] == 0); }
// Store a page reference before build and patch a field crossing the
// boundary between two nonadjacent physical output pages.
memory_write32(msf_stream_data(msf, b).str, first_pn);
U8 patch[] = {1, 2, 3, 4, 5, 6, 7, 8};
MemoryCopy(data.str + 2 * P - 3, patch, sizeof(patch));
String8 expected = push_str8_copy(arena, data);
// Reservation assigns identities but does not extend the logical stream.
T_Ok(msf_stream_reserve(msf, empty, 2 * P));
T_Ok(msf_stream_get_size(msf, empty) == 0);
String8 saved = {0};
T_Ok(msf_save_arena(arena, msf, &saved) == MSF_Error_Ok);
MSF_RawStreamTable *table = msf_raw_stream_table_from_data(arena, saved);
T_Ok(table->streams[a].u.page_indices_u32[0] == first_pn);
T_Ok(table->streams[a].u.page_indices_u32[1] == second_pn);
T_Ok(table->streams[empty].page_count == 0);
T_Ok(str8_match(msf_data_from_stream_number(arena, saved, table, a), expected, 0));
String8 b_data = msf_data_from_stream_number(arena, saved, table, b);
T_Ok(memory_read32(b_data.str) == first_pn);
// Removing a stream leaves a directory hole; later stream numbers remain
// stable, and its physical page can be reused without stale payload bytes.
T_Ok(msf_stream_free(msf, b));
MSF_StreamNumber c = msf_stream_alloc_ex(msf, 1);
T_Ok(c > empty);
T_Ok(msf_find_stream(msf, c)->page_list.first->pn == b_pn);
T_Ok(msf_stream_data(msf, c).str[0] == 0);
T_Ok(msf_stream_write_u8(msf, c, 0x71));
T_Ok(msf_stream_resize(msf, a, P + 1));
T_Ok(msf_save_arena(arena, msf, &saved) == MSF_Error_Ok);
table = msf_raw_stream_table_from_data(arena, saved);
T_Ok(table->stream_count == c + 1);
T_Ok(table->streams[b].size == 0);
T_Ok(table->streams[empty].size == 0);
T_Ok(str8_match(msf_data_from_stream_number(arena, saved, table, a), str8_prefix(expected, P + 1), 0));
T_Ok(msf_data_from_stream_number(arena, saved, table, c).str[0] == 0x71);
// The final partial page is padded, not populated with truncated bytes.
U8 *tail = saved.str + (U64)second_pn * P;
for (U64 i = 1; i < P; i += 1) { T_Ok(tail[i] == 0); }
msf_release(msf);
}
TEST(msf_header_matches_saved_extent)
{
MSF_UInt page_sizes[] = {512, KB(4)};
@@ -7891,9 +7954,9 @@ TEST(msf_header_matches_saved_extent)
// Free the low pages so metadata can reuse them while the payload remains at
// high page numbers. Unused pages below EOF exercise sparse MSF page layouts.
msf_free_pages(msf, &gap);
T_Ok(msf_build(msf) == MSF_Error_OK);
T_Ok(msf->root_page_list.last->pn < last_pn);
T_Ok(msf->st_page_list.last->pn < last_pn);
T_Ok(msf_build(msf) == MSF_Error_Ok);
T_Ok(msf->root.page_list.last->pn < last_pn);
T_Ok(msf->stream_table.page_list.last->pn < last_pn);
// validate that last page was assigned to the payload
U64 save_size = msf_get_save_size(msf);
@@ -7907,6 +7970,31 @@ TEST(msf_header_matches_saved_extent)
MSF_Header70 *header = (MSF_Header70 *)saved.str;
T_Ok((U64)header->page_count * header->page_size == saved.size);
// Validate the serialized allocation bitmap, including free holes
// and the reserved slots at every physical FPM interval.
U8 *used = push_array(arena, U8, header->page_count);
used[0] = 1;
for (U64 pn = 0; pn < header->page_count; pn += 1) {
if (pn % P == MSF_FPM0 || pn % P == MSF_FPM1) { used[pn] = 1; }
}
for EachNode(page, MSF_PageNode, msf->root.page_list.first) { used[page->pn] = 1; }
for EachNode(page, MSF_PageNode, msf->stream_table.page_list.first) { used[page->pn] = 1; }
for EachNode(page, MSF_PageNode, msf_find_stream(msf, sn)->page_list.first) { used[page->pn] = 1; }
for (U64 pn = 0; pn < header->page_count; pn += 1) {
U64 fpm_pn = (pn / (P * 8)) * P + header->active_fpm;
U64 bit = pn % (P * 8);
U8 free_bit = (saved.str[fpm_pn * P + bit / 8] >> (bit % 8)) & 1;
T_Ok(free_bit == !used[pn]);
}
// CDB/DIA reject an otherwise readable PDB if the final bitmap
// marks future reserved FPM slots beyond EOF as allocated.
U64 last_fpm_pn = ((header->page_count - 1) / (P * 8)) * P + header->active_fpm;
U64 end_bit = ((U64)header->page_count - 1) % (P * 8) + 1;
for (U64 bit = end_bit; bit < P * 8; bit += 1) {
U8 free_bit = (saved.str[last_fpm_pn * P + bit / 8] >> (bit % 8)) & 1;
T_Ok(free_bit == 1);
}
// match internal page data list to the serialized file bytes
String8List pages = msf_get_page_data_nodes(arena, msf);
T_Ok(str8_match(saved, str8_list_join(arena, &pages, 0), 0));
@@ -7952,7 +8040,7 @@ data_from_pdb(Arena *arena, PDB_Context *pdb)
TP_Arena *tp_arena = tp_arena_alloc(tp);
pdb_build(tp, tp_arena, pdb, (CV_StringHashTable){0}, 1, 0, 0);
AssertAlways(msf_build(pdb->msf) == MSF_Error_OK);
AssertAlways(msf_build(pdb->msf) == MSF_Error_Ok);
String8List raw_msf_list = msf_get_page_data_nodes(arena, pdb->msf);
AssertAlways(t_write_file_list(str8_lit("test.pdb"), raw_msf_list));