From 2d672d232f60e6fd22da01601fed69e43669af8c Mon Sep 17 00:00:00 2001 From: Ryan Fleury Date: Thu, 2 Oct 2025 16:12:53 -0700 Subject: [PATCH] checksum baking & rdi changes; checksum dumping --- src/lib_rdi/rdi.c | 17 ++++++ src/lib_rdi/rdi.h | 8 ++- src/rdi/rdi.mdesk | 28 +++++++-- src/rdi/rdi_local.c | 28 ++++++++- src/rdi_make/rdi_make_local.c | 103 +++++++++++++--------------------- src/rdi_make/rdi_make_local.h | 9 ++- 6 files changed, 117 insertions(+), 76 deletions(-) diff --git a/src/lib_rdi/rdi.c b/src/lib_rdi/rdi.c index 444f43d4..dc297153 100644 --- a/src/lib_rdi/rdi.c +++ b/src/lib_rdi/rdi.c @@ -321,4 +321,21 @@ rdi_explanation_string_from_eval_conversion_kind(RDI_EvalConversionKind kind, RD return rdi_eval_conversion_kind_message_string_table[kind].str; } +RDI_PROC RDI_SectionKind +rdi_section_kind_from_checksum_kind(RDI_ChecksumKind kind) +{ +RDI_SectionKind result = 0; +switch(kind) +{ +default:{}break; +case RDI_ChecksumKind_NULL:{result = RDI_SectionKind_NULL;}break; +case RDI_ChecksumKind_MD5:{result = RDI_SectionKind_MD5Checksums;}break; +case RDI_ChecksumKind_SHA1:{result = RDI_SectionKind_SHA1Checksums;}break; +case RDI_ChecksumKind_SHA256:{result = RDI_SectionKind_SHA256Checksums;}break; +case RDI_ChecksumKind_Timestamp:{result = RDI_SectionKind_Timestamps;}break; +case RDI_ChecksumKind_COUNT:{result = RDI_SectionKind_NULL;}break; +} +return result; +} + #endif // RDI_C diff --git a/src/lib_rdi/rdi.h b/src/lib_rdi/rdi.h index d782fb31..4d052bce 100644 --- a/src/lib_rdi/rdi.h +++ b/src/lib_rdi/rdi.h @@ -64,7 +64,7 @@ union RDI_SHA256 {RDI_U8 u8[32]; RDI_U64 u64[4];}; // "raddbg\0\0" #define RDI_MAGIC_CONSTANT 0x0000676264646172 -#define RDI_ENCODING_VERSION 15 +#define RDI_ENCODING_VERSION 16 //////////////////////////////////////////////////////////////// //~ Format Types & Functions @@ -321,7 +321,7 @@ RDI_BinarySectionFlag_Write = 1<<1, RDI_BinarySectionFlag_Execute = 1<<2, } RDI_BinarySectionFlagsEnum; -typedef RDI_U16 RDI_ChecksumKind; +typedef RDI_U32 RDI_ChecksumKind; typedef enum RDI_ChecksumKindEnum { RDI_ChecksumKind_NULL = 0, @@ -840,6 +840,8 @@ X(RDI_U32, source_file_idx)\ X(RDI_U32, file_path_node_idx)\ X(RDI_U32, normal_full_path_string_idx)\ X(RDI_U32, source_line_map_idx)\ +X(RDI_ChecksumKind, checksum_kind)\ +X(RDI_U32, checksum_idx)\ #define RDI_Unit_XList \ X(RDI_U32, unit_name_string_idx)\ @@ -1309,6 +1311,8 @@ struct RDI_SourceFile RDI_U32 file_path_node_idx; RDI_U32 normal_full_path_string_idx; RDI_U32 source_line_map_idx; +RDI_ChecksumKind checksum_kind; +RDI_U32 checksum_idx; }; typedef struct RDI_Unit RDI_Unit; diff --git a/src/rdi/rdi.mdesk b/src/rdi/rdi.mdesk index a4a219b2..d106d98d 100644 --- a/src/rdi/rdi.mdesk +++ b/src/rdi/rdi.mdesk @@ -74,7 +74,7 @@ ""; "// \"raddbg\\0\\0\""; "#define RDI_MAGIC_CONSTANT 0x0000676264646172"; - "#define RDI_ENCODING_VERSION 15"; + "#define RDI_ENCODING_VERSION 16"; ""; "////////////////////////////////////////////////////////////////"; "//~ Format Types & Functions"; @@ -564,7 +564,7 @@ RDI_ChecksumKindTable: {COUNT 5 NULL } } -@enum(RDI_U16) RDI_ChecksumKind: +@enum(RDI_U32) RDI_ChecksumKind: { @expand(RDI_ChecksumKindTable a) `$(a.name .. =>10) = $(a.value)` } @@ -590,13 +590,15 @@ RDI_FilePathNodeMemberTable: @table(name type desc) RDI_SourceFileMemberTable: { - {file_path_node_idx RDI_U32 ""} - {normal_full_path_string_idx RDI_U32 ""} + {file_path_node_idx RDI_U32 ""} + {normal_full_path_string_idx RDI_U32 ""} // usage of line map to go from a line number to an array of voffs // (line_map_nums * line_number) -> (nil | index) // (line_map_data * index) -> (range) // (line_map_voff_data * range) -> (array(voff)) - {source_line_map_idx RDI_U32 ""} + {source_line_map_idx RDI_U32 ""} + {checksum_kind RDI_ChecksumKind ""} + {checksum_idx RDI_U32 ""} } @xlist RDI_FilePathNode_XList: @@ -1648,3 +1650,19 @@ rdi_explanation_string_from_eval_conversion_kind(RDI_EvalConversionKind kind, RD return rdi_eval_conversion_kind_message_string_table[kind].str; } ``` + +@gen(functions) @c_file +{ + `RDI_PROC RDI_SectionKind`; + `rdi_section_kind_from_checksum_kind(RDI_ChecksumKind kind)`; + `{`; + `RDI_SectionKind result = 0;`; + `switch(kind)`; + `{`; + `default:{}break;`; + @expand(RDI_ChecksumKindTable a) `case RDI_ChecksumKind_$(a.name):{result = RDI_SectionKind_$(a.section);}break;`, + `}`; + `return result;`; + `}`; + ``; +} diff --git a/src/rdi/rdi_local.c b/src/rdi/rdi_local.c index 8473578c..0d91e869 100644 --- a/src/rdi/rdi_local.c +++ b/src/rdi/rdi_local.c @@ -671,13 +671,39 @@ lane_sync(); if(flags & RDI_DumpSubsetFlag_##name) ProfScope(#name) { U64 count = 0; RDI_SourceFile *v = rdi_table_from_name(rdi, SourceFiles, &count); + U64 checksums_count[RDI_ChecksumKind_COUNT] = {0}; + RDI_U8 *checksums_data[RDI_ChecksumKind_COUNT] = {0}; + RDI_U64 checksums_element_sizes[RDI_ChecksumKind_COUNT] = {0}; + for EachEnumVal(RDI_ChecksumKind, k) + { + RDI_SectionKind section_kind = rdi_section_kind_from_checksum_kind(k); + checksums_data[k] = rdi_section_raw_table_from_kind(rdi, section_kind, &checksums_count[k]); + checksums_element_sizes[k] = rdi_section_element_size_table[section_kind]; + } Rng1U64 range = lane_range(count); for EachInRange(idx, range) { RDI_SourceFile *source_file = &v[idx]; - dumpf("\n { file_path_node_idx: %4u, source_line_map: %4u, path: %-192S } // source_file[%I64u]", + RDI_ChecksumKind checksum_kind = source_file->checksum_kind; + RDI_U32 checksum_idx = source_file->checksum_idx; + String8 checksum_kind_name = {0}; + switch(checksum_kind) + { + default:{checksum_kind_name = str8_lit("Null");}break; +#define X(name, s) case RDI_ChecksumKind_##name:{checksum_kind_name = str8_lit(#name);}break; + RDI_ChecksumKind_XList +#undef X + } + String8 checksum_value = str8(checksums_data[checksum_kind] + checksums_element_sizes[checksum_kind]*checksum_idx, checksums_element_sizes[checksum_kind]); + String8List checksum_vals = numeric_str8_list_from_data(arena, 16, checksum_value, 1); + StringJoin join = {0}; + join.sep = str8_lit(", "); + String8 checksum_val_string = str8_list_join(arena, &checksum_vals, &join); + dumpf("\n { file_path_node_idx: %4u, source_line_map: %4u, checksum_kind: %10S, checksum_value: %192S, path: %-192S } // source_file[%I64u]", source_file->file_path_node_idx, source_file->source_line_map_idx, + checksum_kind_name, + checksum_val_string, push_str8f(arena, "'%S'", str8_from_rdi_string_idx(rdi, source_file->normal_full_path_string_idx)), idx); } diff --git a/src/rdi_make/rdi_make_local.c b/src/rdi_make/rdi_make_local.c index f03b7614..70ccc333 100644 --- a/src/rdi_make/rdi_make_local.c +++ b/src/rdi_make/rdi_make_local.c @@ -1785,6 +1785,11 @@ rdim_bake(Arena *arena, RDIM_BakeParams *params) rdim_shared->lane_chunk_src_file_num_offs = push_array(arena, U64, lane_count()*params->src_files.chunk_count); rdim_shared->lane_chunk_src_file_voff_offs = push_array(arena, U64, lane_count()*params->src_files.chunk_count); rdim_shared->lane_chunk_src_file_map_offs = push_array(arena, U64, lane_count()*params->src_files.chunk_count); + for EachEnumVal(RDI_ChecksumKind, k) + { + rdim_shared->lane_chunk_src_file_checksum_counts[k] = push_array(arena, U64, lane_count()*params->src_files.chunk_count); + rdim_shared->lane_chunk_src_file_checksum_offs[k] = push_array(arena, U64, lane_count()*params->src_files.chunk_count); + } } lane_sync(); @@ -1801,6 +1806,12 @@ rdim_bake(Arena *arena, RDIM_BakeParams *params) rdim_shared->lane_chunk_src_file_num_counts[slot_idx] += map->line_count; rdim_shared->lane_chunk_src_file_voff_counts[slot_idx] += map->voff_range_count; rdim_shared->lane_chunk_src_file_map_counts[slot_idx] += !!map->line_count; + RDI_ChecksumKind k = n->v[idx].checksum_kind; + String8 val = n->v[idx].checksum; + if(RDI_ChecksumKind_NULL < k && k < RDI_ChecksumKind_COUNT && val.size != 0) + { + rdim_shared->lane_chunk_src_file_checksum_counts[k][slot_idx] += 1; + } } chunk_idx += 1; } @@ -1814,6 +1825,7 @@ rdim_bake(Arena *arena, RDIM_BakeParams *params) U64 num_layout_off = 0; U64 voff_layout_off = 0; U64 map_layout_off = 1; + U64 checksum_layout_offs[RDI_ChecksumKind_COUNT] = {0}; for EachNode(n, RDIM_SrcFileChunkNode, params->src_files.first) { for EachIndex(l_idx, lane_count()) @@ -1825,11 +1837,20 @@ rdim_bake(Arena *arena, RDIM_BakeParams *params) num_layout_off += rdim_shared->lane_chunk_src_file_num_counts[slot_idx]; voff_layout_off += rdim_shared->lane_chunk_src_file_voff_counts[slot_idx]; map_layout_off += rdim_shared->lane_chunk_src_file_map_counts[slot_idx]; + for EachEnumVal(RDI_ChecksumKind, k) + { + rdim_shared->lane_chunk_src_file_checksum_offs[k][slot_idx] = checksum_layout_offs[k]; + checksum_layout_offs[k] += rdim_shared->lane_chunk_src_file_checksum_counts[k][slot_idx]; + } } chunk_idx += 1; } rdim_shared->total_src_map_line_count = num_layout_off; rdim_shared->total_src_map_voff_count = voff_layout_off; + for EachEnumVal(RDI_ChecksumKind, k) + { + rdim_shared->total_checksum_counts[k] = checksum_layout_offs[k]; + } } } lane_sync(); @@ -1865,6 +1886,11 @@ rdim_bake(Arena *arena, RDIM_BakeParams *params) U64 dst_map_off = rdim_shared->lane_chunk_src_file_map_offs[slot_idx]; U64 dst_voff_off = rdim_shared->lane_chunk_src_file_voff_offs[slot_idx]; U64 dst_rng_off = dst_num_off + dst_map_off; + U64 dst_checksums_off[RDI_ChecksumKind_COUNT] = {0}; + for EachEnumVal(RDI_ChecksumKind, k) + { + dst_checksums_off[k] = 1 + rdim_shared->lane_chunk_src_file_checksum_offs[k][slot_idx]; + } for EachInRange(idx, range) { RDIM_BakeSrcLineMap *map = &rdim_shared->bake_src_line_maps[n->base_idx + idx]; @@ -1879,11 +1905,20 @@ rdim_bake(Arena *arena, RDIM_BakeParams *params) //- rjf: fill file info Temp scratch = scratch_begin(&arena, 1); String8 normalized_path = rdim_lower_from_str8(scratch.arena, src->path); + B32 has_checksum = (RDI_ChecksumKind_NULL < src->checksum_kind && src->checksum_kind < RDI_ChecksumKind_COUNT && src->checksum.size != 0); dst->file_path_node_idx = rdim_bake_path_node_idx_from_string(path_tree, src->path); dst->normal_full_path_string_idx = rdim_bake_idx_from_string(bake_strings, normalized_path); dst->source_line_map_idx = src->total_line_count ? dst_map_off : 0; + dst->checksum_kind = src->checksum_kind; + dst->checksum_idx = has_checksum ? dst_checksums_off[dst->checksum_kind] : 0; scratch_end(scratch); + //- rjf: advance checksum offset for this kind + if(has_checksum) + { + dst_checksums_off[dst->checksum_kind] += 1; + } + //- rjf: fill map info if(src->total_line_count != 0) { @@ -1920,64 +1955,6 @@ rdim_bake(Arena *arena, RDIM_BakeParams *params) } } - ////////////////////////////////////////////////////////////// - //- rjf: @rdim_bake_stage compute lane checksum layouts - // - ProfScope("compute lane checksum layouts") - { - // rjf: allocate - if(lane_idx() == 0) - { - for EachEnumVal(RDI_ChecksumKind, k) - { - rdim_shared->lane_chunk_src_file_checksum_counts[k] = push_array(arena, U64, lane_count()*params->src_files.chunk_count); - rdim_shared->lane_chunk_src_file_checksum_offs[k] = push_array(arena, U64, lane_count()*params->src_files.chunk_count); - } - } - lane_sync(); - - // rjf: compute counts of all checksums - { - U64 chunk_idx = 0; - for EachNode(n, RDIM_SrcFileChunkNode, params->src_files.first) - { - Rng1U64 range = lane_range(n->count); - U64 slot_idx = lane_idx()*params->src_files.chunk_count + chunk_idx; - for EachInRange(n_idx, range) - { - if(RDI_ChecksumKind_NULL < n->v[n_idx].checksum_kind && n->v[n_idx].checksum_kind < RDI_ChecksumKind_COUNT && n->v[n_idx].checksum.size != 0) - { - rdim_shared->lane_chunk_src_file_checksum_counts[n->v[n_idx].checksum_kind][slot_idx] += 1; - } - } - chunk_idx += 1; - } - } - lane_sync(); - - // rjf: lay out per-lane-chunk offsets - if(lane_idx() == 0) - { - for EachEnumVal(RDI_ChecksumKind, k) - { - U64 off = 0; - U64 chunk_idx = 0; - for EachNode(n, RDIM_SrcFileChunkNode, params->src_files.first) - { - for EachIndex(l_idx, lane_count()) - { - U64 slot_idx = l_idx*params->src_files.chunk_count + chunk_idx; - rdim_shared->lane_chunk_src_file_checksum_offs[k][slot_idx] = off; - off += rdim_shared->lane_chunk_src_file_checksum_counts[k][slot_idx]; - } - chunk_idx += 1; - } - rdim_shared->total_checksum_counts[k] = off; - } - } - } - lane_sync(); - ////////////////////////////////////////////////////////////// //- rjf: @rdim_bake_stage bake checksums // @@ -1986,10 +1963,10 @@ rdim_bake(Arena *arena, RDIM_BakeParams *params) // rjf: allocate if(lane_idx() == 0) { - rdim_shared->baked_checksums.md5s_count = rdim_shared->total_checksum_counts[RDI_ChecksumKind_MD5]; - rdim_shared->baked_checksums.sha1s_count = rdim_shared->total_checksum_counts[RDI_ChecksumKind_SHA1]; - rdim_shared->baked_checksums.sha256s_count = rdim_shared->total_checksum_counts[RDI_ChecksumKind_SHA256]; - rdim_shared->baked_checksums.timestamps_count = rdim_shared->total_checksum_counts[RDI_ChecksumKind_Timestamp]; + rdim_shared->baked_checksums.md5s_count = rdim_shared->total_checksum_counts[RDI_ChecksumKind_MD5] + 1; + rdim_shared->baked_checksums.sha1s_count = rdim_shared->total_checksum_counts[RDI_ChecksumKind_SHA1] + 1; + rdim_shared->baked_checksums.sha256s_count = rdim_shared->total_checksum_counts[RDI_ChecksumKind_SHA256] + 1; + rdim_shared->baked_checksums.timestamps_count = rdim_shared->total_checksum_counts[RDI_ChecksumKind_Timestamp] + 1; rdim_shared->baked_checksums.md5s = push_array(arena, RDI_MD5, rdim_shared->baked_checksums.md5s_count); rdim_shared->baked_checksums.sha1s = push_array(arena, RDI_SHA1, rdim_shared->baked_checksums.sha1s_count); rdim_shared->baked_checksums.sha256s = push_array(arena, RDI_SHA256, rdim_shared->baked_checksums.sha256s_count); @@ -2007,7 +1984,7 @@ rdim_bake(Arena *arena, RDIM_BakeParams *params) U64 dst_offs[RDI_ChecksumKind_COUNT] = {0}; for EachEnumVal(RDI_ChecksumKind, k) { - dst_offs[k] = rdim_shared->lane_chunk_src_file_checksum_offs[k][slot_idx]; + dst_offs[k] = 1 + rdim_shared->lane_chunk_src_file_checksum_offs[k][slot_idx]; } for EachInRange(n_idx, range) { diff --git a/src/rdi_make/rdi_make_local.h b/src/rdi_make/rdi_make_local.h index bd4a0878..b6134a4e 100644 --- a/src/rdi_make/rdi_make_local.h +++ b/src/rdi_make/rdi_make_local.h @@ -131,17 +131,16 @@ struct RDIM_Shared RDI_U64 *lane_chunk_src_file_num_offs; // [lane_count * src_file_chunk_count] RDI_U64 *lane_chunk_src_file_voff_offs; // [lane_count * src_file_chunk_count] RDI_U64 *lane_chunk_src_file_map_offs; // [lane_count * src_file_chunk_count] - RDI_U64 total_src_map_line_count; - RDI_U64 total_src_map_voff_count; - - RDIM_SrcFileBakeResult baked_src_files; - RDI_U64 *lane_chunk_src_file_checksum_counts[RDI_ChecksumKind_COUNT]; // [lane_count * src_file_chunk_count] RDI_U64 *lane_chunk_src_file_checksum_offs[RDI_ChecksumKind_COUNT]; // [lane_count * src_file_chunk_count] U64 total_checksum_counts[RDI_ChecksumKind_COUNT]; + RDI_U64 total_src_map_line_count; + RDI_U64 total_src_map_voff_count; RDIM_ChecksumBakeResult baked_checksums; + RDIM_SrcFileBakeResult baked_src_files; + RDI_U64 *member_chunk_lane_counts; // [lane_count * udt_chunk_count] RDI_U64 *member_chunk_lane_offs; // [lane_count * udt_chunk_count] RDI_U64 *enum_val_chunk_lane_counts; // [lane_count * udt_chunk_count]