diff --git a/src/base/base_strings.c b/src/base/base_strings.c index 540e8a86..8c3ab290 100644 --- a/src/base/base_strings.c +++ b/src/base/base_strings.c @@ -2769,14 +2769,14 @@ str8_serial_push_string(Arena *arena, String8List *srl, String8 str){ internal U64 str8_deserial_read(String8 string, U64 off, void *read_dst, U64 read_size, U64 granularity) { - U64 bytes_left = string.size-Min(off, string.size); + U64 bytes_left = string.size - Min(off, string.size); U64 actually_readable_size = Min(bytes_left, read_size); U64 legally_readable_size = actually_readable_size - actually_readable_size%granularity; if(legally_readable_size > 0) { MemoryCopy(read_dst, string.str+off, legally_readable_size); } - return legally_readable_size; + return actually_readable_size; } internal U64 diff --git a/src/dwarf/dwarf_inc.c b/src/dwarf/dwarf_inc.c index df0cc700..cf8f939e 100644 --- a/src/dwarf/dwarf_inc.c +++ b/src/dwarf/dwarf_inc.c @@ -4,6 +4,7 @@ #include "dwarf/dwarf.c" #include "dwarf/dwarf_expr.c" #include "dwarf/dwarf_parse.c" +#include "dwarf/dwarf_parse_2.c" #include "dwarf/dwarf_coff.c" #include "dwarf/dwarf_elf.c" #include "dwarf/dwarf_unwind.c" diff --git a/src/dwarf/dwarf_inc.h b/src/dwarf/dwarf_inc.h index 44783a34..1f6937af 100644 --- a/src/dwarf/dwarf_inc.h +++ b/src/dwarf/dwarf_inc.h @@ -7,6 +7,7 @@ #include "dwarf/dwarf.h" #include "dwarf/dwarf_expr.h" #include "dwarf/dwarf_parse.h" +#include "dwarf/dwarf_parse_2.h" #include "dwarf/dwarf_coff.h" #include "dwarf/dwarf_elf.h" #include "dwarf/dwarf_unwind.h" diff --git a/src/dwarf/dwarf_parse.c b/src/dwarf/dwarf_parse.c index f564d80e..415e68cc 100644 --- a/src/dwarf/dwarf_parse.c +++ b/src/dwarf/dwarf_parse.c @@ -758,8 +758,6 @@ dw_read_tag(Arena *arena, // fill out node DW_AttribNode *attrib_n = push_array(arena, DW_AttribNode, 1); attrib_n->v.info_off = tag_base + attrib_tag_cursor; - attrib_n->v.abbrev_off = attrib_abbrev_off; - attrib_n->v.abbrev_id = attrib_abbrev.id; attrib_n->v.attrib_kind = attrib_kind; attrib_n->v.form_kind = form_kind; attrib_n->v.form = form; diff --git a/src/dwarf/dwarf_parse.h b/src/dwarf/dwarf_parse.h index 70c0268d..c67123a1 100644 --- a/src/dwarf/dwarf_parse.h +++ b/src/dwarf/dwarf_parse.h @@ -115,8 +115,6 @@ typedef struct DW_Attrib DW_Attrib; struct DW_Attrib { U64 info_off; - U64 abbrev_off; - U64 abbrev_id; DW_AttribKind attrib_kind; DW_FormKind form_kind; DW_Form form; diff --git a/src/dwarf/dwarf_parse_2.c b/src/dwarf/dwarf_parse_2.c new file mode 100644 index 00000000..409d3bf8 --- /dev/null +++ b/src/dwarf/dwarf_parse_2.c @@ -0,0 +1,487 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +//////////////////////////////// +//~ rjf: Basic Parsing Helpers + +internal U64 +dw2_read_initial_length(String8 data, U64 off, U64 *out, DW_Format *fmt_out) +{ + DW_Format format = DW_Format_32Bit; + U64 size = 0; + U64 size_size = 0; + { + U32 size_or_header = 0; + size_size += str8_deserial_read_struct(data, off, &size_or_header); + if(size_or_header == max_U32) + { + size_size += str8_deserial_read_struct(data, off + size_size, &size); + format = DW_Format_64Bit; + } + else + { + size = (U64)size_or_header; + } + } + if(out) + { + out[0] = size; + } + if(fmt_out) + { + fmt_out[0] = format; + } + return size_size; +} + +internal U64 +dw2_read_fmt_u64(String8 data, U64 off, DW_Format format, U64 *out) +{ + U64 start_off = off; + switch(format) + { + default: + case DW_Format_32Bit: + { + U32 out_32 = 0; + off += str8_deserial_read_struct(data, off, &out_32); + out[0] = (U64)out_32; + }break; + case DW_Format_64Bit: + { + off += str8_deserial_read_struct(data, off, out); + }break; + } + U64 bytes_read = (off - start_off); + return bytes_read; +} + +//////////////////////////////// +//~ rjf: Unit Header Parsing + +internal U64 +dw2_read_unit_header(String8 data, U64 off, DW2_UnitHeader *out) +{ + U64 start_off = off; + + // rjf: read unit's .debug_info range size / format + U64 unit_info_size = 0; + DW_Format format = DW_Format_32Bit; + off += dw2_read_initial_length(data, off, &unit_info_size, &format); + + // rjf: read dwarf version + DW_Version version = 0; + off += str8_deserial_read_struct(data, off, &version); + + // rjf: read header properties + U64 abbrev_off = max_U64; + U64 addr_size = 0; + DW_CompUnitKind kind = DW_CompUnitKind_Reserved; + U64 dwo_id = 0; + switch((DW_VersionEnum)version) + { + case DW_Version_Null: + case DW_Version_1: + {}break; + + // rjf: @dwarf2 + case DW_Version_2: + { + // rjf: read abbrev off + U32 abbrev_off_32 = 0; + off += str8_deserial_read_struct(data, off, &abbrev_off_32); + abbrev_off = (U64)abbrev_off_32; + + // rjf: read address size + U8 addr_size_8 = 0; + off += str8_deserial_read_struct(data, off, &addr_size_8); + addr_size = (U64)addr_size_8; + }break; + + // rjf: @dwarf3 @dwarf4 + case DW_Version_3: + case DW_Version_4: + { + // rjf: read abbrev off + off += dw2_read_fmt_u64(data, off, format, &abbrev_off); + + // rjf: read address size + U8 addr_size_8 = 0; + off += str8_deserial_read_struct(data, off, &addr_size_8); + addr_size = (U64)addr_size_8; + }break; + + // rjf: @dwarf5 + case DW_Version_5: + { + // rjf: read unit kind + U8 comp_unit_kind_8 = 0; + off += str8_deserial_read_struct(data, off, &comp_unit_kind_8); + kind = (DW_CompUnitKind)comp_unit_kind_8; + + // rjf: read address size + U8 addr_size_8 = 0; + off += str8_deserial_read_struct(data, off, &addr_size_8); + addr_size = (U64)addr_size_8; + + // rjf: read abbrev off + off += dw2_read_fmt_u64(data, off, format, &abbrev_off); + + // rjf: read DWO ID, if applicable + if(kind == DW_CompUnitKind_Skeleton || + kind == DW_CompUnitKind_SplitCompile || + kind == DW_CompUnitKind_SplitType) + { + off += str8_deserial_read_struct(data, off, &dwo_id); + } + }break; + } + + // rjf: fill + out->version = version; + out->format = format; + out->abbrev_off = abbrev_off; + out->addr_size = addr_size; + out->kind = kind; + out->dwo_id = dwo_id; + + // rjf: return # of bytes read + U64 bytes_read = (off - start_off); + return bytes_read; +} + +//////////////////////////////// +//~ rjf: Abbreviation Map Parsing + +internal DW2_AbbrevMap +dw2_abbrev_map_from_data(Arena *arena, String8 data, U64 off) +{ + DW2_AbbrevMap map = {0}; + + // rjf: loop - first to determine the number of tag formats encoded + // by this abbrev table, second to actually build a hash table for + // mapping (id -> .debug_abbrev offset) + for(B32 build = 0; build <= 1; build += 1) + { + // rjf: parse whole abbrev table + U64 tag_count = 0; + for(U64 read_off = off;;) + { + U64 start_off = read_off; + + // rjf: read tag abbrev + U64 id = 0; + U64 tag_kind = 0; + U8 has_children = 0; + { + read_off += str8_deserial_read_uleb128(data, read_off, &id); + if(id != 0) + { + read_off += str8_deserial_read_uleb128(data, read_off, &tag_kind); + read_off += str8_deserial_read_struct(data, read_off, &has_children); + } + } + + // rjf: count tag + if(id != 0) + { + tag_count += 1; + } + + // rjf: read all tag attribute abbreviations + if(id != 0) + { + for(;;) + { + U64 attrib_start_off = read_off; + U64 attrib_kind = 0; + U64 attrib_form_kind = 0; + U64 implicit_const = 0; + read_off += str8_deserial_read_uleb128(data, read_off, &attrib_kind); + read_off += str8_deserial_read_uleb128(data, read_off, &attrib_form_kind); + if(attrib_form_kind == DW_Form_ImplicitConst) + { + read_off += str8_deserial_read_uleb128(data, read_off, &implicit_const); + } + if(read_off == attrib_start_off || attrib_kind == 0) + { + break; + } + } + } + + // rjf: if building (second loop) -> insert into map + if(build) + { + U64 hash = u64_hash_from_str8(str8_struct(&id)); + U64 slot_idx = hash%map.slots_count; + DW2_AbbrevMapNode *n = push_array(arena, DW2_AbbrevMapNode, 1); + SLLStackPush(map.slots[slot_idx], n); + n->id = id; + n->off = start_off; + } + + // rjf: no tag ID, or no movement? -> exit + if(read_off == start_off || id == 0) + { + break; + } + } + + // rjf: on first loop iteration, given the tag format count, construct the table + if(!build) + { + map.slots_count = Max(1, tag_count + tag_count/4); + map.slots = push_array(arena, DW2_AbbrevMapNode *, map.slots_count); + } + } + + return map; +} + +//////////////////////////////// +//~ rjf: Tag Parsing + +internal U64 +dw2_read_tag(Arena *arena, DW2_TagParseCtx *ctx, String8 data, U64 off, DW2_Tag *tag_out) +{ + U64 start_off = off; + + // rjf: read tag's abbrev ID + U64 abbrev_id = 0; + off += str8_deserial_read_uleb128(data, off, &abbrev_id); + + // rjf: map abbrev ID -> abbrev offset + U64 abbrev_off = 0; + { + DW2_AbbrevMap *abbrev_map = ctx->abbrev_map; + U64 hash = u64_hash_from_str8(str8_struct(&abbrev_id)); + U64 slot_idx = hash%abbrev_map->slots_count; + for EachNode(n, DW2_AbbrevMapNode, abbrev_map->slots[slot_idx]) + { + if(n->id == abbrev_id) + { + abbrev_off = n->off; + break; + } + } + } + + // rjf: read abbrev header + U64 tag_kind = 0; + U8 has_children = 0; + U64 attrib_abbrev_read_off = 0; + { + String8 abbrev_data = ctx->abbrev_data; + U64 abbrev_read_off = abbrev_off; + U64 id = 0; + abbrev_read_off += str8_deserial_read_uleb128(abbrev_data, abbrev_read_off, &id); + if(id != 0) + { + abbrev_read_off += str8_deserial_read_uleb128(abbrev_data, abbrev_read_off, &tag_kind); + abbrev_read_off += str8_deserial_read_struct(abbrev_data, abbrev_read_off, &has_children); + } + attrib_abbrev_read_off = abbrev_read_off; + } + + // rjf: walk abbrev / info in lock-step; read all tag attributes + DW2_AttribList attribs = {0}; + if(abbrev_id != 0) + { + String8 abbrev_data = ctx->abbrev_data; + U64 abbrev_read_off = attrib_abbrev_read_off; + for(;;) + { + U64 abbrev_start_read_off = abbrev_read_off; + + // rjf: read next attribute abbreviation from .debug_abbrev + U64 attrib_kind = 0; + U64 attrib_form_kind = 0; + U64 implicit_const = 0; + abbrev_read_off += str8_deserial_read_uleb128(abbrev_data, abbrev_read_off, &attrib_kind); + abbrev_read_off += str8_deserial_read_uleb128(abbrev_data, abbrev_read_off, &attrib_form_kind); + if(attrib_form_kind == DW_Form_ImplicitConst) + { + abbrev_read_off += str8_deserial_read_uleb128(abbrev_data, abbrev_read_off, &implicit_const); + } + + // rjf: indirect form -> form kind is encoded in .debug_info (because why not) + if(attrib_form_kind == DW_Form_Indirect) + { + off += str8_deserial_read_uleb128(data, off, &attrib_form_kind); + } + + // rjf: read attribute's value from .debug_info + DW2_AttribVal val = {0}; + if(attrib_kind != 0) + { + U64 bytes_to_read = 0; + switch(attrib_form_kind) + { + //- rjf: no-ops + default: + case DW_Form_Indirect: + case DW_Form_Null: + {}break; + + //- rjf: 1-byte uint reads + case DW_Form_Ref1: + case DW_Form_Data1: + case DW_Form_Flag: + case DW_Form_Strx1: + case DW_Form_Addrx1: + bytes_to_read = 1; goto read_fixed_uint; + + //- rjf: 2-byte uint reads + case DW_Form_Ref2: + case DW_Form_Data2: + case DW_Form_Strx2: + case DW_Form_Addrx2: + bytes_to_read = 2; goto read_fixed_uint; + + //- rjf: 3-byte uint reads + case DW_Form_Strx3: + case DW_Form_Addrx3: + bytes_to_read = 3; goto read_fixed_uint; + + //- rjf: 4-byte uint reads + case DW_Form_Data4: + case DW_Form_Ref4: + case DW_Form_RefSup4: + case DW_Form_Strx4: + case DW_Form_Addrx4: + bytes_to_read = 4; goto read_fixed_uint; + + //- rjf: 8-byte uint reads + case DW_Form_Data8: + case DW_Form_Ref8: + case DW_Form_RefSig8: + case DW_Form_RefSup8: + bytes_to_read = 8; goto read_fixed_uint; + + //- rjf: 16-byte uint reads + case DW_Form_Data16: + bytes_to_read = 16; goto read_fixed_uint; + + //- rjf: address-sized reads + case DW_Form_Addr: + bytes_to_read = ctx->addr_size; goto read_fixed_uint; + + //- rjf: format-defined-size reads + case DW_Form_RefAddr: + case DW_Form_SecOffset: + case DW_Form_LineStrp: + case DW_Form_Strp: + case DW_Form_StrpSup: + bytes_to_read = dw_size_from_format(ctx->format); goto read_fixed_uint; + + //- rjf: fixed-size uint reads + read_fixed_uint: + { + off += str8_deserial_read(data, off, &val.u128.u64[0], bytes_to_read, bytes_to_read); + }break; + + //- rjf: uleb128 reads + case DW_Form_UData: + case DW_Form_RefUData: + case DW_Form_Strx: + case DW_Form_Addrx: + case DW_Form_LocListx: + case DW_Form_RngListx: + { + off += str8_deserial_read_uleb128(data, off, &val.u128.u64[0]); + }break; + + //- rjf: sleb128 reads + case DW_Form_SData: + { + off += str8_deserial_read_sleb128(data, off, &val.u128.u64[0]); + }break; + + //- rjf: fixed-size uint reads / skips + case DW_Form_Block1: bytes_to_read = 1; goto read_fixed_uint_and_skip; + case DW_Form_Block2: bytes_to_read = 2; goto read_fixed_uint_and_skip; + case DW_Form_Block4: bytes_to_read = 4; goto read_fixed_uint_and_skip; + read_fixed_uint_and_skip: + { + U64 size = 0; + U64 og_read_off = off; + off += str8_deserial_read(data, off, &size, bytes_to_read, bytes_to_read); + val.u128.u64[0] = size; + val.u128.u64[1] = og_read_off; + off += size; + }break; + + //- rjf: uleb128 read & skip + case DW_Form_Block: + { + U64 size = 0; + U64 og_read_off = off; + off += str8_deserial_read_uleb128(data, off, &size); + val.u128.u64[0] = size; + val.u128.u64[1] = og_read_off; + off += size; + }break; + + //- rjf: strings + case DW_Form_String: + { + String8 string = {0}; + U64 og_read_off = off; + off += str8_deserial_read_cstr(data, off, &string); + val.u128.u64[0] = og_read_off; + val.u128.u64[1] = og_read_off + string.size - 1; + }break; + + //- rjf: implicit constants (no reading in .debug_info; value comes from .debug_abbrev) + case DW_Form_ImplicitConst: + { + val.u128.u64[0] = implicit_const; + }break; + + //- rjf: exprloc + case DW_Form_ExprLoc: + { + U64 size = 0; + U64 og_read_off = off; + U64 bytes_read = str8_deserial_read_uleb128(data, off, &size); + val.u128.u64[0] = og_read_off + bytes_read; + val.u128.u64[1] = size; + off += bytes_read; + }break; + + //- rjf: flag present + case DW_Form_FlagPresent: + { + val.u128.u64[0] = 1; + }break; + } + } + + // rjf: push + if(attrib_kind != 0) + { + DW2_AttribNode *n = push_array(arena, DW2_AttribNode, 1); + SLLQueuePush(attribs.first, attribs.last, n); + attribs.count += 1; + n->v.attrib_kind = attrib_kind; + n->v.form_kind = attrib_form_kind; + } + + // rjf: no movement, or no attrib kind -> done + if(abbrev_read_off == abbrev_start_read_off || attrib_kind == 0) + { + break; + } + } + } + + // rjf: fill output + tag_out->kind = (DW_TagKind)tag_kind; + tag_out->has_children = !!has_children; + tag_out->attribs = attribs; + + // rjf: return # of bytes read + U64 bytes_read = (off - start_off); + return bytes_read; +} diff --git a/src/dwarf/dwarf_parse_2.h b/src/dwarf/dwarf_parse_2.h new file mode 100644 index 00000000..258b808a --- /dev/null +++ b/src/dwarf/dwarf_parse_2.h @@ -0,0 +1,117 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +#ifndef DWARF_PARSE_2_H +#define DWARF_PARSE_2_H + +//////////////////////////////// +//~ rjf: Unit Headers + +typedef struct DW2_UnitHeader DW2_UnitHeader; +struct DW2_UnitHeader +{ + DW_Version version; + DW_Format format; + U64 abbrev_off; + U64 addr_size; + DW_CompUnitKind kind; + U64 dwo_id; +}; + +//////////////////////////////// +//~ rjf: Abbreviation Map (ID -> .debug_abbrev Offset) + +typedef struct DW2_AbbrevMapNode DW2_AbbrevMapNode; +struct DW2_AbbrevMapNode +{ + DW2_AbbrevMapNode *next; + U64 id; + U64 off; +}; + +typedef struct DW2_AbbrevMap DW2_AbbrevMap; +struct DW2_AbbrevMap +{ + DW2_AbbrevMapNode **slots; + U64 slots_count; +}; + +//////////////////////////////// +//~ rjf: Tag Parsing Context + +typedef struct DW2_TagParseCtx DW2_TagParseCtx; +struct DW2_TagParseCtx +{ + DW_Version version; + DW_Format format; + U64 addr_size; + String8 abbrev_data; + DW2_AbbrevMap *abbrev_map; +}; + +//////////////////////////////// +//~ rjf: Tag Attributes + +typedef struct DW2_AttribVal DW2_AttribVal; +struct DW2_AttribVal +{ + DW_SectionKind section_kind; + U128 u128; +}; + +typedef struct DW2_Attrib DW2_Attrib; +struct DW2_Attrib +{ + DW_AttribKind attrib_kind; + DW_FormKind form_kind; + DW2_AttribVal val; +}; + +typedef struct DW2_AttribNode DW2_AttribNode; +struct DW2_AttribNode +{ + DW2_AttribNode *next; + DW2_Attrib v; +}; + +typedef struct DW2_AttribList DW2_AttribList; +struct DW2_AttribList +{ + DW2_AttribNode *first; + DW2_AttribNode *last; + U64 count; +}; + +//////////////////////////////// +//~ rjf: Tags + +typedef struct DW2_Tag DW2_Tag; +struct DW2_Tag +{ + DW_TagKind kind; + B32 has_children; + DW2_AttribList attribs; +}; + +//////////////////////////////// +//~ rjf: Basic Parsing Helpers + +internal U64 dw2_read_initial_length(String8 data, U64 off, U64 *out, DW_Format *fmt_out); +internal U64 dw2_read_fmt_u64(String8 data, U64 off, DW_Format format, U64 *out); + +//////////////////////////////// +//~ rjf: Unit Header Parsing + +internal U64 dw2_read_unit_header(String8 data, U64 off, DW2_UnitHeader *out); + +//////////////////////////////// +//~ rjf: Abbreviation Map Parsing + +internal DW2_AbbrevMap dw2_abbrev_map_from_data(Arena *arena, String8 data, U64 off); + +//////////////////////////////// +//~ rjf: Tag Parsing + +internal U64 dw2_read_tag(Arena *arena, DW2_TagParseCtx *ctx, String8 data, U64 off, DW2_Tag *tag_out); + +#endif // DWARF_PARSE_2_H diff --git a/src/elf/elf_parse.c b/src/elf/elf_parse.c index 77efa627..297965ff 100644 --- a/src/elf/elf_parse.c +++ b/src/elf/elf_parse.c @@ -125,13 +125,16 @@ internal U64 elf_base_addr_from_bin(ELF_Bin *bin) { U64 base_vaddr = 0; - for EachIndex(phdr_idx, bin->phdrs.count) + if(bin->hdr.e_type != ELF_Type_Dyn) { - ELF_Phdr64 *phdr = &bin->phdrs.v[phdr_idx]; - if(phdr->p_type == ELF_PType_Load && - (base_vaddr == 0 || phdr->p_vaddr < base_vaddr)) + for EachIndex(phdr_idx, bin->phdrs.count) { - base_vaddr = phdr->p_vaddr; + ELF_Phdr64 *phdr = &bin->phdrs.v[phdr_idx]; + if(phdr->p_type == ELF_PType_Load && + (base_vaddr == 0 || phdr->p_vaddr < base_vaddr)) + { + base_vaddr = phdr->p_vaddr; + } } } return base_vaddr; @@ -169,42 +172,42 @@ internal ELF_NoteList elf_parse_note(Arena *arena, String8 raw_note, ELF_Class elf_class, ELF_MachineKind e_machine) { ELF_NoteList result = {0}; - + for (U64 cursor = 0; cursor < raw_note.size; ) { U32 owner_size; U64 owner_size_size = str8_deserial_read_struct(raw_note, cursor, &owner_size); if (owner_size_size == 0) { goto exit; } cursor += owner_size_size; - + U32 desc_size; U64 desc_size_size = str8_deserial_read_struct(raw_note, cursor, &desc_size); if (desc_size_size == 0) { goto exit; } cursor += desc_size_size; - + ELF_NoteType type; U64 type_size = str8_deserial_read_struct(raw_note, cursor, &type); if (type_size == 0) { goto exit; } cursor += type_size; - + if (cursor + owner_size > raw_note.size) { goto exit; } String8 owner = str8_cstring_capped(raw_note.str + cursor, raw_note.str + cursor + owner_size); cursor += owner_size; - + if (cursor + desc_size > raw_note.size) { goto exit; } String8 desc = str8_substr(raw_note, r1u64(cursor, cursor + desc_size)); cursor += desc_size; cursor = AlignPow2(cursor, 4); - + ELF_NoteNode *n = push_array(arena, ELF_NoteNode, 1); n->v.owner = owner; n->v.desc = desc; n->v.type = type; - + SLLQueuePush(result.first, result.last, n); result.count += 1; } - -exit:; + + exit:; return result; } diff --git a/src/radbin/radbin.c b/src/radbin/radbin.c index 802b4ae7..04a74ea5 100644 --- a/src/radbin/radbin.c +++ b/src/radbin/radbin.c @@ -613,12 +613,10 @@ rb_thread_entry_point(void *p) }break; } - //- rjf: convert inputs to RDI info + //- rjf: convert DWARF in inputs to RDI info B32 convert_done = 0; - RDIM_BakeParams pdb_bake_params = {0}; RDIM_BakeParams dwarf_bake_params = {0}; { - //- rjf: PE inputs w/ DWARF, or ELF inputs => DWARF -> RDI conversion B32 pe_w_dwarf = (input_files_from_format_table[RB_FileFormat_PE].count != 0 && input_files_from_format_table[RB_FileFormat_PE].first->v->format_flags & RB_FileFormatFlag_HasDWARF); B32 elf_w_dwarf = (input_files_from_format_table[RB_FileFormat_ELF32].count != 0 || @@ -630,7 +628,103 @@ rb_thread_entry_point(void *p) else if(pe_w_dwarf) { log_infof("PEs w/ DWARF specified; converting DWARF data to RDI\n"); } else if(elf_w_dwarf) { log_infof("ELFs specified; converting DWARF data to RDI\n"); } + // rjf: input files -> exe, exe metadata, & raw dwarf + RB_File *exe_file = &rb_file_nil; + Arch arch = Arch_Null; + U64 base_vaddr = 0; + RDIM_BinarySectionList binary_sections = {0}; + DW_Raw raw = {0}; + B32 got_dwarf = 0; + { + for(RB_FileNode *n = input_files.first; n != 0; n = n->next) + { + // rjf: the first PE w/ DWARF, *or* the first ELF => pick as our exectable + if(exe_file == &rb_file_nil && + ((n->v->format == RB_FileFormat_PE && n->v->format_flags & RB_FileFormatFlag_HasDWARF) || + n->v->format == RB_FileFormat_ELF64 || + n->v->format == RB_FileFormat_ELF32)) + { + exe_file = n->v; + } + + // rjf: PE => get PE info + PE_BinInfo pe = {0}; + if(n->v->format == RB_FileFormat_PE) + { + pe = pe_bin_info_from_data(arena, n->v->data); + if(arch == Arch_Null) + { + arch = pe.arch; + } + if(base_vaddr == 0) + { + base_vaddr = pe.image_base; + } + if(binary_sections.count == 0) + { + String8 raw_sections = str8_substr(n->v->data, pe.section_table_range); + COFF_SectionHeader *section_table = str8_deserial_get_raw_ptr(raw_sections, 0, sizeof(COFF_SectionHeader)*pe.section_count); + String8 string_table = str8_substr(n->v->data, pe.string_table_range); + binary_sections = c2r_rdi_binary_sections_from_coff_sections(arena, n->v->data, string_table, pe.section_count, section_table); + } + } + + // rjf: ELF => get ELF info + ELF_Bin elf = {0}; + if(n->v->format == RB_FileFormat_ELF32 || + n->v->format == RB_FileFormat_ELF64) + { + elf = elf_bin_from_data(arena, n->v->data); + if(arch == Arch_Null) + { + arch = arch_from_elf_machine(elf.hdr.e_machine); + } + if(base_vaddr == 0) + { + base_vaddr = elf_base_addr_from_bin(&elf); + } + if(binary_sections.count == 0) + { + binary_sections = e2r_rdi_binary_sections_from_elf_section_table(arena, n->v->data, &elf, elf.shdrs); + } + } + + // rjf: PE w/ DWARF => gather DWARF sections + if(!got_dwarf && n->v->format == RB_FileFormat_PE && n->v->format_flags & RB_FileFormatFlag_HasDWARF) + { + String8 raw_sections = str8_substr(n->v->data, pe.section_table_range); + COFF_SectionHeader *section_table = str8_deserial_get_raw_ptr(raw_sections, 0, sizeof(COFF_SectionHeader)*pe.section_count); + String8 string_table = str8_substr(n->v->data, pe.string_table_range); + raw = dw_raw_from_coff_section_table(arena, n->v->data, string_table, pe.section_count, section_table); + got_dwarf = 1; + } + + // rjf: ELF w/ DWARF => gather DWARF sections + if(!got_dwarf && (n->v->format == RB_FileFormat_ELF32 || n->v->format == RB_FileFormat_ELF64) && n->v->format_flags & RB_FileFormatFlag_HasDWARF) + { + raw = dw_raw_from_elf_bin(arena, n->v->data, &elf); + got_dwarf = 1; + } + } + } + + // rjf: convert (NEW) +#if 0 + D2R2_ConvertParams convert_params = {0}; + { + convert_params.exe_name = exe_file->path; + convert_params.exe_data = exe_file->data; + convert_params.arch = arch; + convert_params.base_vaddr = base_vaddr; + convert_params.binary_sections = binary_sections; + convert_params.raw = raw; + convert_params.subset_flags = subset_flags; + convert_params.deterministic = cmd_line_has_flag(cmdline, str8_lit("deterministic")); + } + ProfScope("convert") dwarf_bake_params = d2r2_convert(arena, &convert_params); + // rjf: convert +#else D2R_ConvertParams convert_params = {0}; { B32 got_exe = 0; @@ -710,34 +804,37 @@ rb_thread_entry_point(void *p) convert_params.deterministic = cmd_line_has_flag(cmdline, str8_lit("deterministic")); } ProfScope("convert") dwarf_bake_params = d2r_convert(arena, &convert_params); +#endif } + } + lane_sync(); + + //- rjf: convert PDB in inputs to RDI info + RDIM_BakeParams pdb_bake_params = {0}; + if(input_files_from_format_table[RB_FileFormat_PDB].count != 0) + { + convert_done = 1; + log_infof("PDBs specified; converting PDB data to RDI\n"); - //- rjf: PDB inputs => PDB -> RDI conversion - if(input_files_from_format_table[RB_FileFormat_PDB].count != 0) + // rjf: get EXE/PDB file data + RB_File *exe_file = rb_file_list_first(&input_files_from_format_table[RB_FileFormat_PE]); + RB_File *pdb_file = rb_file_list_first(&input_files_from_format_table[RB_FileFormat_PDB]); + String8 exe_path = exe_file->path; + String8 pdb_path = pdb_file->path; + String8 exe_data = exe_file->data; + String8 pdb_data = pdb_file->data; + + // rjf: convert + P2R_ConvertParams convert_params = {0}; { - convert_done = 1; - log_infof("PDBs specified; converting PDB data to RDI\n"); - - // rjf: get EXE/PDB file data - RB_File *exe_file = rb_file_list_first(&input_files_from_format_table[RB_FileFormat_PE]); - RB_File *pdb_file = rb_file_list_first(&input_files_from_format_table[RB_FileFormat_PDB]); - String8 exe_path = exe_file->path; - String8 pdb_path = pdb_file->path; - String8 exe_data = exe_file->data; - String8 pdb_data = pdb_file->data; - - // rjf: convert - P2R_ConvertParams convert_params = {0}; - { - convert_params.input_pdb_name = pdb_path; - convert_params.input_exe_name = exe_path; - convert_params.input_pdb_data = pdb_data; - convert_params.input_exe_data = exe_data; - convert_params.subset_flags = subset_flags; - convert_params.deterministic = cmd_line_has_flag(cmdline, str8_lit("deterministic")); - } - ProfScope("convert") pdb_bake_params = p2r_convert(arena, &convert_params); + convert_params.input_pdb_name = pdb_path; + convert_params.input_exe_name = exe_path; + convert_params.input_pdb_data = pdb_data; + convert_params.input_exe_data = exe_data; + convert_params.subset_flags = subset_flags; + convert_params.deterministic = cmd_line_has_flag(cmdline, str8_lit("deterministic")); } + ProfScope("convert") pdb_bake_params = p2r_convert(arena, &convert_params); } lane_sync(); diff --git a/src/radbin/radbin_main.c b/src/radbin/radbin_main.c index 5b742c04..2769ef3e 100644 --- a/src/radbin/radbin_main.c +++ b/src/radbin/radbin_main.c @@ -34,6 +34,7 @@ #include "rdi_from_elf/rdi_from_elf.h" #include "rdi_from_pdb/rdi_from_pdb.h" #include "rdi_from_dwarf/rdi_from_dwarf.h" +#include "rdi_from_dwarf/rdi_from_dwarf_2.h" #include "radbin/radbin.h" //- rjf: [c] @@ -60,6 +61,7 @@ #include "rdi_from_elf/rdi_from_elf.c" #include "rdi_from_pdb/rdi_from_pdb.c" #include "rdi_from_dwarf/rdi_from_dwarf.c" +#include "rdi_from_dwarf/rdi_from_dwarf_2.c" #include "radbin/radbin.c" //////////////////////////////// diff --git a/src/rdi_from_dwarf/rdi_from_dwarf.c b/src/rdi_from_dwarf/rdi_from_dwarf.c index 760ea3dd..caa0fe21 100644 --- a/src/rdi_from_dwarf/rdi_from_dwarf.c +++ b/src/rdi_from_dwarf/rdi_from_dwarf.c @@ -2531,13 +2531,6 @@ d2r_convert(Arena *arena, D2R_ConvertParams *params) Temp scratch = scratch_begin(&arena, 1); if(lane_idx() == 0) { - //////////////////////////// - //- rjf: compute exe hash - // - ProfBegin("compute exe hash"); - U64 exe_hash = rdi_hash(params->exe_data.str, params->exe_data.size); - ProfEnd(); - //////////////////////////// //- rjf: unpack input image info // @@ -2592,6 +2585,13 @@ d2r_convert(Arena *arena, D2R_ConvertParams *params) } } + //////////////////////////// + //- rjf: compute exe hash + // + ProfBegin("compute exe hash"); + U64 exe_hash = rdi_hash(params->exe_data.str, params->exe_data.size); + ProfEnd(); + //////////////////////////// //- rjf: convert top-level-info // diff --git a/src/rdi_from_dwarf/rdi_from_dwarf_2.c b/src/rdi_from_dwarf/rdi_from_dwarf_2.c new file mode 100644 index 00000000..f83d7197 --- /dev/null +++ b/src/rdi_from_dwarf/rdi_from_dwarf_2.c @@ -0,0 +1,311 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +//////////////////////////////// +//~ rjf: Main Conversion Entry Point (New) + +internal RDIM_BakeParams +d2r2_convert(Arena *arena, D2R2_ConvertParams *params) +{ + Temp scratch = scratch_begin(&arena, 1); + DW_Raw *raw = ¶ms->raw; + RDIM_BinarySectionList binary_sections = params->binary_sections; + Arch arch = params->arch; + U64 base_vaddr = params->base_vaddr; + + //////////////////////////// + //- rjf: determine acceptable address range + // + // in many cases, linkers seem to trample over addresses in various DWARF sections, + // potentially due to optimizations. we'd like to filter out those busted addresses + // from our final debug info - a good enough heuristic is to disqualify them by + // whether or not they actually fall into the ranges covered by the binary sections. + // + Rng1U64 acceptable_vaddr_range = {0}; + { + acceptable_vaddr_range.min = max_U64; + acceptable_vaddr_range.max = 0; + for EachNode(n, RDIM_BinarySectionNode, binary_sections.first) + { + acceptable_vaddr_range.min = Min(base_vaddr + n->v.voff_first, acceptable_vaddr_range.min); + acceptable_vaddr_range.max = Max(base_vaddr + n->v.voff_opl, acceptable_vaddr_range.max); + } + } + + //////////////////////////// + //- rjf: compute exe hash + // + U64 exe_hash = 0; + ProfScope("compute exe hash") + { + if(lane_idx() == 0) + { + exe_hash = rdi_hash(params->exe_data.str, params->exe_data.size); + } + lane_sync_u64(&exe_hash, 0); + } + + //////////////////////////// + //- rjf: produce top-level-info + // + RDIM_TopLevelInfo top_level_info = {0}; + ProfScope("produce top-level-info") + { + // rjf: base arch -> rdi + RDI_Arch arch_rdi = RDI_Arch_NULL; + switch(arch) + { + case Arch_Null: + case Arch_arm64: + case Arch_arm32: + case Arch_COUNT: + {}break; + case Arch_x64:{arch_rdi = RDI_Arch_X64;}break; + case Arch_x86:{arch_rdi = RDI_Arch_X86;}break; + } + + // rjf: fill + top_level_info.arch = arch_rdi; + top_level_info.exe_name = params->exe_name; + top_level_info.exe_hash = exe_hash; + top_level_info.voff_max = acceptable_vaddr_range.max - base_vaddr; + if(!params->deterministic) + { + // TODO(rjf): top_level_info.guid = ...; + top_level_info.producer_name = str8_lit(BUILD_TITLE_STRING_LITERAL); + } + } + + //////////////////////////// + //- rjf: gather unit ranges from .debug_info + // + Rng1U64Array *unit_info_ranges = 0; + ProfScope("gather unit ranges from .debug_info") if(lane_idx() == 0) + { + Temp scratch2 = scratch_begin(&scratch.arena, 1); + String8 data = raw->sec[DW_Section_Info].data; + Rng1U64List unit_info_ranges_list = {0}; + for(U64 off = 0; off < data.size;) + { + U64 start_off = off; + + // rjf: read next unit info size + U64 unit_info_size = 0; + U64 unit_info_size_size = dw2_read_initial_length(data, off, &unit_info_size, 0); + + // rjf: push + if(unit_info_size > 0) + { + rng1u64_list_push(scratch2.arena, &unit_info_ranges_list, r1u64(off, off + unit_info_size_size + unit_info_size)); + } + + // rjf: advance + off += unit_info_size_size; + off += unit_info_size; + + // rjf: break if no movement + if(off == start_off) + { + break; + } + } + unit_info_ranges = push_array(scratch.arena, Rng1U64Array, 1); + unit_info_ranges[0] = rng1u64_array_from_list(scratch.arena, &unit_info_ranges_list); + scratch_end(scratch2); + } + lane_sync_u64(&unit_info_ranges, 0); + U64 unit_count = unit_info_ranges->count; + + //////////////////////////// + //- rjf: parse all unit headers + // + DW2_UnitHeader *unit_headers = 0; + Rng1U64 *unit_info_tag_ranges = 0; + ProfScope("parse all unit headers") + { + // rjf: set up + if(lane_idx() == 0) + { + unit_headers = push_array(scratch.arena, DW2_UnitHeader, unit_count); + unit_info_tag_ranges = push_array(scratch.arena, Rng1U64, unit_count); + } + lane_sync_u64(&unit_headers, 0); + lane_sync_u64(&unit_info_tag_ranges, 0); + + // rjf: parse all unit headers + String8 data = raw->sec[DW_Section_Info].data; + Rng1U64 range = lane_range(unit_count); + for EachInRange(idx, range) + { + Rng1U64 unit_info_range = unit_info_ranges->v[idx]; + U64 bytes_read = dw2_read_unit_header(str8_substr(data, unit_info_range), 0, &unit_headers[idx]); + unit_info_tag_ranges[idx] = r1u64(unit_info_range.min + bytes_read, unit_info_range.max); + } + } + lane_sync(); + + //////////////////////////// + //- rjf: build all abbreviation maps, build (unit -> abbrev map) + // + U64 abbrev_map_count = 0; + U64 *abbrev_map_off_from_idx_table = 0; + DW2_AbbrevMap *abbrev_map_from_idx_table = 0; + DW2_AbbrevMap **abbrev_map_from_unit_idx_table = 0; + ProfScope("build all abbreviation maps, build (unit -> abbrev map)") + { + //- rjf: gather deduplicated .debug_abbrev offsets, representing beginnings + // of abbreviation tables. we want to do this because many units may refer + // to the same abbreviation table. + // + typedef struct AbbrevOffNode AbbrevOffNode; + struct AbbrevOffNode + { + AbbrevOffNode *order_next; + AbbrevOffNode *hash_next; + U64 off; + U64 idx; + }; + U64 abbrev_off_slots_count = unit_count; + AbbrevOffNode **abbrev_off_slots = 0; + ProfScope("gather deduplicated .debug_abbrev offsets") if(lane_idx() == 0) + { + AbbrevOffNode *abbrev_off_first = 0; + AbbrevOffNode *abbrev_off_last = 0; + abbrev_off_slots = push_array(scratch.arena, AbbrevOffNode *, abbrev_off_slots_count); + for EachIndex(unit_idx, unit_count) + { + U64 abbrev_off = unit_headers[unit_idx].abbrev_off; + U64 hash = u64_hash_from_str8(str8_struct(&abbrev_off)); + U64 slot_idx = hash%abbrev_off_slots_count; + AbbrevOffNode *node = 0; + for(AbbrevOffNode *n = abbrev_off_slots[slot_idx]; n != 0; n = n->hash_next) + { + if(n->off == abbrev_off) + { + node = n; + break; + } + } + if(node == 0) + { + node = push_array(scratch.arena, AbbrevOffNode, 1); + SLLStackPush_N(abbrev_off_slots[slot_idx], node, hash_next); + SLLQueuePush_N(abbrev_off_first, abbrev_off_last, node, order_next); + node->off = abbrev_off; + node->idx = abbrev_map_count; + abbrev_map_count += 1; + } + } + abbrev_map_off_from_idx_table = push_array(scratch.arena, U64, abbrev_map_count); + abbrev_map_from_idx_table = push_array(scratch.arena, DW2_AbbrevMap, abbrev_map_count); + abbrev_map_from_unit_idx_table = push_array(scratch.arena, DW2_AbbrevMap *, unit_count); + { + U64 abbrev_map_idx = 0; + for(AbbrevOffNode *n = abbrev_off_first; n != 0; n = n->order_next) + { + abbrev_map_off_from_idx_table[abbrev_map_idx] = n->off; + abbrev_map_idx += 1; + } + } + } + lane_sync_u64(&abbrev_off_slots, 0); + lane_sync_u64(&abbrev_map_count, 0); + lane_sync_u64(&abbrev_map_off_from_idx_table, 0); + lane_sync_u64(&abbrev_map_from_idx_table, 0); + lane_sync_u64(&abbrev_map_from_unit_idx_table, 0); + + //- rjf: build all unique abbreviation maps + ProfScope("build all unique abbreviation maps") + { + String8 abbrev_data = raw->sec[DW_Section_Abbrev].data; + U64 abbrev_map_take_idx = 0; + U64 *abbrev_map_take_idx_ptr = &abbrev_map_take_idx; + lane_sync_u64(&abbrev_map_take_idx_ptr, 0); + for(;;) + { + U64 abbrev_map_idx = ins_atomic_u64_inc_eval(abbrev_map_take_idx_ptr) - 1; + if(abbrev_map_idx >= abbrev_map_count) + { + break; + } + abbrev_map_from_idx_table[abbrev_map_idx] = dw2_abbrev_map_from_data(scratch.arena, abbrev_data, abbrev_map_off_from_idx_table[abbrev_map_idx]); + } + } + + //- rjf: build unit -> abbrev map table + ProfScope("build unit -> abbrev map table") + { + Rng1U64 range = lane_range(unit_count); + for EachInRange(unit_idx, range) + { + U64 abbrev_off = unit_headers[unit_idx].abbrev_off; + U64 hash = u64_hash_from_str8(str8_struct(&abbrev_off)); + U64 slot_idx = hash%abbrev_off_slots_count; + U64 abbrev_map_idx = 0; + for(AbbrevOffNode *n = abbrev_off_slots[slot_idx]; n != 0; n = n->hash_next) + { + if(n->off == abbrev_off) + { + abbrev_map_idx = n->idx; + break; + } + } + abbrev_map_from_unit_idx_table[unit_idx] = &abbrev_map_from_idx_table[abbrev_map_idx]; + } + } + } + lane_sync(); + + //////////////////////////// + //- rjf: build per-unit tag parsing contexts + // + DW2_TagParseCtx *unit_tag_parse_ctxs = 0; + { + if(lane_idx() == 0) + { + unit_tag_parse_ctxs = push_array(scratch.arena, DW2_TagParseCtx, unit_count); + } + lane_sync_u64(&unit_tag_parse_ctxs, 0); + Rng1U64 range = lane_range(unit_count); + for EachInRange(unit_idx, range) + { + DW2_UnitHeader *hdr = &unit_headers[unit_idx]; + DW2_TagParseCtx *ctx = &unit_tag_parse_ctxs[unit_idx]; + ctx->version = hdr->version; + ctx->format = hdr->format; + ctx->addr_size = hdr->addr_size; + ctx->abbrev_data = raw->sec[DW_Section_Abbrev].data; + ctx->abbrev_map = abbrev_map_from_unit_idx_table[unit_idx]; + } + } + lane_sync(); + + //////////////////////////// + //- rjf: parse each unit's root-level tag + // + DW2_Tag *unit_root_tags = 0; + { + if(lane_idx() == 0) + { + unit_root_tags = push_array(scratch.arena, DW2_Tag, unit_count); + } + lane_sync_u64(&unit_root_tags, 0); + Rng1U64 range = lane_range(unit_count); + for EachInRange(unit_idx, range) + { + dw2_read_tag(scratch.arena, &unit_tag_parse_ctxs[unit_idx], raw->sec[DW_Section_Info].data, unit_info_tag_ranges[unit_idx].min, &unit_root_tags[unit_idx]); + } + } + lane_sync(); + + //////////////////////////// + //- rjf: fill result + // + RDIM_BakeParams result = {0}; + { + // TODO(rjf) + } + + scratch_end(scratch); + return result; +} diff --git a/src/rdi_from_dwarf/rdi_from_dwarf_2.h b/src/rdi_from_dwarf/rdi_from_dwarf_2.h new file mode 100644 index 00000000..b25d3d57 --- /dev/null +++ b/src/rdi_from_dwarf/rdi_from_dwarf_2.h @@ -0,0 +1,28 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +#ifndef RDI_FROM_DWARF_2_H +#define RDI_FROM_DWARF_2_H + +//////////////////////////////// +//~ rjf: Conversion Stage Inputs (New) + +typedef struct D2R2_ConvertParams D2R2_ConvertParams; +struct D2R2_ConvertParams +{ + String8 exe_name; + String8 exe_data; + Arch arch; + U64 base_vaddr; + RDIM_BinarySectionList binary_sections; + DW_Raw raw; + RDIM_SubsetFlags subset_flags; + B32 deterministic; +}; + +//////////////////////////////// +//~ rjf: Main Conversion Entry Point (New) + +internal RDIM_BakeParams d2r2_convert(Arena *arena, D2R2_ConvertParams *params); + +#endif // RDI_FROM_DWARF_2_H