From f6b4d9895e8a1b93b1968fb90bb22f15992318bd Mon Sep 17 00:00:00 2001 From: Ed_ Date: Tue, 21 Jul 2026 20:52:13 -0400 Subject: [PATCH] Adjustments to offset convention (don't want 1s based addresssing to mess with the spec defined encoding) --- scripts/duffle.lua | 6 +- scripts/elf_dwarf.lua | 136 ++++++++++++++--------------- scripts/passes/dwarf_injection.lua | 57 ++++++------ scripts/ps1_meta.lua | 3 +- 4 files changed, 101 insertions(+), 101 deletions(-) diff --git a/scripts/duffle.lua b/scripts/duffle.lua index 3de5c78..66542fc 100644 --- a/scripts/duffle.lua +++ b/scripts/duffle.lua @@ -228,7 +228,9 @@ end -- Section 3: I/O primitives -- ════════════════════════════════════════════════════════════════════════════ --- TODO(Ed): Review - Convert to use lfs, or remove if not utilized. +-- File contents intentionally use io.open below. LuaFileSystem handles path +-- metadata, directory iteration, the current directory, and mkdir; it does not +-- expose file-content read/write streams. function M.read_file(path) local f = io.open(path, "r") if not f then error("Cannot open " .. path) end @@ -236,14 +238,12 @@ function M.read_file(path) return content end --- TODO(Ed): Review - Convert to use lfs, or remove if not utilized. function M.write_file(path, content) local f = io.open(path, "w") if not f then error("Cannot write " .. path) end f:write(content); f:close() end --- TODO(Ed): Review - Convert to use lfs, or remove if not utilized. -- Write content to disk in binary mode so LF line endings are preserved on Windows -- (text mode would convert LF -> CRLF, breaking byte-identical diffs against git-tracked gen/*.h files which are stored as LF). -- @param path string diff --git a/scripts/elf_dwarf.lua b/scripts/elf_dwarf.lua index 6614ee4..5f86829 100644 --- a/scripts/elf_dwarf.lua +++ b/scripts/elf_dwarf.lua @@ -104,50 +104,47 @@ M.MIPS_BYTES_PER_WORD = 0x04 -- ---------------------------------------------------------------------------- -- ELF32 (System V ABI gABI v1.2) -- ---------------------------------------------------------------------------- --- All offsets are 1-INDEXED (matching Lua string.sub convention), --- expressed in hex so they map directly to the wire-format byte positions in the binary file. --- To compute the 0-indexed file offset, subtract 1. --- --- Example: e_shoff_offset = 0x21 means the 4-byte e_shoff field starts at --- string.sub byte 0x21 (= 33 in 1-indexed), i.e. file offset 0x20 (= 32). +--- **Wire-offset contract:** format offsets, fixed-width reader offsets, LEB/parser cursors, +--- and section-relative values are zero-based wire offsets. Only Lua string APIs receive +--- a `+ 1` conversion at their boundary (`byte`, `sub`, and `find`). +--- +--- ELF/DWARF field offsets are expressed in hex so they map directly to the +--- zero-based byte positions in the binary file. --- TODO(Ed): I don't like baking the 1-index offset here. It should be "handled" --- by a wrapper read/write function or inline as + 1 offset for lua index addressing compensation. --- change this unless its just better to deal with this grime here than at the addressing site. --- spec: System V ABI gABI v1.2 §"ELF Header" (Table 1) + §"Section Header Table" M.ELF32 = { - magic_offset = 0x01, -- 4-byte magic "\127ELF" at file offset 0x00 + magic_offset = 0x00, -- 4-byte magic "\127ELF" at file offset 0x00 magic = "\127ELF", - class_offset = 0x05, -- 1-byte; 1 = ELF32, 2 = ELF64 + class_offset = 0x04, -- 1-byte; 1 = ELF32, 2 = ELF64 class_elf32 = 1, - endian_offset = 0x06, -- 1-byte; 1 = little-endian, 2 = big-endian + endian_offset = 0x05, -- 1-byte; 1 = little-endian, 2 = big-endian endian_little = 1, header_bytes = 0x34, -- spec: gABI v1.2 §"ELF Header" — ELF32 header is 52 bytes total - e_shoff_offset = 0x21, -- 4-byte LE; section-header table file offset - e_shentsize_offset = 0x2F, -- 2-byte LE; section-header entry size in bytes - e_shnum_offset = 0x31, -- 2-byte LE; number of section headers - e_shstrndx_offset = 0x33, -- 2-byte LE; index of section-name string table + e_shoff_offset = 0x20, -- 4-byte LE; section-header table file offset + e_shentsize_offset = 0x2E, -- 2-byte LE; section-header entry size in bytes + e_shnum_offset = 0x30, -- 2-byte LE; number of section headers + e_shstrndx_offset = 0x32, -- 2-byte LE; index of section-name string table sh_size_bytes = 0x28, -- spec: gABI v1.2 §"Section Header Table" — each entry is 40 bytes - sh_name_offset = 0x01, -- 4-byte LE; offset into .shstrtab - sh_type_offset = 0x05, -- 4-byte LE; section type (SHT_*) - sh_offset_offset = 0x11, -- 4-byte LE; section's file offset - sh_size_offset = 0x15, -- 4-byte LE; section's size in bytes + sh_name_offset = 0x00, -- 4-byte LE; offset into .shstrtab + sh_type_offset = 0x04, -- 4-byte LE; section type (SHT_*) + sh_offset_offset = 0x10, -- 4-byte LE; section's file offset + sh_size_offset = 0x14, -- 4-byte LE; section's size in bytes dw_dwarf32_terminator = 0xFFFFFFFF, -- spec: DWARF4 spec §7.4 — 32-bit DWARF initial-length terminator } -- ---------------------------------------------------------------------------- -- DWARF4 .debug_aranges (per DWARF5 spec §7.4 — Address Range Table) -- ---------------------------------------------------------------------------- --- All offsets are 1-INDEXED (matching Lua string.sub convention), in hex. +-- All offsets are zero-based wire offsets. --- spec: DWARF5 spec §7.4 (Address Range Table) — 32-bit DWARF form M.DWARF4_ARANGES = { - unit_length_offset = 0x01, -- 4-byte LE; length of unit body (excludes these 4 bytes) - version_offset = 0x05, -- 2-byte LE; expected = 2 - cu_offset_offset = 0x07, -- 4-byte LE; CU DIE offset in .debug_info - addr_size_offset = 0x0B, -- 1-byte; expected = 4 (32-bit MIPS) - seg_size_offset = 0x0C, -- 1-byte; expected = 0 + unit_length_offset = 0x00, -- 4-byte LE; length of unit body (excludes these 4 bytes) + version_offset = 0x04, -- 2-byte LE; expected = 2 + cu_offset_offset = 0x06, -- 4-byte LE; CU DIE offset in .debug_info + addr_size_offset = 0x0A, -- 1-byte; expected = 4 (32-bit MIPS) + seg_size_offset = 0x0B, -- 1-byte; expected = 0 entry_size = 0x08, -- 4-byte addr + 4-byte length (per §7.4) terminator_size = 0x08, -- 8 zero bytes (per §7.4 end-of-list marker) version_expected = 2, @@ -158,16 +155,16 @@ M.DWARF4_ARANGES = { -- ---------------------------------------------------------------------------- -- DWARF5 .debug_rnglists (per DWARF5 spec §2.17 + §7.21) -- ---------------------------------------------------------------------------- --- All offsets are 1-INDEXED (matching Lua string.sub convention), in hex. +-- All offsets are zero-based wire offsets. --- spec: DWARF5 spec §2.17 + §7.21 (Range List Table) — 32-bit DWARF form M.DWARF5_RNGLISTS = { - unit_length_offset = 0x01, -- 4-byte LE - version_offset = 0x05, -- 2-byte LE; expected = 5 - addr_size_offset = 0x07, -- 1-byte; expected = 4 - seg_size_offset = 0x08, -- 1-byte; expected = 0 - offset_count_offset = 0x09, -- 4-byte LE; expected = 0 - first_entry_offset = 0x0D, + unit_length_offset = 0x00, -- 4-byte LE + version_offset = 0x04, -- 2-byte LE; expected = 5 + addr_size_offset = 0x06, -- 1-byte; expected = 4 + seg_size_offset = 0x07, -- 1-byte; expected = 0 + offset_count_offset = 0x08, -- 4-byte LE; expected = 0 + first_entry_offset = 0x0C, end_of_list = 0x00, -- spec: DWARF5 §7.7 — DW_RLE_end_of_list byte value start_length = 0x07, -- spec: DWARF5 §7.7 — DW_RLE_start_length byte value version_expected = 5, @@ -211,32 +208,34 @@ M.DWARF_LINE_OPS = { -- I/O helpers: little-endian byte read/write -- ════════════════════════════════════════════════════════════════════════════ ---- Read a 4-byte little-endian unsigned integer from `buf` at 1-indexed offset `off`. ---- Equivalent to `string.unpack(" {addr, size_bytes}}` for every `code_` symbol. --- --- **Conventions:** ---- - ELF32 symtab entry = 16 bytes (`st_name:4 + st_value:4 + st_size:4 + st_info:1 + st_other:1 + st_shndx:2`). 1-indexed for Lua string.sub. +--- - ELF32 symtab entry = 16 bytes (`st_name:4 + st_value:4 + st_size:4 + st_info:1 + st_other:1 + st_shndx:2`); offsets within each entry are zero-based wire offsets. +--- - Direct Lua `string.byte`/`string.sub`/`string.find` boundaries receive `+ 1`. --- - We filter on STB_GLOBAL (high nibble of st_info = 1) to match `nm`'s default (external symbols only). STB_WEAK excluded. --- - We strip the `code_` prefix to match the previous `read_nm` output. --- - `st_size > 0` filter excludes undefined/imported symbols. @@ -549,31 +549,31 @@ function M.read_nm(elf_path) end -- Iterate the 16-byte ELF32 symtab entries. - -- Each entry (1-indexed): st_name at 1, st_value at 5, st_size at 9, st_info at 13, st_other at 14, st_shndx at 15. + -- Each entry (zero-based): st_name at 0, st_value at 4, st_size at 8, st_info at 12, st_other at 13, st_shndx at 14. local SYM_ENTRY_BYTES = 0x10 - local SYM_ST_NAME = 0x01 - local SYM_ST_VALUE = 0x05 - local SYM_ST_SIZE = 0x09 - local SYM_ST_INFO = 0x0D + local SYM_ST_NAME = 0x00 + local SYM_ST_VALUE = 0x04 + local SYM_ST_SIZE = 0x08 + local SYM_ST_INFO = 0x0C local n_syms = #symtab / SYM_ENTRY_BYTES for i = 0, n_syms - 1 do - local entry_off = i * SYM_ENTRY_BYTES + 1 -- 1-indexed - local st_info = symtab:byte(entry_off + SYM_ST_INFO - 1) + local entry_off = i * SYM_ENTRY_BYTES + local st_info = symtab:byte(entry_off + SYM_ST_INFO + 1) -- High nibble = binding (STB_LOCAL=0, STB_GLOBAL=1, STB_WEAK=2). -- Use math.floor(/16) instead of bit.rshift for LuaJIT 2.1 compat -- (LuaJIT's `>>` is 5.3+, but math.floor(x/16) works on all versions). local binding = math.floor(st_info / 16) if binding == 0 or binding == 1 then -- STB_LOCAL or STB_GLOBAL - local st_size = M.read_u32_le(symtab, entry_off + SYM_ST_SIZE - 1) + local st_size = M.read_u32_le(symtab, entry_off + SYM_ST_SIZE) if st_size > 0 then - local st_name_off = M.read_u32_le(symtab, entry_off + SYM_ST_NAME - 1) + local st_name_off = M.read_u32_le(symtab, entry_off + SYM_ST_NAME) -- Extract the name from .strtab (null-terminated C string). local name_end = strtab:find("\0", st_name_off + 1, true) or (st_name_off + 1) local name = strtab:sub(st_name_off + 1, name_end - 1) -- Filter: keep all symbol-table symbols (atoms emit their name as the bare `` since the `code_` prefix was removed from the MipsAtom_ macro). -- The atoms_source_map pass already filters out non-atom symbols via the source-map.txt cross-ref. if name and #name > 0 then - local st_value = M.read_u32_le(symtab, entry_off + SYM_ST_VALUE - 1) + local st_value = M.read_u32_le(symtab, entry_off + SYM_ST_VALUE) addrs[name] = { st_value, st_size } end end diff --git a/scripts/passes/dwarf_injection.lua b/scripts/passes/dwarf_injection.lua index c4cc7b8..2023ad7 100644 --- a/scripts/passes/dwarf_injection.lua +++ b/scripts/passes/dwarf_injection.lua @@ -1035,24 +1035,24 @@ local function build_dwarf_line_section(existing, atom_table) -- Walk DWARF32 line units and retain the final unit's bounds. -- The main C CU points at this final unit (DW_AT_stmt_list = 0x5b in today's ELF). - local unit_pos, last_pos, last_length, last_end = 1, nil, nil, nil - while unit_pos <= #existing do - if unit_pos + 3 > #existing then return existing end - local unit_length = elf_dwarf.read_u32_le(existing, unit_pos) + local unit_pos, last_pos, last_length, last_end = 0, nil, nil, nil + while unit_pos < #existing do + if unit_pos + 4 > #existing then return existing end + local unit_length = elf_dwarf.read_u32_le(existing, unit_pos) if unit_length == elf_dwarf.ELF32.dw_dwarf32_terminator then return existing end - local unit_end = unit_pos + 3 + unit_length - if unit_end > #existing then return existing end - last_pos, last_length, last_end = unit_pos, unit_length, unit_end - unit_pos = unit_end + 1 + local unit_end_excl = unit_pos + 4 + unit_length + if unit_end_excl > #existing then return existing end + last_pos, last_length, last_end = unit_pos, unit_length, unit_end_excl + unit_pos = unit_end_excl end - if unit_pos ~= #existing + 1 or not last_pos then return existing end + if unit_pos ~= #existing or not last_pos then return existing end local new_length = last_length + #appended local new_length_bytes = elf_dwarf.write_u32_le(new_length) - return existing:sub(1, last_pos - 1) + return existing:sub(1, last_pos) .. new_length_bytes - .. existing:sub(last_pos + 4, last_end) + .. existing:sub(last_pos + 5, last_end) .. appended end @@ -1091,10 +1091,10 @@ local function build_dwarf_aranges_section(existing, atom_table) -- Walk all units and emit each one (preserving existing structure). -- For the LAST unit, replace the terminator with my entries + new term. local result = {} - local i = 1 -- 1-indexed + local i = 0 -- zero-based wire offset local is_last_unit = false - while i <= #existing do + while i < #existing do -- Read this unit's length. local ul = elf_dwarf.read_u32_le(existing, i) if ul == elf_dwarf.ELF32.dw_dwarf32_terminator then @@ -1103,8 +1103,8 @@ local function build_dwarf_aranges_section(existing, atom_table) end local unit_start = i - local unit_end = i + 3 + ul -- last byte of unit content - is_last_unit = (unit_end == #existing) + local unit_end_excl = i + 4 + ul + is_last_unit = (unit_end_excl == #existing) if is_last_unit then -- The old terminator is replaced by entries + a new terminator, so net section growth (and unit_length growth) is entries only. @@ -1113,7 +1113,7 @@ local function build_dwarf_aranges_section(existing, atom_table) local new_ul_bytes = elf_dwarf.write_u32_le(new_ul) -- Emit everything EXCEPT the last 8 bytes (terminator). result[#result + 1] = new_ul_bytes - .. existing:sub(i + 4, unit_end - elf_dwarf.DWARF4_ARANGES.terminator_size) + .. existing:sub(i + 5, unit_end_excl - elf_dwarf.DWARF4_ARANGES.terminator_size) -- Append my atom entries. for _, atom in ipairs(atom_table) do local a = atom.addr @@ -1124,10 +1124,10 @@ local function build_dwarf_aranges_section(existing, atom_table) result[#result + 1] = string.rep("\0", elf_dwarf.DWARF4_ARANGES.terminator_size) else -- Emit this unit unchanged. - result[#result + 1] = existing:sub(unit_start, unit_end) + result[#result + 1] = existing:sub(unit_start + 1, unit_end_excl) end - i = unit_end + 1 + i = unit_end_excl end if not is_last_unit then @@ -1150,12 +1150,12 @@ end --- @param atom_table table --- @return string local function build_dwarf_rnglists_section(existing, atom_table) - if #existing < elf_dwarf.DWARF5_RNGLISTS.first_entry_offset or #atom_table == 0 then return existing end + if #existing <= elf_dwarf.DWARF5_RNGLISTS.first_entry_offset or #atom_table == 0 then return existing end local unit_length = elf_dwarf.read_u32_le(existing, elf_dwarf.DWARF5_RNGLISTS.unit_length_offset) local version = elf_dwarf.read_u16_le(existing, elf_dwarf.DWARF5_RNGLISTS.version_offset) - local address_size = existing:byte(elf_dwarf.DWARF5_RNGLISTS.addr_size_offset) - local segment_size = existing:byte(elf_dwarf.DWARF5_RNGLISTS.seg_size_offset) + local address_size = existing:byte(elf_dwarf.DWARF5_RNGLISTS.addr_size_offset + 1) + local segment_size = existing:byte(elf_dwarf.DWARF5_RNGLISTS.seg_size_offset + 1) local offset_entry_count = elf_dwarf.read_u32_le(existing, elf_dwarf.DWARF5_RNGLISTS.offset_count_offset) if unit_length + 4 ~= #existing @@ -1251,8 +1251,7 @@ end -- 4. Patch the main CU's debug_abbrev_offset to the duplicate. -- 5. Insert our DIEs as children of the main CU (just before its root terminator). -- --- All offsets in this section are 0-based (matching the user's "section offset 0x24" convention). --- When reading via elf_dwarf.read_u32_le (1-indexed), call with `pos + 1`. +-- All binary coordinates in this section are zero-based wire offsets. Direct Lua string APIs add `+ 1` at the boundary. -- -- Pure-Lua 5.3 LEB decoders (no `bit` library; `2^shift` arithmetic matches elf_dwarf.lua's -- "math.floor(/16) instead of bit.rshift for LuaJIT 2.1 compat" comment at line 352). @@ -1289,7 +1288,7 @@ local function find_main_cu_layout(existing) local main_cu_start = nil local main_cu_end_excl = nil while pos + 4 <= buf_len do - local unit_length = elf_dwarf.read_u32_le(existing, pos + 1) + local unit_length = elf_dwarf.read_u32_le(existing, pos) if unit_length == DWARF32_TERMINATOR then return nil end local unit_end_excl = pos + 4 + unit_length if unit_end_excl > buf_len then return nil end @@ -1308,10 +1307,10 @@ local function find_main_cu_layout(existing) -- [7] address_size -- [8..11] debug_abbrev_offset local hdr = main_cu_start + 4 - local version = elf_dwarf.read_u16_le(existing, hdr + 1) - local unit_type = existing:byte(hdr + 3) - local address_size = existing:byte(hdr + 4) - local abbrev_off = elf_dwarf.read_u32_le(existing, hdr + 5) + local version = elf_dwarf.read_u16_le(existing, hdr) + local unit_type = existing:byte(hdr + 2 + 1) + local address_size = existing:byte(hdr + 3 + 1) + local abbrev_off = elf_dwarf.read_u32_le(existing, hdr + 4) if version ~= DW_VERSION_5 or unit_type ~= DW_UT_compile or address_size ~= 4 then return nil end @@ -2167,7 +2166,7 @@ local function build_debug_info_section(existing, main_cu_start, main_cu_end_exc local inserted_len = #inserted -- 2) Patch main CU's unit_length += inserted_len. - local old_unit_length = elf_dwarf.read_u32_le(existing, main_cu_start + 1) + local old_unit_length = elf_dwarf.read_u32_le(existing, main_cu_start) local new_unit_length = old_unit_length + inserted_len local new_unit_length_bytes = elf_dwarf.write_u32_le(new_unit_length) diff --git a/scripts/ps1_meta.lua b/scripts/ps1_meta.lua index 0c2ed29..bd3cf4d 100644 --- a/scripts/ps1_meta.lua +++ b/scripts/ps1_meta.lua @@ -390,7 +390,8 @@ end local function build_ctx(args) local sources = {} for _, path in ipairs(args.sources) do - -- TODO(Ed): io.open?? why not lfs? + -- lfs handles path metadata and directories, not file-content streams. + -- Keep this io.open local so this entry point preserves its tailored diagnostic and exit path below. local f = io.open(path, "r") if not f then io.stderr:write("ps1_meta: cannot open --source " .. path .. "\n")