From 2b6fe53ce88e15d532d1a1f009eca71bd00cb4a8 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Thu, 6 Aug 2026 10:41:45 -0400 Subject: [PATCH] Stuff kept from hot-reload attempt --- .gitignore | 1 + code/duffle/lottes_tape.h | 38 +-- scripts/build_psyq.ps1 | 16 +- scripts/elf32.lua | 418 +++++++++++++++++++++++++++++ scripts/elf_dwarf.lua | 293 ++++++++++---------- scripts/passes/dwarf_injection.lua | 4 +- 6 files changed, 599 insertions(+), 171 deletions(-) create mode 100644 scripts/elf32.lua diff --git a/.gitignore b/.gitignore index a9f6298..555f412 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ toolchain/lpeg scratch toolchain/libpsn00b +scripts/pcsx_debug_helper.zip diff --git a/code/duffle/lottes_tape.h b/code/duffle/lottes_tape.h index dd1f132..c28ded6 100644 --- a/code/duffle/lottes_tape.h +++ b/code/duffle/lottes_tape.h @@ -31,10 +31,11 @@ * From here various conventions can be further applied. * To make things easier to understand it may be better to focus on what this * ABI does not have. It does not have have any branching within the tape but - * relative branches between atoms. Branching nearly is always downstream. - * Stack usage is non-existent. Push/Pop, FIFO, or Arena/Bump data structures - * are used by atoms explicitly. In it's current form withe C11 macro dsl, - * the user also has to do manual register allocation per atom. + * relative branches within atoms or between atoms. + * Branching nearly is always downstream. Stack usage is non-existent. + * Push/Pop, FIFO, or Arena/Bump data structures are used by atoms explicitly. + * In it's current form withe C11 macro dsl, the user also has to do manual register + * allocation per atom. * * One of the remarkable things about utilizing this abi is its essentially * interopable with CPUs, GPUs, FPGA, or, basically anything @@ -43,37 +44,38 @@ * in order to execute digital logic effectively on current era tech. * On the PS1 we don't have access to a few features like multi-threading, * speculative execution, or L3 cache; but, we can set the foundation for legoing - * whats required baseline wise for eventually expanding the harness and core atoms + * whats required for eventually expanding this ABI's paradigm and core atoms * to take those newer hardware features into account. For example, you can easily * expand this to support wave-based execution model on a PS2 or PS3. * Not having a stack or automatic register allocation means the user can't ignore * excessive argument shuffle across workload or waves and thier phases. - * Crossing ABI boundaries to other runtimes that do has an obviouss penalties. + * Crossing ABI boundaries to other runtimes that do has obviouss penalties. * * Learning data-oreinted code becomes a natural progression. Your not fighting - * a stack-based procedural paradigm that wants to argument shuffle on the stack - * by lack of constraints on how the user may "call" a procedure. The user doesn't + * a stack-based procedural paradigm that wants to argument shuffle on by lack of + * constraints on how the user may "call" a procedure. The user doesn't * have to hammer down "rules" or patterns to know how to massage the compiler - * to get the asesmbly into its natural form. The form is obvious, and once - * the user gets to author their compoonents it becomes a game of tetris. + * to dissolve those call frames to get the asesmbly into its desired form. + * The form is obvious, and once the user gets to author these compoonents + * it becomes a game of tetris. * * Another feature is this ABI is very compatible with bootstrapping and developing * simple toolchains built off of bit-packed annotated command streams the user can - * directly author, maintatain, and immediately execute. That being a color forth. + * directly author, maintatain, and immediately execute. That being like a color forth. * This can make the tetris less of a chore with some helpful policy generation for * allocation of registers, helping to choose resuable components, designing DSL on * the fly, etc. * ----------------------------------------------------------------------------- - * TODO(Ed): We ned pretty ascii diagrams and proper guides, articles, etc. + * TODO(Ed): We need pretty ascii diagrams and proper guides, articles, etc. * ----------------------------------------------------------------------------- - * For now this thing is just functioning and I'm abusing C11 + a lua metaprogram + * For now this ideation is just started functioning. I'm abusing C11 & a lua metaprogram * to help establish a hybrid toolchain to ideate on a traditional text-based * authoring UX for this paradigm. - * If pcsx-redux gets me viable hot-reload and persistent data storage beyond - * save-states (just copying ram to filesystem). I can author a color forth to - * mess around with, with an editor in-emulator or on the actual machine itself. - * Assembly is tedius, but I think this codebase most likely has some of the most, - * ergonomic you can come across.. + * If pcsx-redux provides viable hot-reload and persistent data storage beyond + * save-states (just copying ram to filesystem), I can author a color forth to + * mess around with. With either an editor in-emulator or on the actual machine itself. + * Assembly is tedius, but I think this codebase most likely has a pretty ergonomic + * flavor worst case... * */ /* Register Allocation Info */ enum { diff --git a/scripts/build_psyq.ps1 b/scripts/build_psyq.ps1 index 2c8321a..2d424c3 100644 --- a/scripts/build_psyq.ps1 +++ b/scripts/build_psyq.ps1 @@ -180,12 +180,9 @@ function link-modules { param([string[]]$link_modules, [string] $elf, [string[] $link_args += ($f_link_pass_through_prefix + $f_link_mapfile + $map) $link_args += ($f_link_pass_through_prefix + $f_link_start_group) - # raw_sio_pad_poll_20260802 — Task 5.1c surgical library-list trim. - # The 16 removed entries (c2, card, cd, comb, ds, gs, gun, hmd, math, - # mcrd, mcx, press, sio, snd, spu, tap) had LOAD lines in the map but - # ZERO .o files pulled in — they were unused. The 5 kept libraries - # (api, c, etc, gpu, gte) are required by the C-side calls in - # hello_joypad.c (reset_graph, draw_sync, vsync, etc.). + # 16 removed entries (c2, card, cd, comb, ds, gs, gun, hmd, math, mcrd, mcx, press, sio, snd, spu, tap) + # had LOAD lines in the map but ZERO .o files pulled in — they were unused. + # 5 kept libraries (api, c, etc, gpu, gte) are required by the C-side calls in hello_joypad.c (reset_graph, draw_sync, vsync, etc.). $libraries = @( "api", "c", @@ -227,9 +224,7 @@ function ps1-meta { param( [string[]]$passes = @('--pre-link'), [string[]]$extra_args = @() ) - # `--unity-root` and `--source` are - # mutually exclusive. Exactly one of `$unity_root` / `$sources` must - # be supplied; the other must be absent. + # `--unity-root` and `--source` are mutually exclusive. Exactly one of `$unity_root` / `$sources` must be supplied; the other must be absent. if ($null -ne $unity_root -and $unity_root -ne '') { if ($null -ne $sources -and $sources.Count -gt 0) { @@ -522,7 +517,7 @@ function build-hello_camera { $path_build_gen = join-path $path_build 'gen' $src_c = join-path $path_module 'hello_camera.c' - ps1-meta -unity_root $src_c -metadata $path_atom_metadata -out_root $path_build_gen + ps1-meta -unity_root $src_c -metadata $path_atom_metadata -out_root $path_build_gen -passes @('--pre-link') $assemble_args = @() $assemble_args += $f_debug @@ -557,7 +552,6 @@ function build-hello_camera { link-modules $link_modules $elf $link_args make-binary $elf $exe - # Post-link: gdb-runtime + dwarf-injection in a single Lua invocation (one luajit cold start). ps1-meta -unity_root $src_c -metadata $path_atom_metadata -out_root $path_build_gen -passes @('--post-link') ` -extra_args @('--elf', $elf) inject-dwarf $elf $path_build_gen diff --git a/scripts/elf32.lua b/scripts/elf32.lua new file mode 100644 index 0000000..a92144d --- /dev/null +++ b/scripts/elf32.lua @@ -0,0 +1,418 @@ +-- elf32.lua — Pure-Lua ELF32 format helpers with no lfs / no lpeg dependency. +-- The reload helper's `parse_manifest` (scripts/pcsx_debug_helper/reload.lua) +-- and the metaprogram's `read_elf_sections` + `read_nm` (scripts/elf_dwarf.lua) +-- both parsed ELF32 headers from wire bytes. +-- +-- This module contains the format constants and the byte-level walker. +--- The metaprogram side keeps `read_u32_le` / `read_u16_le` as local forwarders; the helper side calls `E.*` directly. +-- +-- **Adapter contract (explicit pass style):** +-- The helper VM's `Support.File` exposes byte-read methods that require `self` (fileffi.lua:225-227), +-- so callers wrap once in a 1-line adapter that strips `self`. +-- The parsers here operate on the unwrapped form. +-- Reads are flat function calls — `E.read_u8(adapter, off)`, `E.read_u32(adapter, off)`, `E.size(adapter)`. +-- read_u8(adapter, off) -> integer | nil +-- read_u16(adapter, off) -> integer | nil +-- read_u32(adapter, off) -> integer | nil +-- size(adapter) -> integer +-- +-- **Convention:** every offset in the constants tables is a zero-based wire offset. +-- The `+ 1` conversion happens only at the `string.byte` boundary inside the readers. +-- +-- spec: System V ABI gABI v1.2 §"ELF Header" (Table 1) + §"Section Header Table" +-- spec: System V ABI gABI v1.2 §"Symbol Table" (Elf32_Sym layout) + +local M = {} + +-- ════════════════════════════════════════════════════════════════════════════ +-- Little-endian readers (bit-weighted accumulator, math.floor only) +-- ════════════════════════════════════════════════════════════════════════════ + +--- Read a 4-byte little-endian unsigned integer from `adapter` at zero-based wire offset `off`. +--- +--- Bit weights are written as `0x100`, `0x10000`, `0x1000000` (i.e. 2^8, 2^16, 2^24) so the LE byte positions are visually explicit: +--- byte 0 contributes its value directly; +--- byte 1 is shifted left by 8; byte 2 by 16; byte 3 by 24. +--- +--- math.floor (not LuaJIT's `>>`) keeps the body portable across LuaJIT 2.0/2.1 and plain Lua 5.x. `string.byte` receives `+ 1` at the boundary. +--- +--- **Call form:** explicit-pass. The reader receives `adapter` as the first positional argument and the offset as the second; no `self` is passed. +--- Test fixtures declare `function(offset) ... end` and the parsers call them via dot syntax `adapter.read_u8_at(off)`. +--- The colon form `adapter:read_u8_at(off)` would prepend the adapter table as `offset` and break the contract. +--- @param adapter table +--- @param off integer -- zero-based wire offset +--- @return integer|nil +function M.read_u32(adapter, off) + return adapter.read_u8_at(off) + + adapter.read_u8_at(off + 0x01) * 0x00000100 + + adapter.read_u8_at(off + 0x02) * 0x00010000 + + adapter.read_u8_at(off + 0x03) * 0x01000000 +end + +--- Read a 2-byte little-endian unsigned integer from `adapter` at zero-based wire offset `off`. +--- @param adapter table +--- @param off integer -- zero-based wire offset +--- @return integer|nil +function M.read_u16(adapter, off) + return adapter.read_u8_at(off) + + adapter.read_u8_at(off + 0x01) * 0x00000100 +end + +--- Read a 1-byte unsigned integer from `adapter` at zero-based wire offset `off`. +--- @param adapter table +--- @param off integer -- zero-based wire offset +--- @return integer|nil +function M.read_u8(adapter, off) + return adapter.read_u8_at(off) +end + +--- Total adapter byte length. +--- @param adapter table +--- @return integer +function M.size(adapter) + return adapter.read_size() +end + +--- Forwarders kept for backward compat with scripts/elf_dwarf.lua. +--- The metaprogram side keeps `read_u32_le` / `read_u16_le`; +--- both layers now use the same byte-level helpers under the hood. +function M.read_u32_le(buf, off) + local byte_off = off + 1 + return buf:byte(byte_off) + + buf:byte(byte_off + 0x01) * 0x00000100 + + buf:byte(byte_off + 0x02) * 0x00010000 + + buf:byte(byte_off + 0x03) * 0x01000000 +end + +--- Read a 2-byte little-endian unsigned integer from `buf` at zero-based wire offset `off`. +--- @param buf string +--- @param off integer -- zero-based wire offset +--- @return integer +function M.read_u16_le(buf, off) + local byte_off = off + 1 + return buf:byte(byte_off) + buf:byte(byte_off + 0x01) * 0x00000100 +end + +-- ════════════════════════════════════════════════════════════════════════════ +-- Format constants +-- ════════════════════════════════════════════════════════════════════════════ + +-- ELF format constants (System V ABI gABI v1.2). +M.ELFCLASS32 = 1 -- spec: gABI v1.2 §"ELF Header" — EI_CLASS byte +M.ELFDATA2LSB = 1 -- spec: gABI v1.2 §"ELF Header" — EI_DATA byte +M.EM_MIPS = 8 -- spec: gABI v1.2 §"Machine Information" — MIPS architecture + +-- Section type constants (System V ABI gABI v1.2 §"Section Header Table"). +M.SHT_SYMTAB = 2 -- spec: gABI v1.2 §"Section Types" — symbol table +M.SHT_STRTAB = 3 -- spec: gABI v1.2 §"Section Types" — string table +M.SHT_NOBITS = 8 -- spec: gABI v1.2 §"Section Types" — no space in file + +-- Section flag constants (System V ABI gABI v1.2 §"Section Header Table"). +M.SHF_WRITE = 0x1 -- spec: gABI v1.2 §"Section Attributes" — writable +M.SHF_ALLOC = 0x2 -- spec: gABI v1.2 §"Section Attributes" — occupies memory +M.SHF_EXECINSTR = 0x4 -- spec: gABI v1.2 §"Section Attributes" — executable + +-- --------------------------------------------------------------------------- +-- ELF32 header layout (System V ABI gABI v1.2 §"ELF Header" Table 1) +-- --------------------------------------------------------------------------- +-- All offsets are zero-based wire offsets. The header is 52 bytes total (header_bytes = 0x34 = 52). +M.ELF32_HEADER = { + magic_offset = 0x00, -- 4 bytes; expected "\127ELF" + magic = "\127ELF", + class_offset = 0x04, -- 1 byte; 1 = ELF32, 2 = ELF64 + endian_offset = 0x05, -- 1 byte; 1 = little-endian, 2 = big-endian + header_bytes = 0x34, -- ELF32 header is 52 bytes total + e_entry_offset = 0x18, -- 4-byte LE; entry-point virtual address + e_shoff_offset = 0x20, -- 4-byte LE; section-header table file offset + e_shentsize_offset = 0x2E, -- 2-byte LE; section-header entry size in bytes + e_shnum_offset = 0x30, -- 2-byte LE; number of section headers + e_shstrndx_offset = 0x32, -- 2-byte LE; index of section-name string table +} + +-- --------------------------------------------------------------------------- +-- ELF32 section-header layout (System V ABI gABI v1.2 §"Section Header Table") +-- --------------------------------------------------------------------------- +-- Each entry is 40 bytes (sh_entsize_bytes = 0x28 = 40); +-- zero-based, field offsets relative to the start of the entry. +M.ELF32_SECTION = { + sh_name_offset = 0x00, -- 4-byte LE; offset into .shstrtab + sh_type_offset = 0x04, -- 4-byte LE; section type (SHT_*) + sh_flags_offset = 0x08, -- 4-byte LE; section flags (SHF_*) + sh_addr_offset = 0x0C, -- 4-byte LE; virtual address at execution + sh_offset_offset = 0x10, -- 4-byte LE; section's file offset + sh_size_offset = 0x14, -- 4-byte LE; section's size in bytes + sh_link_offset = 0x18, -- 4-byte LE; link to a related section + sh_entsize_bytes = 0x28, -- spec: gABI v1.2 §"Section Header Table" — 40 bytes per entry +} + +-- --------------------------------------------------------------------------- +-- ELF32 symbol-table entry layout (System V ABI gABI v1.2 §"Symbol Table") +-- --------------------------------------------------------------------------- +-- Each entry is 16 bytes (sym_entry_bytes = 0x10 = 16); +-- zero-based, field offsets relative to the start of the entry. +M.ELF32_SYM = { + st_name = 0x00, -- 4-byte LE; offset into the linked string table + st_value = 0x04, -- 4-byte LE; symbol value (address / absolute) + st_size = 0x08, -- 4-byte LE; symbol size in bytes + st_info = 0x0C, -- 1 byte; binding (high nibble) + type (low nibble) + sym_entry_bytes = 0x10, -- spec: gABI v1.2 §"Symbol Table" — 16 bytes per entry +} + +-- DWARF32 initial-length terminator (DWARF4 §7.4) — kept here so the metaprogram's elf_dwarf.lua can drop its own copy of the same constant. +M.dw_dwarf32_terminator = 0xFFFFFFFF + +-- ════════════════════════════════════════════════════════════════════════════ +-- Adapter validation +-- ════════════════════════════════════════════════════════════════════════════ + +--- Validate that `adapter` exposes the byte-read surface. +--- Returns true on success, false + a stable error code on failure. +--- The helper side calls this before parse_manifest to reject callers before any byte is read. +--- @param adapter any +--- @return boolean, string|nil +function M.validate_adapter(adapter) + if type(adapter) ~= "table" then return false, "bad_file_adapter" end + if type(adapter.read_u8_at) ~= "function" then return false, "bad_file_adapter" end + if type(adapter.read_u16_at) ~= "function" then return false, "bad_file_adapter" end + if type(adapter.read_u32_at) ~= "function" then return false, "bad_file_adapter" end + if type(adapter.read_size) ~= "function" then return false, "bad_file_adapter" end + return true, nil +end + +-- ════════════════════════════════════════════════════════════════════════════ +-- String-table reader +-- ════════════════════════════════════════════════════════════════════════════ + +--- Extract a NUL-terminated C string from `strtab` at zero-based offset `off`. +--- Returns nil if `off` is out of range or the string is not NUL-terminated. +--- @param strtab string +--- @param off integer +--- @return string|nil +function M.get_str(strtab, off) + if off < 0 or off >= #strtab then return nil end + local end_pos = strtab:find("\0", off + 1, true) + if not end_pos then return nil end + return strtab:sub(off + 1, end_pos - 1) +end + +-- ════════════════════════════════════════════════════════════════════════════ +-- Header / section / symbol walkers +-- ════════════════════════════════════════════════════════════════════════════ + +--- Read the ELF32 header through `adapter` and validate the magic, class, and data encoding. +--- Returns a table on success: +--- { e_entry, e_shoff, e_shentsize, e_shnum, e_shstrndx, error = nil } +--- On failure returns nil + a stable error code: +--- bad_magic, unsupported_elf_class, unsupported_elf_data, truncated_header +--- The header's machine field is NOT validated here — callers (e.g. the helper's prime path) decide whether to require EM_MIPS before symbol reads. +--- @param adapter table +--- @return table|nil, string|nil +function M.parse_elf32_headers(adapter) + local ok, err = M.validate_adapter(adapter) + if not ok then return nil, err end + + -- 4-byte magic: 0x7F 'E' 'L' 'F'. + -- The byte readers take the adapter explicitly. + -- The production `Support.File` adapter is wrapped by the caller to drop its implicit `self` so the parser shape is flat pass-style. + local b1 = M.read_u8(adapter, 0) + local b2 = M.read_u8(adapter, 1) + local b3 = M.read_u8(adapter, 2) + local b4 = M.read_u8(adapter, 3) + if not (b1 and b2 and b3 and b4) + or not (b1 == 0x7f and b2 == 0x45 and b3 == 0x4c and b4 == 0x46) then + return nil, "bad_magic" + end + + local class = M.read_u8(adapter, M.ELF32_HEADER.class_offset) + if class ~= M.ELFCLASS32 then + return nil, "unsupported_elf_class" + end + + local data = M.read_u8(adapter, M.ELF32_HEADER.endian_offset) + if data ~= M.ELFDATA2LSB then + return nil, "unsupported_elf_data" + end + + local e_entry = M.read_u32(adapter, M.ELF32_HEADER.e_entry_offset) + local e_shoff = M.read_u32(adapter, M.ELF32_HEADER.e_shoff_offset) + local e_shentsize = M.read_u16(adapter, M.ELF32_HEADER.e_shentsize_offset) + local e_shnum = M.read_u16(adapter, M.ELF32_HEADER.e_shnum_offset) + local e_shstrndx = M.read_u16(adapter, M.ELF32_HEADER.e_shstrndx_offset) + if not (e_entry and e_shoff and e_shentsize and e_shnum and e_shstrndx) then + return nil, "truncated_header" + end + + return { + e_entry = e_entry, + e_shoff = e_shoff, + e_shentsize = e_shentsize, + e_shnum = e_shnum, + e_shstrndx = e_shstrndx, + error = nil, + } +end + +--- Read one section-header entry from `adapter` at `sh_off`. +--- Returns a table with the wire fields plus a (yet-unresolved) `name` field. +--- @param adapter table +--- @param sh_off integer +--- @return table|nil, string|nil -- entry, error +local function read_section_entry(adapter, sh_off) + local entry = { + sh_name = M.read_u32(adapter, sh_off + M.ELF32_SECTION.sh_name_offset), + sh_type = M.read_u32(adapter, sh_off + M.ELF32_SECTION.sh_type_offset), + sh_flags = M.read_u32(adapter, sh_off + M.ELF32_SECTION.sh_flags_offset), + sh_addr = M.read_u32(adapter, sh_off + M.ELF32_SECTION.sh_addr_offset), + sh_offset = M.read_u32(adapter, sh_off + M.ELF32_SECTION.sh_offset_offset), + sh_size = M.read_u32(adapter, sh_off + M.ELF32_SECTION.sh_size_offset), + sh_link = M.read_u32(adapter, sh_off + M.ELF32_SECTION.sh_link_offset), + name = "", + } + if not (entry.sh_name and entry.sh_type and entry.sh_flags and entry.sh_addr + and entry.sh_offset and entry.sh_size and entry.sh_link) then + return nil, "truncated_section_headers" + end + return entry, nil +end + +--- Walk every section header in `hdr` and return a 1-based array of entries +--- (the section at logical index 0 is at array position 1, etc.). +--- Each entry has the wire fields plus a resolved `name` derived from `.shstrtab`. +--- Returns nil + a stable error code on failure: truncated_section_headers, missing_shstrtab, truncated_strtab +--- @param adapter table +--- @param hdr table -- the table returned by parse_elf32_headers +--- @return table|nil, string|nil +function M.walk_sections(adapter, hdr) + if not hdr or hdr.error then return nil, hdr and hdr.error or "truncated_section_headers" end + + local file_size = M.size(adapter) + if hdr.e_shoff + hdr.e_shnum * hdr.e_shentsize > file_size then + return nil, "truncated_section_headers" + end + + -- Read every section header first; we need .shstrtab to resolve names. + local sections = {} + for i = 0, hdr.e_shnum - 1 do + local sh_off = hdr.e_shoff + i * hdr.e_shentsize + local entry, err = read_section_entry(adapter, sh_off) + if not entry then return nil, err end + sections[i + 1] = entry + end + + if hdr.e_shstrndx >= hdr.e_shnum then + return nil, "missing_shstrtab" + end + + local shstrtab = sections[hdr.e_shstrndx + 1] + if not shstrtab or shstrtab.sh_type ~= M.SHT_STRTAB then + return nil, "missing_shstrtab" + end + if shstrtab.sh_offset + shstrtab.sh_size > file_size then + return nil, "truncated_section_headers" + end + local shstrtab_bytes = M.read_section_bytes(adapter, shstrtab) + if not shstrtab_bytes then return nil, "truncated_section_headers" end + + for _, s in ipairs(sections) do + s.name = M.get_str(shstrtab_bytes, s.sh_name) or "" + end + + return sections, nil +end + +--- Read the bytes of one section. Returns a string, or nil if the adapter returns nil for any byte (out-of-bounds). +--- The caller is responsible fors sizing the buffer (the section's sh_offset + sh_size must fit in adapter.size). +--- @param adapter table +--- @param section table -- one entry from walk_sections +--- @return string|nil +function M.read_section_bytes(adapter, section) + local size = section.sh_size + if size == 0 then return "" end + local out = {} + for i = 0, size - 1 do + local b = M.read_u8(adapter, section.sh_offset + i) + if b == nil then return nil end + out[#out + 1] = string.char(b) + end + return table.concat(out) +end + +--- Convenience: walk sections, then look up the named section, then read its bytes. +--- Returns nil + a stable error code if the section is absent or out-of-bounds. +--- @param adapter table +--- @param sections table -- 1-based array from walk_sections +--- @param name string +--- @return string|nil, string|nil +function M.read_named_section(adapter, sections, name) + if not sections then return nil, "missing_section" end + for _, s in ipairs(sections) do + if s.name == name then + local bytes = M.read_section_bytes(adapter, s) + if not bytes then return nil, "truncated_section_data" end + return bytes, nil + end + end + return nil, "missing_section" +end + +--- Walk every SHT_SYMTAB section in `sections` and accumulate symbols by name. +--- Each stored entry is `{ value = st_value, size = st_size, info = st_info, shndx = st_shndx }`. +--- Both STB_LOCAL and STB_GLOBAL symbols are included; the live ELF stores `smem` as a local symbol. +--- Returns nil + a stable error code on failure: missing_symtab_strtab, truncated_section_headers +--- @param adapter table +--- @param sections table +--- @return table|nil, string|nil +function M.collect_symbols(adapter, sections) + if not sections then return nil, "missing_sections" end + local symbols = {} + local file_size = M.size(adapter) + for _, s in ipairs(sections) do + if s.sh_type == M.SHT_SYMTAB then + local strtab = sections[s.sh_link + 1] + if not strtab or strtab.sh_type ~= M.SHT_STRTAB then + return nil, "missing_symtab_strtab" + end + if strtab.sh_offset + strtab.sh_size > file_size then + return nil, "truncated_section_headers" + end + local strtab_bytes = M.read_section_bytes(adapter, strtab) + if not strtab_bytes then return nil, "truncated_section_headers" end + if s.sh_offset + s.sh_size > file_size then + return nil, "truncated_section_headers" + end + local symtab_bytes = M.read_section_bytes(adapter, s) + if not symtab_bytes then return nil, "truncated_section_headers" end + local n = #symtab_bytes / M.ELF32_SYM.sym_entry_bytes + for j = 0, n - 1 do + local e = s.sh_offset + j * M.ELF32_SYM.sym_entry_bytes + local st_name = M.read_u32(adapter, e + M.ELF32_SYM.st_name) + if st_name then + local st_value = M.read_u32(adapter, e + M.ELF32_SYM.st_value) + local st_size = M.read_u32(adapter, e + M.ELF32_SYM.st_size) + local st_info = M.read_u8(adapter, e + M.ELF32_SYM.st_info) + -- st_shndx is at offset 14 (2 bytes) — derived from the layout + -- the metaprogram reads too. Inline the read to keep the + -- adapter as the only I/O surface. + local b1 = M.read_u8(adapter, e + 14) + local b2 = M.read_u8(adapter, e + 15) + if not (b1 and b2) then + return nil, "truncated_section_headers" + end + local st_shndx = b1 + b2 * 0x100 + local name = M.get_str(strtab_bytes, st_name) or "" + if name ~= "" then + symbols[name] = { + value = st_value, + size = st_size, + info = st_info, + shndx = st_shndx, + } + end + end + end + end + end + return symbols, nil +end + +return M diff --git a/scripts/elf_dwarf.lua b/scripts/elf_dwarf.lua index fae8927..b645721 100644 --- a/scripts/elf_dwarf.lua +++ b/scripts/elf_dwarf.lua @@ -11,6 +11,11 @@ -- lfs is wired into package.cpath by `duffle_paths.lua` (vendored under `toolchain/lfs/lfs.dll`). local lfs = require("lfs") +-- scripts/elf32.lua contains format-constant tables + the byte-level walker. +-- The this file re-exports `read_u32_le` / `read_u16_le` (and the DWARF32 terminator). +-- TODO(Ed): Remove re-export. +local E = require("elf32") + local M = {} -- ════════════════════════════════════════════════════════════════════════════ @@ -102,27 +107,13 @@ M.MIPS_BYTES_PER_WORD = 0x04 --- **Wire-offset contract:** format offsets, fixed-width reader offsets, LEB/parser cursors, and section-relative values are zero-based wire offsets. --- Only Lua string APIs receive a `+ 1` conversion at their boundary (`byte`, `sub`, and `find`). --- ELF/DWARF field offsets are expressed in hex so they map directly to the zero-based byte positions in the binary file. +--- +--- The ELF32 header / section / sym layout tables are within scripts/elf32.lua. +--- The metaprogram re-exports the DWARF32 initial-length terminator. ---- spec: System V ABI gABI v1.2 §"ELF Header" (Table 1) + §"Section Header Table" -M.ELF32 = { - magic_offset = 0x00, -- 4-byte magic "\127ELF" at file offset 0x00 - magic = "\127ELF", - class_offset = 0x04, -- 1-byte; 1 = ELF32, 2 = ELF64 - class_elf32 = 1, - endian_offset = 0x05, -- 1-byte; 1 = little-endian, 2 = big-endian - endian_little = 1, - header_bytes = 0x34, -- spec: gABI v1.2 §"ELF Header" — ELF32 header is 52 bytes total - e_shoff_offset = 0x20, -- 4-byte LE; section-header table file offset - e_shentsize_offset = 0x2E, -- 2-byte LE; section-header entry size in bytes - e_shnum_offset = 0x30, -- 2-byte LE; number of section headers - e_shstrndx_offset = 0x32, -- 2-byte LE; index of section-name string table - sh_size_bytes = 0x28, -- spec: gABI v1.2 §"Section Header Table" — each entry is 40 bytes - sh_name_offset = 0x00, -- 4-byte LE; offset into .shstrtab - sh_type_offset = 0x04, -- 4-byte LE; section type (SHT_*) - sh_offset_offset = 0x10, -- 4-byte LE; section's file offset - sh_size_offset = 0x14, -- 4-byte LE; section's size in bytes - dw_dwarf32_terminator = 0xFFFFFFFF, -- spec: DWARF4 spec §7.4 — 32-bit DWARF initial-length terminator -} +--- spec: DWARF4 spec §7.4 — 32-bit DWARF initial-length terminator +M.dw_dwarf32_terminator = E.dw_dwarf32_terminator +-- TODO(Ed): Remove re-export. -- ---------------------------------------------------------------------------- -- DWARF4 .debug_aranges (per DWARF5 spec §7.4 — Address Range Table) @@ -241,27 +232,24 @@ M.DWARF5_DEBUG_LINE = { --- (which has partial `string.unpack` coverage). --- **Convention:** `off` is a zero-based wire offset; `+ 1` is applied only at the `string.byte` boundary. --- ---- **Byte weights** are written as `0x100`, `0x10000`, `0x1000000` (i.e. 2^8, 2^16, 2^24) so the LE byte positions are visually explicit: ---- byte 0 contributes its value directly; byte 1 is shifted left by 8 (= 0x100); byte 2 by 16 (= 0x10000); byte 3 by 24 (= 0x1000000). +--- Thin forwarder: the canonical implementation lives in scripts/elf32.lua. +--- The "second caller lifts" pattern keeps the metaprogram side fluent +--- (`M.read_u32_le(buf, off)`) while the body is deduped. --- @param buf string --- @param off integer -- zero-based wire offset --- @return integer function M.read_u32_le(buf, off) - local byte_off = off + 1 - return buf:byte(byte_off) - + buf:byte(byte_off + 0x01) * 0x00000100 - + buf:byte(byte_off + 0x02) * 0x00010000 - + buf:byte(byte_off + 0x03) * 0x01000000 + return E.read_u32_le(buf, off) end --- Read a 2-byte little-endian unsigned integer from `buf` at zero-based wire offset `off`. --- (`off` is zero-based; `+ 1` is applied only at the `string.byte` boundary.) +--- Thin forwarder — see `M.read_u32_le` for the rationale. --- @param buf string --- @param off integer -- zero-based wire offset --- @return integer function M.read_u16_le(buf, off) - local byte_off = off + 1 - return buf:byte(byte_off) + buf:byte(byte_off + 0x01) * 0x00000100 + return E.read_u16_le(buf, off) end -- Pure-Lua 5.3 LEB128 readers (no `bit` library). `2^shift` arithmetic matches the existing parser. @@ -442,20 +430,20 @@ function M.read_ref_sig8(buf, pos) return M.read_u32_le(buf, pos), M.read_u32_le(buf, pos + 4), pos + 8 end --- DWARF5 §7.5.6 (Type Entries). --- Walk all units in `info` and return the 0-based offset of the first unit --- whose `DW_AT_type_signature` (8-byte value at the end of the unit header) equals `target_sig`. --- The signature is interpreted as two 32-bit halves (low/high) per the read_ref_sig8 contract; --- we match both halves (i.e. the 8-byte value as a whole). Returns nil if no matching unit exists. --- --- Unit header layout (from pos 0): --- unit_length(4) + version(2) + unit_type(1) + address_size(1) + debug_abbrev_offset(4) --- followed by type_unit_specific fields: type_signature(8) + type_offset(4) --- The type_signature is at byte offset 8 of the body (right after debug_abbrev_offset). --- @param info string -- the .debug_info section bytes --- @param target_sig_lo integer -- low 4 bytes (LE) of the desired signature --- @param target_sig_hi integer -- high 4 bytes (LE) of the desired signature --- @return integer|nil, integer|nil -- unit offset, type_offset within the unit +--- DWARF5 §7.5.6 (Type Entries). +--- Walk all units in `info` and return the 0-based offset of the first unit whose `DW_AT_type_signature` +--- (8-byte value at the end of the unit header) equals `target_sig`. +--- The signature is interpreted as two 32-bit halves (low/high) per the read_ref_sig8 contract; +--- we match both halves (i.e. the 8-byte value as a whole). Returns nil if no matching unit exists. +--- +--- Unit header layout (from pos 0): +--- unit_length(4) + version(2) + unit_type(1) + address_size(1) + debug_abbrev_offset(4) +--- followed by type_unit_specific fields: type_signature(8) + type_offset(4) +--- The type_signature is at byte offset 8 of the body (right after debug_abbrev_offset). +--- @param info string -- the .debug_info section bytes +--- @param target_sig_lo integer -- low 4 bytes (LE) of the desired signature +--- @param target_sig_hi integer -- high 4 bytes (LE) of the desired signature +--- @return integer|nil, integer|nil -- unit offset, type_offset within the unit function M.find_type_unit_by_signature(info, target_sig_lo, target_sig_hi) local pos = 0 local section_len = #info @@ -539,7 +527,7 @@ end --- (we walk all `e_shnum` headers regardless of how many names are requested, to find the .shstrtab first). --- For frequent callers, pass the union of all needed sections in one call. -- Can add `.debug_info` + `.debug_loc` + `.debug_str_offsets` to the list without writing a 2nd ELF walker. ---- @param elf_path Path +--- @param elf_path Path --- @param section_names string[] -- list of section names to read --- @return table function M.read_elf_sections(elf_path, section_names) @@ -564,69 +552,58 @@ function M.read_elf_sections(elf_path, section_names) return result end - -- Read the ELF32 header. - local header = f:read(M.ELF32.header_bytes) - if not header or #header < M.ELF32.header_bytes then - io.stderr:write("[elf_dwarf.read_elf_sections] ELF too small for ELF32 header\n") + local file_size + do + f:seek("end", 0) + file_size = f:seek("cur", 0) + end + local adapter = { + read_u8_at = function(offset) + f:seek("set", offset) + local b = f:read(1) + if not b then return nil end + return b:byte() + end, + read_u16_at = function(offset) + f:seek("set", offset) + local b1 = f:read(1) + local b2 = f:read(1) + if not b1 or not b2 then return nil end + return b1:byte() + b2:byte() * 0x100 + end, + read_u32_at = function(offset) + f:seek("set", offset) + local b1 = f:read(1) + local b2 = f:read(1) + local b3 = f:read(1) + local b4 = f:read(1) + if not b1 or not b2 or not b3 or not b4 then return nil end + return b1:byte() + b2:byte() * 0x100 + + b3:byte() * 0x10000 + b4:byte() * 0x1000000 + end, + read_size = function() return file_size end, + } + + -- Delegate the header parse + section walk to E.*. + local hdr, hdr_err = E.parse_elf32_headers(adapter) + if not hdr then + io.stderr:write(string.format("[elf_dwarf.read_elf_sections] header parse failed: %s\n", tostring(hdr_err))) f:close() return result end - -- Sanity-check magic + class + endianness. - if header:sub(M.ELF32.magic_offset + 1, M.ELF32.magic_offset + 0x04) ~= M.ELF32.magic then - io.stderr:write("[elf_dwarf.read_elf_sections] not an ELF file\n") - f:close() - return result - end - if header:byte(M.ELF32.class_offset + 1) ~= M.ELF32.class_elf32 then - io.stderr:write(string.format("[elf_dwarf.read_elf_sections] not ELF32 (class=%d)\n", header:byte(M.ELF32.class_offset + 1))) - f:close() - return result - end - if header:byte(M.ELF32.endian_offset + 1) ~= M.ELF32.endian_little then - io.stderr:write("[elf_dwarf.read_elf_sections] not little-endian; unsupported\n") + local sections, walk_err = E.walk_sections(adapter, hdr) + if not sections then + io.stderr:write(string.format("[elf_dwarf.read_elf_sections] section walk failed: %s\n", tostring(walk_err))) f:close() return result end - -- Parse section-header table location + dimensions from the header. - local e_shoff = M.read_u32_le(header, M.ELF32.e_shoff_offset) - local e_shentsize = M.read_u16_le(header, M.ELF32.e_shentsize_offset) - local e_shnum = M.read_u16_le(header, M.ELF32.e_shnum_offset) - local e_shstrndx = M.read_u16_le(header, M.ELF32.e_shstrndx_offset) - - -- Read the section-header string table (.shstrtab) so we can resolve section names from their `sh_name` offsets. - f:seek("set", e_shoff + e_shstrndx * e_shentsize) - local strtab_hdr = f:read(e_shentsize) - if not strtab_hdr or #strtab_hdr < e_shentsize then - io.stderr:write("[elf_dwarf.read_elf_sections] could not read .shstrtab header\n") - f:close() - return result - end - local strtab_offset = M.read_u32_le(strtab_hdr, M.ELF32.sh_offset_offset) - local strtab_size = M.read_u32_le(strtab_hdr, M.ELF32.sh_size_offset) - f:seek("set", strtab_offset) - local strtab = f:read(strtab_size) or "" - - -- Walk all section headers; collect (offset, size) for the wanted names. - local function read_section_bytes(sh_offset, sh_size) - f:seek("set", sh_offset) - return f:read(sh_size) or "" - end - - for sh_idx = 0, e_shnum - 1 do - f:seek("set", e_shoff + sh_idx * e_shentsize) - local sh = f:read(e_shentsize) - if not sh or #sh < e_shentsize then break end - local sh_name = M.read_u32_le(sh, M.ELF32.sh_name_offset) - local sh_offset = M.read_u32_le(sh, M.ELF32.sh_offset_offset) - local sh_size = M.read_u32_le(sh, M.ELF32.sh_size_offset) - - -- Extract the name (null-terminated C string in strtab). - local name_end = strtab:find("\0", sh_name + 1, true) or (sh_name + 1) - local name = strtab:sub(sh_name + 1, name_end - 1) - if wanted[name] then - result[name] = read_section_bytes(sh_offset, sh_size) + -- Resolve the requested sections. + for _, s in ipairs(sections) do + if wanted[s.name] then + local bytes = E.read_section_bytes(adapter, s) + if bytes then result[s.name] = bytes end end end @@ -643,48 +620,87 @@ end --- - We filter on STB_GLOBAL (high nibble of st_info = 1) to match `nm`'s default (external symbols only). STB_WEAK excluded. --- - The `code_` prefix is stripped (MipsAtom_ macros emit bare atom names, no `code_` prefix). --- - `st_size > 0` filter excludes undefined/imported symbols. +--- --- @param elf_path Path --- @return table function M.read_nm(elf_path) local addrs = {} - -- Read .symtab + .strtab via the existing ELF walker (no subprocess). - local sections = M.read_elf_sections(elf_path, {".symtab", ".strtab"}) - local symtab = sections[".symtab"] - local strtab = sections[".strtab"] - if not symtab or not strtab or #symtab == 0 or #strtab == 0 then - -- No symbol table (e.g. stripped ELF). Return empty. + -- Existence check first; an empty or missing ELF returns an empty map. + if lfs.attributes(elf_path, "mode") ~= "file" then return addrs end - -- Iterate the 16-byte ELF32 symtab entries. - -- Each entry (zero-based): st_name at 0, st_value at 4, st_size at 8, st_info at 12, st_other at 13, st_shndx at 14. - local SYM_ENTRY_BYTES = 0x10 - local SYM_ST_NAME = 0x00 - local SYM_ST_VALUE = 0x04 - local SYM_ST_SIZE = 0x08 - local SYM_ST_INFO = 0x0C - local n_syms = #symtab / SYM_ENTRY_BYTES - for i = 0, n_syms - 1 do - local entry_off = i * SYM_ENTRY_BYTES - local st_info = symtab:byte(entry_off + SYM_ST_INFO + 1) - -- High nibble = binding (STB_LOCAL=0, STB_GLOBAL=1, STB_WEAK=2). - -- Use math.floor(/16) instead of bit.rshift for LuaJIT 2.1 compat (LuaJIT's `>>` is 5.3+, but math.floor(x/16) works on all versions). - local binding = math.floor(st_info / 16) - if binding == 0 or binding == 1 then -- STB_LOCAL or STB_GLOBAL - local st_size = M.read_u32_le(symtab, entry_off + SYM_ST_SIZE) - if st_size > 0 then - local st_name_off = M.read_u32_le(symtab, entry_off + SYM_ST_NAME) - -- Extract the name from .strtab (null-terminated C string). - local name_end = strtab:find("\0", st_name_off + 1, true) or (st_name_off + 1) - local name = strtab:sub(st_name_off + 1, name_end - 1) - -- Filter: keep all symbol-table symbols (atoms emit their name as the bare `` — MipsAtom_ macros strip the `code_` prefix). - -- The atoms_source_map pass already filters out non-atom symbols via the source-map.txt cross-ref. - if name and #name > 0 then - local st_value = M.read_u32_le(symtab, entry_off + SYM_ST_VALUE) - addrs[name] = { st_value, st_size } - end - end + local f = io.open(elf_path, "rb") + if not f then + return addrs + end + + -- Build the file adapter for E.*. + local file_size + do + f:seek("end", 0) + file_size = f:seek("cur", 0) + end + local adapter = { + read_u8_at = function(offset) + f:seek("set", offset) + local b = f:read(1) + if not b then return nil end + return b:byte() + end, + read_u16_at = function(offset) + f:seek("set", offset) + local b1 = f:read(1) + local b2 = f:read(1) + if not b1 or not b2 then return nil end + return b1:byte() + b2:byte() * 0x100 + end, + read_u32_at = function(offset) + f:seek("set", offset) + local b1 = f:read(1) + local b2 = f:read(1) + local b3 = f:read(1) + local b4 = f:read(1) + if not b1 or not b2 or not b3 or not b4 then return nil end + return b1:byte() + b2:byte() * 0x100 + + b3:byte() * 0x10000 + b4:byte() * 0x1000000 + end, + read_size = function() return file_size end, + } + + -- Delegate the header + section walk to E.*. + local hdr, hdr_err = E.parse_elf32_headers(adapter) + if not hdr then + io.stderr:write(string.format("[elf_dwarf.read_nm] header parse failed: %s\n", tostring(hdr_err))) + f:close() + return addrs + end + + local sections, walk_err = E.walk_sections(adapter, hdr) + if not sections then + io.stderr:write(string.format("[elf_dwarf.read_nm] section walk failed: %s\n", tostring(walk_err))) + f:close() + return addrs + end + + -- E.collect_symbols returns every defined symbol (no binding filter). + -- The metaprogram then applies its STB_LOCAL / STB_GLOBAL + size>0 filter, matching `nm`'s default (external symbols only). + local symbols, sym_err = E.collect_symbols(adapter, sections) + if not symbols then + io.stderr:write(string.format("[elf_dwarf.read_nm] symbol collection failed: %s\n", tostring(sym_err))) + f:close() + return addrs + end + + f:close() + + for name, entry in pairs(symbols) do + -- High nibble of st_info = binding (STB_LOCAL=0, STB_GLOBAL=1, STB_WEAK=2). + -- math.floor(/16) is portable across LuaJIT 2.0/2.1 and plain Lua 5.x. + local binding = math.floor(entry.info / 16) + if (binding == 0 or binding == 1) and entry.size > 0 then + addrs[name] = { entry.value, entry.size } end end @@ -822,12 +838,11 @@ end --- * The `.debug_line` section may contain MULTIPLE line-program units --- File indices are 1-based, **per unit**; we concatenate all units and the index ranges from 1..N₁ in unit 1, N₁+1..N₁+N₂ in unit 2, etc. --- Per-unit indices (the way gcc emits them, and the way `DW_LNS_set_file` references them in the line program) ---- are returned via the `basename_to_index` map only when the unit boundary happens to align with the metaprogram's per-atom ---- `inv.call_file` (true today for hello_joypad — the C unit is the LAST unit, and atom-side file indices fit 1-based). +--- are returned via the `basename_to_index` map only when the unit boundary happens to align with the metaprogram's per-atom `inv.call_file` --- * Per spec, the `.debug_line_str` section (DWARF5 §7.5.6) holds the strings referenced by `DW_FORM_line_strp`. --- The legacy DWARF3 format embeds strings directly with null terminators. This helper handles BOTH. ---- * File entries may have multiple forms (gcc -gdwarf-5 with `DW_LNCT_directory_index` ---- emits 2 forms: path + dir_index). The helper supports: +--- * File entries may have multiple forms (gcc -gdwarf-5 with `DW_LNCT_directory_index` emits 2 forms: path + dir_index). +--- The helper supports: --- - DW_FORM_line_strp (DWARF5; offset into .debug_line_str) --- - DW_FORM_string (DWARF4-compat; inline null-terminated in .debug_line) --- - DW_FORM_udata (ULEB128) @@ -837,9 +852,7 @@ end --- --- Behavior on failure: writes to stderr and returns nil. --- Helpers consumed by `passes/dwarf_injection.lua::init_file_index_lookup(elf_path)` calls this once at pass start to populate the module-level `basename_to_index` map; ---- downstream `resolve_provenance_file_index(path)` consumers ---- (which replaced the former hardcoded `ATOM_SOURCE_FILE_INDEX` + `PROVENANCE_BASENAME_TO_FILE_INDEX` table per `conductor/tracks/dwarf_file_index_lookup_20260731/`) ---- consult the map directly. +--- downstream `resolve_provenance_file_index(path)` consumers consult the map directly. --- --- @param elf_path string -- absolute path to the post-link ELF (typically the gcc-emitted `.elf` BEFORE dwarf_injector's splice; both shapes work since the splice preserves `.debug_line`) --- @return table|nil, table|nil, table|nil diff --git a/scripts/passes/dwarf_injection.lua b/scripts/passes/dwarf_injection.lua index 7ffddb6..f0b706d 100644 --- a/scripts/passes/dwarf_injection.lua +++ b/scripts/passes/dwarf_injection.lua @@ -992,7 +992,7 @@ local function build_dwarf_line_section(existing, atom_table) while unit_pos < #existing do if unit_pos + 4 > #existing then return existing end local unit_length = elf_dwarf.read_u32_le(existing, unit_pos) - if unit_length == elf_dwarf.ELF32.dw_dwarf32_terminator then return existing end + if unit_length == elf_dwarf.dw_dwarf32_terminator then return existing end local unit_end_excl = unit_pos + 4 + unit_length if unit_end_excl > #existing then return existing end last_pos, last_length, last_end = unit_pos, unit_length, unit_end_excl @@ -1053,7 +1053,7 @@ local function build_dwarf_aranges_section(existing, atom_table) while i < #existing do -- Read this unit's length. local ul = elf_dwarf.read_u32_le(existing, i) - if ul == elf_dwarf.ELF32.dw_dwarf32_terminator then + if ul == elf_dwarf.dw_dwarf32_terminator then -- DWARF64 marker - not supported. io.stderr:write("[dwarf_injection] WARN: .debug_aranges contains a DWARF64 marker (0xFFFFFFFF); the 64-bit extension is not supported by this metaprogram; passing through unchanged\n") return existing