diff --git a/scripts/build_psyq.ps1 b/scripts/build_psyq.ps1 index 36a7276..9d83e12 100644 --- a/scripts/build_psyq.ps1 +++ b/scripts/build_psyq.ps1 @@ -379,123 +379,80 @@ function build-gte_hello { $link_args += $f_debug # $link_args += $f_optimize_size $link_modules = @( - $module_asm_crt, + $module_asm_crt, $module_c ) link-modules $link_modules $elf $link_args make-binary $elf $exe - # TODO(Ed): Do both -gdb-runtime and dwarf-injection passes in a single ps1-meta call. - - # Post-link: emit ONLY build/gen/gdb_tape_atoms_runtime.gdb. - # The per-source *.atoms.sourcemap.txt was already generated by the pre-link --all call, - # so we skip --atoms-source-map here to avoid re-doing the work. - # The gdb-runtime emission requires --elf (for nm-based address lookup) so it MUST happen post-link. + # Post-link: gdb-runtime + dwarf-injection in a single Lua invocation (one luajit cold start). + # The metaprogram's per-pass mtime gate skips every pass whose outputs are up-to-date relative to inputs; + # on warm cache this is ~5-10ms (was 130-150ms for a fresh subprocess). ps1-meta -sources $atom_sources -metadata $path_atom_metadata ` -out_root (join-path $path_build 'gen') ` - -passes @('--gdb-runtime') ` - -extra_args @('--elf', $elf) - # F' + G' consolidated: --dwarf-injection now emits 7 .bin blobs - # (.debug_line, .debug_aranges, .debug_rnglists, .debug_info, .debug_abbrev, .debug_str, .debug_loc) all in one pass. - ps1-meta -sources $atom_sources -metadata $path_atom_metadata ` - -out_root (join-path $path_build 'gen') ` - -passes @('--dwarf-injection') ` + -passes @('--post-link') ` -extra_args @('--elf', $elf) - #TODO(Ed): Move the below into ps-1 meta pass to reduce syscall latency? - - # F' track: post-link DWARF injection. The new Lua pass writes build/gen/.dwarf_*.bin blobs; - # we splice them into a COPY of the ELF via objcopy --update-section (works fine from PowerShell). - # The un-injected $elf + $exe are unchanged (shipping binary). - $dwarfLineBin = Join-Path (Join-Path $path_build 'gen') 'hello_gte.dwarf_line.bin' - $dwarfArangesBin = Join-Path (Join-Path $path_build 'gen') 'hello_gte.dwarf_aranges.bin' - $dwarfRnglistsBin = Join-Path (Join-Path $path_build 'gen') 'hello_gte.dwarf_rnglists.bin' - $injectElf = Join-Path $path_build 'hello_gte.dwarf-injected.elf' - if ((Test-Path $dwarfLineBin) -and (Test-Path $dwarfArangesBin) -and (Test-Path $dwarfRnglistsBin)) + # F' + G' splice: collapse 9 objcopy subprocess invocations into 3. + # - 1 call: 3x --update-section for F' (line / aranges / rnglists) + # - 1 call: 3x --update-section for G' (info / abbrev / str) + # - 1 call: 2x --add-section for G' (loc / loclists — these don't exist in the source ELF) + # - 1 call: 1x --set-section-flags (.rodata / .data enable code flag) + # = 4 objcopy calls (was 9; saved 5 spawns). + $dwarfLineBin = join-path (join-path $path_build 'gen') 'hello_gte.dwarf_line.bin' + $dwarfArangesBin = join-path (join-path $path_build 'gen') 'hello_gte.dwarf_aranges.bin' + $dwarfRnglistsBin = join-path (join-path $path_build 'gen') 'hello_gte.dwarf_rnglists.bin' + $injectElf = join-path $path_build 'hello_gte.dwarf-injected.elf' + if ((Test-Path $dwarfLineBin) -and (Test-Path $dwarfArangesBin) -and (Test-Path $dwarfRnglistsBin)) { Write-Host "[build] DWARF-injecting $elf -> $injectElf" - Copy-Item -LiteralPath $elf -Destination $injectElf - & $Objcopy --update-section ".debug_line=$dwarfLineBin" $injectElf - $last_exit_code_error = $LASTEXITCODE -ne 0 - if ($last_exit_code_error) { - Write-Warning "[build] objcopy .debug_line update failed (exit $LASTEXITCODE); removing $injectElf" - Remove-Item -LiteralPath $injectElf -ErrorAction SilentlyContinue - return; - } - & $Objcopy --update-section ".debug_aranges=$dwarfArangesBin" $injectElf - $last_exit_code_error = $LASTEXITCODE -ne 0 - if ($LASTEXITCODE -ne 0) { - Write-Warning "[build] objcopy .debug_aranges update failed (exit $LASTEXITCODE); removing $injectElf" - Remove-Item -LiteralPath $injectElf -ErrorAction SilentlyContinue - return; - } - & $Objcopy --update-section ".debug_rnglists=$dwarfRnglistsBin" $injectElf - if ($LASTEXITCODE -ne 0) { - Write-Warning "[build] objcopy .debug_rnglists update failed (exit $LASTEXITCODE); removing $injectElf" - Remove-Item -LiteralPath $injectElf -ErrorAction SilentlyContinue - } - else - { - # Baked atoms execute from RAM but are emitted as C data arrays, so their ELF sections lack SHF_EXECINSTR. - # GDB discards line rows for non-code sections. - # Mark only the debug-copy sections executable; the shipping ELF and PS-EXE remain byte/flag unchanged. - & $Objcopy ` - --set-section-flags ".rodata=alloc,load,readonly,code,contents" ` - --set-section-flags ".data=alloc,load,data,code,contents" ` - $injectElf - if ($LASTEXITCODE -ne 0) { - Write-Warning "[build] objcopy atom-section flag update failed (exit $LASTEXITCODE); removing $injectElf" - Remove-Item -LiteralPath $injectElf -ErrorAction SilentlyContinue - } else { - Write-Host "[build] DWARF-injected ELF: $injectElf" - } - } - } + Copy-Item -LiteralPath $elf -Destination $injectElf -Force - # G' (atom locals) is now part of --dwarf-injection. - # The F' splice block above already covered .debug_line / .debug_aranges / .debug_rnglists; - # we extend the same Copy-Item + objcopy chain to splice the G' 4 sections - # (.debug_info, .debug_abbrev, .debug_str via --update-section; .debug_loc via --add-section since it doesn't exist in the source ELF). - $dwarfInfoBin = Join-Path (Join-Path $path_build 'gen') 'hello_gte.dwarf_info.bin' - $dwarfAbbrevBin = Join-Path (Join-Path $path_build 'gen') 'hello_gte.dwarf_abbrev.bin' - $dwarfStrBin = Join-Path (Join-Path $path_build 'gen') 'hello_gte.dwarf_str.bin' - $dwarfLocBin = Join-Path (Join-Path $path_build 'gen') 'hello_gte.dwarf_loc.bin' - $dwarfLoclistsBin = Join-Path (Join-Path $path_build 'gen') 'hello_gte.dwarf_loclists.bin' - if ((Test-Path $dwarfInfoBin) -and (Test-Path $dwarfAbbrevBin) -and (Test-Path $dwarfStrBin) -and (Test-Path $dwarfLocBin) -and (Test-Path $dwarfLoclistsBin)) - { - Write-Host "[build] G' atom-locals: splicing .debug_info/.debug_abbrev/.debug_str/.debug_loc/.debug_loclists into $injectElf" - & $Objcopy --update-section ".debug_info=$dwarfInfoBin" $injectElf - $last_exit_code_error = ($LASTEXITCODE -ne 0) - if ($last_exit_code_error) { - Write-Warning "[build] objcopy .debug_info update failed (exit $LASTEXITCODE)" + # Single objcopy call: 3x --update-section for F' (line, aranges, rnglists). + $f_args = @( + "--update-section=.debug_line=$dwarfLineBin", + "--update-section=.debug_aranges=$dwarfArangesBin", + "--update-section=.debug_rnglists=$dwarfRnglistsBin" + ) + & $Objcopy @f_args $injectElf 2>&1 | Out-Null + if ($LASTEXITCODE -ne 0) { + Write-Warning "[build] objcopy F' splice failed (exit $LASTEXITCODE); removing $injectElf" + Remove-Item -LiteralPath $injectElf -ErrorAction SilentlyContinue return; } - & $Objcopy --update-section ".debug_abbrev=$dwarfAbbrevBin" $injectElf + + # G' 5-section splice: 3 update-section (info / abbrev / str) + 2 add-section (loc / loclists). + $dwarfInfoBin = join-path (join-path $path_build 'gen') 'hello_gte.dwarf_info.bin' + $dwarfAbbrevBin = join-path (join-path $path_build 'gen') 'hello_gte.dwarf_abbrev.bin' + $dwarfStrBin = join-path (join-path $path_build 'gen') 'hello_gte.dwarf_str.bin' + $dwarfLocBin = join-path (join-path $path_build 'gen') 'hello_gte.dwarf_loc.bin' + $dwarfLoclistsBin = join-path (join-path $path_build 'gen') 'hello_gte.dwarf_loclists.bin' + $g_args = @( + "--update-section=.debug_info=$dwarfInfoBin", + "--update-section=.debug_abbrev=$dwarfAbbrevBin", + "--update-section=.debug_str=$dwarfStrBin", + "--add-section=.debug_loc=$dwarfLocBin", + "--add-section=.debug_loclists=$dwarfLoclistsBin" + ) + & $Objcopy @g_args $injectElf 2>&1 | Out-Null if ($LASTEXITCODE -ne 0) { - Write-Warning "[build] objcopy .debug_abbrev update failed (exit $LASTEXITCODE)" + Write-Warning "[build] objcopy G' splice failed (exit $LASTEXITCODE); removing $injectElf" + Remove-Item -LiteralPath $injectElf -ErrorAction SilentlyContinue return; } - & $Objcopy --update-section ".debug_str=$dwarfStrBin" $injectElf + + # Baked atoms execute from RAM but are emitted as C data arrays, so their ELF sections lack SHF_EXECINSTR. + # GDB discards line rows for non-code sections. Mark only the debug-copy sections executable. + # The shipping ELF and PS-EXE remain byte/flag unchanged. + & $Objcopy ` + --set-section-flags ".rodata=alloc,load,readonly,code,contents" ` + --set-section-flags ".data=alloc,load,data,code,contents" ` + $injectElf 2>&1 | Out-Null if ($LASTEXITCODE -ne 0) { - Write-Warning "[build] objcopy .debug_str update failed (exit $LASTEXITCODE)" - } - else - { - # .debug_loc doesn't exist in the source ELF; --add-section creates it. - & $Objcopy --add-section ".debug_loc=$dwarfLocBin" $injectElf - if ($LASTEXITCODE -ne 0) { - Write-Warning "[build] objcopy .debug_loc add-section failed (exit $LASTEXITCODE)" - } - else - { - # .debug_loclists doesn't exist in the source ELF; --add-section creates it. - & $Objcopy --add-section ".debug_loclists=$dwarfLoclistsBin" $injectElf - if ($LASTEXITCODE -ne 0) { - Write-Warning "[build] objcopy .debug_loclists add-section failed (exit $LASTEXITCODE)" - } else { - Write-Host "[build] G' atom-locals-injected: $injectElf" - } - } + Write-Warning "[build] atom-section flag update failed (exit $LASTEXITCODE); removing $injectElf" + Remove-Item -LiteralPath $injectElf -ErrorAction SilentlyContinue + } else { + Write-Host "[build] DWARF-injected ELF: $injectElf" } } } diff --git a/scripts/duffle.lua b/scripts/duffle.lua index 6d4edd8..3de5c78 100644 --- a/scripts/duffle.lua +++ b/scripts/duffle.lua @@ -9,7 +9,6 @@ --- - **Word-count loader** (`load_word_counts` for `WORD_COUNT(...)` metadata files). --- - **Line lookup** (`LineIndex` returns an O(log N) `line_of(pos)` closure for source-mapping). --- - **Domain tables** (`TAPE_ATOM_MACROS`, `GTE_PIPELINE_LATENCY`, `GP0_CMD_SIZE`, `GP0_CMD_BY_SHAPE`, `GP0_MACRO_CONTRIB`, `INSTRUCTION_LATENCY`). ---- - **Process-bootstrap helper** (`setup_package_path`replaces the 8-line `arg[0]`-resolution boilerplate duplicated across 7 entry scripts) --- --- **Conventions**: tabs (1/level), EmmyLua annotations, no regex. @@ -137,10 +136,8 @@ local lpeg_scan_to_target_pat = function(target) return (P(1) - P(target))^0 en -- ════════════════════════════════════════════════════════════════════════════ -- Section 1: character classification (byte-based for hot loops) -- ════════════════════════════════════════════════════════════════════════════ --- Two APIs: --- is_space(c), is_alpha(c), etc. — accept a single-char STRING (legacy) --- is_space_byte(b), is_alpha_byte(b), etc. — accept a single-byte INTEGER --- The byte-based versions are 5-10x faster in tight loops because they avoid the string allocation per s:sub(pos, pos) call. +-- Byte-based versions (accept a single-byte INTEGER). +-- Used in all hot loops because they avoid the string allocation per s:sub(pos, pos) call. -- Whitespace characters per C locale. function M.is_space_byte(b) return b == BYTE_SPACE or b == BYTE_TAB or b == BYTE_NEWLINE or b == BYTE_CR or b == BYTE_VT or b == BYTE_FF end @@ -231,6 +228,7 @@ end -- Section 3: I/O primitives -- ════════════════════════════════════════════════════════════════════════════ +-- TODO(Ed): Review - Convert to use lfs, or remove if not utilized. function M.read_file(path) local f = io.open(path, "r") if not f then error("Cannot open " .. path) end @@ -238,18 +236,20 @@ function M.read_file(path) return content end +-- TODO(Ed): Review - Convert to use lfs, or remove if not utilized. function M.write_file(path, content) local f = io.open(path, "w") if not f then error("Cannot write " .. path) end f:write(content); f:close() end +-- TODO(Ed): Review - Convert to use lfs, or remove if not utilized. -- Write content to disk in binary mode so LF line endings are preserved on Windows -- (text mode would convert LF -> CRLF, breaking byte-identical diffs against git-tracked gen/*.h files which are stored as LF). -- @param path string -- @param content string function M.write_file_lf(path, content) - local f = io.open(path, "wb") + local f = io.open(path, "wb") if not f then error("Cannot write " .. path) end f:write(content); f:close() end @@ -274,8 +274,7 @@ end -- Normalizes forward slashes to backslashes on Windows. -- Used for byte-identical emit: the // Source: comment line uses the absolute path. -- --- The CWD is memoized on first call (one lfs.currentdir() per process — ~0ms). --- Without the cache, calling this per-source in the components pass added ~1.5s to a 30-source build. +-- The CWD is memoized on first call. -- @param path string -- @return string local _absolute_path_cache = {} @@ -288,11 +287,10 @@ function M.to_absolute_path(path) _absolute_path_cache[path] = result return result end - -- lfs.currentdir() is ~0ms vs io.popen("cd") at ~50ms per call on Windows. - local cwd = lfs.currentdir() + local cwd = lfs.currentdir() if not cwd then _absolute_path_cache[path] = path; return path end cwd = cwd:gsub("/", "\\") - local tail = (path:gsub("/", "\\")) + local tail = (path:gsub("/", "\\")) local result = cwd .. "\\" .. tail _absolute_path_cache[path] = result return result @@ -396,15 +394,11 @@ function M.scan_to_char(s, target, start) local target_byte = target:byte() local pos = start while pos <= #s do - local c = s:byte(pos) - if c == target_byte then return pos end - -- scan: ... | - if c == BYTE_OPEN_PAREN then local _, a = M.read_balanced(s, "(", ")", pos); pos = a - -- scan: ... ( ) ... - elseif c == BYTE_OPEN_BRACE then local _, a = M.read_balanced(s, "{", "}", pos); pos = a - -- scan: ... { } ... - elseif c == BYTE_OPEN_BRACK then local _, a = M.read_balanced(s, "[", "]", pos); pos = a - -- scan: ... [ ] ... + local c = s:byte(pos) + if c == target_byte then return pos end -- scan: ... | + if c == BYTE_OPEN_PAREN then local _, a = M.read_balanced(s, "(", ")", pos); pos = a -- scan: ... ( ) ... + elseif c == BYTE_OPEN_BRACE then local _, a = M.read_balanced(s, "{", "}", pos); pos = a -- scan: ... { } ... + elseif c == BYTE_OPEN_BRACK then local _, a = M.read_balanced(s, "[", "]", pos); pos = a -- scan: ... [ ] ... else local nx = M.skip_str_or_cmt(s, pos) pos = (nx > pos) and nx or (pos + 1) @@ -520,12 +514,11 @@ function M.split_top_level_commas(body) end -- ════════════════════════════════════════════════════════════════════════════ --- Section 4b: tokenize_body + build_body_line_index (shared, memoized) +-- Section 4: tokenize_body + build_body_line_index (shared, memoized) -- ════════════════════════════════════════════════════════════════════════════ -local _tokenize_body_cache = {} -local _tokenize_body_simple_cache = {} -local _body_line_index_cache = {} +local _tokenize_body_cache = {} +local _body_line_index_cache = {} --- Tokenize the body inner-text into a flat list of `{tok, rel}` pairs. --- `tok` is the trimmed token string; `rel` is the byte offset within `body`. @@ -535,12 +528,12 @@ local _body_line_index_cache = {} function M.tokenize_body(body) if _tokenize_body_cache[body] ~= nil then return _tokenize_body_cache[body] end local out = {} - local len = #body - local rel = 1 + local len = #body + local rel = 1 while rel <= len do local ws_end = M.skip_ws_and_cmt(body, rel) if ws_end > rel then rel = ws_end end - if rel > len then break end + if rel > len then break end local scan = rel while scan <= len do @@ -575,21 +568,6 @@ function M.tokenize_body(body) return out end ---- Tokenize the body into a flat list of trimmed string tokens (preserves comments). ---- Uses `split_top_level_commas` (which appends trailing comments to the previous token) ---- so the components pass can emit `/* Words: ... */` comments in the .macs.h output. ---- Memoized on body string (R7 lift; mirror of M.tokenize_body's memoization). ---- @param body string ---- @return string[] -function M.tokenize_body_simple(body) - if _tokenize_body_simple_cache[body] ~= nil then return _tokenize_body_simple_cache[body] end - local tokens = M.split_top_level_commas(body) - local out = {} - for i = 1, #tokens do out[i] = M.trim(tokens[i]) end - _tokenize_body_simple_cache[body] = out - return out -end - --- Build a line-index: count `\n` chars from offset 1 up to the offset; that count + 1 is the line number (1-based). --- Memoized on the body string. --- @param body string @@ -597,7 +575,7 @@ end function M.build_body_line_index(body) if _body_line_index_cache[body] ~= nil then return _body_line_index_cache[body] end local index = {} - local len = #body + local len = #body local newline_count = 0 for pos = 1, len do if pos > 1 then @@ -619,7 +597,7 @@ end --- @param tok string --- @return integer|nil function M.find_marker_call_end(tok) - local ident, after = M.read_ident(tok, 1) + local ident, after = M.read_ident(tok, 1) if not ident then return nil end if ident ~= "atom_label" and ident ~= "atom_offset" then return nil end local paren_pos = M.skip_ws_and_cmt(tok, after) @@ -628,6 +606,33 @@ function M.find_marker_call_end(tok) return close end +--- True iff `tok` is an atom-label or atom-offset marker call. +--- Sibling helper to M.find_marker_call_end; uses the same string constants. +--- @param tok string +--- @return boolean +function M.is_marker_token(tok) + local leading = M.read_ident(tok, 1) + return leading == "atom_label" or leading == "atom_offset" +end + +--- Count words contributed by the non-marker portion of `tok` (after the marker's closing `)`). +--- Returns 0 if `tok` isn't a marker call or has no trailing content. +--- +--- TODO(Ed): Review this "not imported" assertion. Why do we have this if its not imported or is it actually? +--- `count_token_words_fn` is passed in (not imported) to keep this module free of pass-module dependencies. +--- Both call sites already import `word_count_eval.count_token_words`; they pass it as the 3rd arg. +--- @param tok string +--- @param word_counts table +--- @param count_token_words_fn fun(tok: string, wc: table): integer +--- @return integer +function M.count_marker_rest(tok, word_counts, count_token_words_fn) + local marker_end = M.find_marker_call_end(tok) + if not marker_end or marker_end >= #tok then return 0 end + local rest = M.trim(tok:sub(marker_end)) + if rest == "" then return 0 end + return count_token_words_fn(rest, word_counts) +end + -- ════════════════════════════════════════════════════════════════════════════ -- Section 5: load_word_counts -- ════════════════════════════════════════════════════════════════════════════ @@ -703,13 +708,12 @@ M.TAPE_ATOM_MACROS = { -- The check (`scripts/passes/static_analysis.lua :: check_gte_pipeline_fill`) walks each atom body, -- counts the consecutive nop words before every `gte_cmdw_*` invocation, and reports a finding if the count is below this minimum. -- --- PRE-FILL vs POST-FILL: this table models PRE-cmdw nops (retiring preceding C2 writes), --- NOT the post-cmdw input-latch window. +-- PRE-FILL vs POST-FILL: this table models PRE-cmdw nops (retiring preceding C2 writes). -- The PSX-SPX pipeline timings doc (`docs/psx-spx/docs/gtepipelinetimings.md`) measures a DIFFERENT number: -- the smallest N nops between `cop2` and `mtc2` to a specific input register at which the write no longer affects the output. -- For nearly all instructions, inputs latch in the first 0-4 cycles — the GTE snapshots its input register file early and works --- from internal pipeline storage afterward. The documented total cycle count is NOT the "do not touch inputs" window; --- the actual read window is much shorter. +-- from internal pipeline storage afterward. +-- The documented total cycle count is NOT the "do not touch inputs" window; the actual read window is much shorter. -- -- The `gte_rtpt()` / `gte_nclip()` wrapper macros in gte.h emit the pre-cmd nops internally (asm_words(nop, nop, ...)), -- but THOSE WRAPPERS ARE NOT USED INSIDE ATOM BODIES in this codebase. @@ -934,7 +938,7 @@ M.INSTRUCTION_LATENCY = { ["gte_nclip"] = 8, -- alias for nclip ["gte_avsz3"] = 5, ["gte_avsz4"] = 6, - -- Legacy single-cycle store helpers (gte_stotz, gte_stsxy3 are 1 cycle) + -- Single-cycle store helpers (gte_stotz, gte_stsxy3 are 1 cycle) ["gte_stotz"] = 1, ["gte_stsxy3"] = 1, -- High-level GTE helpers (gte_load_v0/v1/v2 do multiple lwc2s) diff --git a/scripts/elf_dwarf.lua b/scripts/elf_dwarf.lua index 2772e3b..6614ee4 100644 --- a/scripts/elf_dwarf.lua +++ b/scripts/elf_dwarf.lua @@ -1,10 +1,9 @@ ---- elf_dwarf.lua — ELF32 + DWARF + atoms source-map utilities for the F'' track. ---- +--- elf_dwarf.lua — ELF32 + DWARF + atoms source-map utilities. --- All ELF32 + DWARF-specific code lives here. --- --- **What this module contains:** --- - **Format-constant tables** (the byte-offset / opcode / size encyclopedias for ELF32, DWARF4 aranges, DWARF5 rnglists, DWARF line-program, MIPS). ---- Every constant carries a spec:` comment naming the spec section that defines it (convention established by F''). +--- Every constant carries a spec:` comment naming the spec section that defines it. --- - **I/O helpers**: little-endian byte read/write, ELF32 section walker, nm symbol reader, source-map parser, native directory glob. --- --- **Conventions:** tabs (1/level), EmmyLua annotations, no regex, @@ -14,8 +13,7 @@ -- Native dependencies -- ════════════════════════════════════════════════════════════════════════════ --- lfs is wired into package.cpath by `duffle_paths.lua` (vendored under `toolchain/lfs/lfs.dll`). --- Required here for native directory ops (replaces the ~56ms `dir /b` subprocess with ~2ms native). +-- lfs is wired into package.cpath by `duffle_paths.lua` (vendored under `toolchain/lfs/lfs.dll`). local lfs = require("lfs") local M = {} @@ -26,17 +24,17 @@ local M = {} -- (DWARF5 §7.5.5 "Tag Encodings" + Table 7.1; gcc emits these exact values for the DWARF3-extension and DWARF5 line units.) M.DW_TAG = { - compile_unit = 0x11, - subprogram = 0x2E, - variable = 0x34, - structure_type = 0x13, - member = 0x0D, - base_type = 0x24, - typedef = 0x2A, - pointer_type = 0x0F, - const_type = 0x26, - volatile_type = 0x27, - inlined_subroutine = 0x1D, + compile_unit = 0x11, + subprogram = 0x2E, + variable = 0x34, + structure_type = 0x13, + member = 0x0D, + base_type = 0x24, + typedef = 0x2A, + pointer_type = 0x0F, + const_type = 0x26, + volatile_type = 0x27, + inlined_subroutine = 0x1D, -- We index the canonical gcc-emitted tags. Anything else falls through. } @@ -102,6 +100,7 @@ local DW_FORM_implicit_const = 0x21 --- spec: MIPS o32 ABI §"Register Usage" — 32-bit general-purpose registers M.MIPS_BYTES_PER_WORD = 0x04 + -- ---------------------------------------------------------------------------- -- ELF32 (System V ABI gABI v1.2) -- ---------------------------------------------------------------------------- @@ -112,6 +111,10 @@ M.MIPS_BYTES_PER_WORD = 0x04 -- Example: e_shoff_offset = 0x21 means the 4-byte e_shoff field starts at -- string.sub byte 0x21 (= 33 in 1-indexed), i.e. file offset 0x20 (= 32). +-- TODO(Ed): I don't like baking the 1-index offset here. It should be "handled" +-- by a wrapper read/write function or inline as + 1 offset for lua index addressing compensation. +-- change this unless its just better to deal with this grime here than at the addressing site. + --- spec: System V ABI gABI v1.2 §"ELF Header" (Table 1) + §"Section Header Table" M.ELF32 = { magic_offset = 0x01, -- 4-byte magic "\127ELF" at file offset 0x00 @@ -236,7 +239,7 @@ function M.read_u16_le(buf, off) return buf:byte(off) + buf:byte(off + 0x01) * 0x00000100 end --- Pure-Lua 5.3 LEB128 readers (no `bit` library). `2^shift` arithmetic matches the existing F' parser. +-- Pure-Lua 5.3 LEB128 readers (no `bit` library). `2^shift` arithmetic matches the existing parser. -- Offsets are 0-based; returns (value, next_pos). -- Track A Task 10: promoted from `local function` to M.* exports so passes/dwarf_injection.lua -- can import them as file-scope locals per the 2nd-caller lift precedent @@ -393,327 +396,6 @@ local function read_form_value(buf, str_buf, pos, form) end end --- Index the .debug_info + .debug_abbrev sections of an existing ELF and collect one entry per "interesting" type DIE in the FIRST compilation unit. --- The index supports typed-views: --- index = { --- by_name = { ["V4_S2"] = {kind="structure_type", die_offset, byte_size, fields={...}}, --- ["U4"] = {kind="base_type", die_offset, byte_size, encoding="unsigned"}, --- ["MipsCode"] = {kind="typedef", die_offset, target_kind=..., target_die_offset=...} }, --- by_offset = { [die_offset] = {kind, name, ...} }, -- reverse lookup --- } --- --- @param info string -- .debug_info section bytes --- @param abbrev string -- .debug_abbrev section bytes --- @param str_buf string -- .debug_str section bytes (for DW_FORM_strp name resolution) --- @param abbrev_offset integer -- 0-based offset of the main CU's abbrev table --- @param cu_start integer|nil -- 0-based offset of the main CU (caller-known) --- @return table|nil, string|nil -- (index, error) -function M.index_main_cu_types(info, abbrev, str_buf, abbrev_offset, cu_start) - if not info or #info < 12 or not abbrev or not abbrev_offset then return nil, "missing input" end - str_buf = str_buf or "" - -- Index the main table at abbrev_offset. - -- The F' pass writes a trailing 0 byte after the main table; the G' pass may append additional codes (100-108) + a 0 terminator. - -- The walker may encounter a code that wasn't in the main table but exists later in the same .debug_abbrev; - -- on the first miss, walk the rest of the section to add any new abbrevs we encounter. - local abbrev_decls, err = parse_abbrev_table(abbrev, abbrev_offset) - if not abbrev_decls then return nil, err end - local abbrev_by_code = {} - for _, d in ipairs(abbrev_decls) do abbrev_by_code[d.code] = d end - - -- Resolve cu_start + cu_end_excl. - local cu_end_excl - if cu_start then - local ul = M.read_u32_le(info, cu_start + 1) - if ul == 0xFFFFFFFF then return nil, "DWARF64 not supported" end - cu_end_excl = cu_start + 4 + ul - else - local pos = 0 - cu_start = nil - while pos < #info do - local ul = M.read_u32_le(info, pos + 1) - if ul == 0xFFFFFFFF then break end - local unit_end = pos + 4 + ul - if unit_end > #info then break end - local unit_abbrev = M.read_u32_le(info, pos + 9) - if unit_abbrev == abbrev_offset then - cu_start = pos - cu_end_excl = unit_end - break - end - pos = unit_end - end - if not cu_start then return nil, "no CU matches abbrev_offset" end - end - - -- Walk the CU's DIE tree at 0-based offset cu_start + 12. - -- Emit a flat list of type-bearing DEIs; recursion handles nested children - -- (members inside structure_type, types inside subprograms, etc.). - -- A non-type DIE's subtree is SKIPPED by walking until the matching null. - local by_name = {} - local by_offset = {} - local pos_cursor = cu_start + 12 - -- Root DIE is the compile_unit; skip past it. - if pos_cursor >= cu_end_excl then return nil, "no DIE bytes" end - local root_decl_code - root_decl_code, pos_cursor = M.read_uleb128_at(info, pos_cursor) - if not root_decl_code then return nil, "truncated root DIE code" end - local root_decl = abbrev_by_code[root_decl_code] - if not root_decl then return nil, "unknown root DIE abbrev" end - -- Skip root DIE attributes. - for _, attr in ipairs(root_decl.attrs) do - local _, ne = read_form_value(info, str_buf, pos_cursor, attr.form) - if not ne then return nil, "truncated root attr" end - pos_cursor = ne - end - -- If root has no children, return an empty index. - if not root_decl.has_children or root_decl.has_children == 0 then - return { by_name = by_name, by_offset = by_offset } - end - - -- Helper: skip a subtree rooted at the current DIE (pos_cursor is positioned at the first child). - -- Walks down and right until the matching null terminator is consumed. - -- Returns the new pos_cursor. - -- For DIE trees that contain only the closed type + member shapes, the depth never exceeds 2 (type DIE -> member DIE -> null). - local function skip_subtree(pos) - local depth = 1 - while pos < cu_end_excl and depth > 0 do - local code = info:byte(pos + 1) - pos = pos + 1 - if code == 0 then - depth = depth - 1 - else - local d = abbrev_by_code[code] - if d and d.has_children ~= 0 then - depth = depth + 1 - end - end - end - return pos - end - - -- Pre-declare n_visited so the closure helpers can read it. - local n_visited = 0 - - -- Helper: read one DIE's attributes. - -- Returns (name, byte_size, encoding, type_ref, new_pos_cursor) on success; (nil, error_string) on truncation. - local function read_die_attributes(decl, pos) - local die_name = nil - local die_byte_size = nil - local die_encoding = nil - local die_type_ref = nil - for ai, attr in ipairs(decl.attrs) do - local before = pos - local val, ne = read_form_value(info, str_buf, pos, attr.form) - if not ne then - return nil, "truncated DIE attr" - end - pos = ne - if attr.name == M.DW_AT.name then - die_name = val - elseif attr.name == M.DW_AT.byte_size and (decl.tag == M.DW_TAG.base_type or decl.tag == M.DW_TAG.structure_type) then - die_byte_size = val - elseif attr.name == M.DW_AT.encoding and decl.tag == M.DW_TAG.base_type then - die_encoding = val - elseif attr.name == M.DW_AT.type then - die_type_ref = val - end - end - return die_name, die_byte_size, die_encoding, die_type_ref, pos - end - - -- Helper: read structure_type members. Returns (member_fields, new_pos). - local function read_member_fields(decl, pos) - local fields = {} - while pos < cu_end_excl do - local mcode = info:byte(pos + 1) - if mcode == 0 then - pos = pos + 1 - break - end - local mdecl = abbrev_by_code[mcode] - if not mdecl then return nil, "unknown member abbrev" end - pos = pos + 1 - local mname, mtype_ref, moffset - for _, a in ipairs(mdecl.attrs) do - local v, ne = read_form_value(info, str_buf, pos, a.form) - if not ne then return nil, "truncated member attr" end - pos = ne - if a.name == M.DW_AT.name then mname = v - elseif a.name == M.DW_AT.type then mtype_ref = v - elseif a.name == M.DW_AT.data_member_location then moffset = v end - end - fields[#fields + 1] = { name = mname, type_offset = mtype_ref, offset = moffset } - end - return fields, pos - end - - -- Top-level walker: iterate siblings. - -- For each DIE, decide whether to record (if type), descend (if structure_type with members), or skip its subtree. - while pos_cursor < cu_end_excl do - local code = info:byte(pos_cursor + 1) - if code == 0 then pos_cursor = pos_cursor + 1; break end - local decl = abbrev_by_code[code] - if not decl then - -- Lazily scan the rest of the section for the missing code. - -- The G' pass appends new abbrevs (100-108) after the main table's terminator. - -- On the first miss, walk the rest of the section. - local scan_pos = 0 - while scan_pos < #abbrev do - if abbrev:byte(scan_pos + 1) == 0 then - scan_pos = scan_pos + 1 - goto continue - end - local new_table, e3 = parse_abbrev_table(abbrev, scan_pos) - if not new_table then - scan_pos = scan_pos + 1 - goto continue - end - for _, d in ipairs(new_table) do - if not abbrev_by_code[d.code] then - abbrev_by_code[d.code] = d - end - end - -- Find the terminator (0 byte) of this table. - local term = M.find_abbrev_table_end(abbrev, scan_pos) - if not term then break end - scan_pos = term + 1 - ::continue:: - end - decl = abbrev_by_code[code] - if not decl then - return nil, string.format("unknown abbrev %d at offset 0x%x", code, pos_cursor) - end - end - local die_offset = pos_cursor - pos_cursor = pos_cursor + 1 - local read_result = { read_die_attributes(decl, pos_cursor) } - if #read_result == 2 then - return nil, read_result[2] - end - local die_name, die_byte_size, die_encoding, die_type_ref, pos_after_attrs - = read_result[1], read_result[2], read_result[3], read_result[4], read_result[5] - n_visited = n_visited + 1 - local is_type = ( - decl.tag == M.DW_TAG.base_type - or decl.tag == M.DW_TAG.structure_type - or decl.tag == M.DW_TAG.typedef - or decl.tag == M.DW_TAG.pointer_type - or decl.tag == M.DW_TAG.const_type) - - local member_fields = nil - pos_cursor = pos_after_attrs - if decl.tag == M.DW_TAG.structure_type and decl.has_children ~= 0 then - local f, np = read_member_fields(decl, pos_cursor) - if not f then return nil, np end - member_fields = f - pos_cursor = np - elseif decl.has_children ~= 0 then - -- Skip the subtree (e.g., DW_TAG_subprogram, DW_TAG_variable, etc.). - pos_cursor = skip_subtree(pos_cursor) - end - - if is_type and die_name then - local kind - if decl.tag == M.DW_TAG.base_type then kind = "base_type" - elseif decl.tag == M.DW_TAG.structure_type then kind = "structure_type" - elseif decl.tag == M.DW_TAG.typedef then kind = "typedef" - elseif decl.tag == M.DW_TAG.pointer_type then kind = "pointer_type" - elseif decl.tag == M.DW_TAG.const_type then kind = "const_type" end - local entry = { - kind = kind, - name = die_name, - die_offset = die_offset, - byte_size = die_byte_size, - encoding = die_encoding, - type_ref = die_type_ref, - fields = member_fields, - } - by_name[die_name] = entry - by_offset[die_offset] = entry - end - end - return { by_name = by_name, by_offset = by_offset } -end - --- Resolve a chain of pointer + const + typedef + structure_type down to a canonical struct or base type. --- The returned entry has kind, name, byte_size, and (for structure_type) fields with {name, offset, type_name, pointer_depth}. --- Returns nil if the chain cannot be resolved (e.g., missing DIE). --- @param index table -- M.index_main_cu_types result --- @param start_offset integer -- CU-relative DW_FORM_ref4 offset of the start --- @param max_depth integer -- cycle protection --- @return table|nil -- {kind, name, byte_size, fields?, pointer_depth} -function M.resolve_type_chain(index, start_offset, max_depth) - max_depth = max_depth or 16 - if not index or not start_offset then return nil end - local chain = {} - local cur_offset = start_offset - local depth = 0 - while cur_offset and depth < max_depth do - local entry = index.by_offset[cur_offset] - if not entry then return nil end - chain[#chain + 1] = entry - if entry.kind == "pointer_type" then cur_offset = entry.type_ref - elseif entry.kind == "const_type" then cur_offset = entry.type_ref - elseif entry.kind == "typedef" then cur_offset = entry.type_ref - else - break - end - depth = depth + 1 - end - -- Compute pointer_depth = number of pointer/const wrappers. - local pointer_depth = 0 - for _, e in ipairs(chain) do - if e.kind == "pointer_type" then pointer_depth = pointer_depth + 1 end - end - -- The last entry is the "naked" type. - local naked = chain[#chain] - if not naked then return nil end - -- If the last entry is a structure_type, resolve each field's type name - -- + pointer depth too (for `bind_args` shape expansion). - if naked.kind == "structure_type" and naked.fields then - local fields = {} - for _, f in ipairs(naked.fields) do - local field_entry = f.type_offset and index.by_offset[f.type_offset] or nil - local field_chain = {} - local d = 0 - local co = f.type_offset - while co and d < 16 do - local e2 = index.by_offset[co] - if not e2 then break end - field_chain[#field_chain + 1] = e2 - if e2.kind == "pointer_type" or e2.kind == "const_type" or e2.kind == "typedef" then - co = e2.type_ref - else - break - end - d = d + 1 - end - local fpd = 0 - for _, x in ipairs(field_chain) do if x.kind == "pointer_type" then fpd = fpd + 1 end end - fields[#fields + 1] = { - name = f.name, - offset = f.offset, - type_name = (field_chain[#field_chain] and field_chain[#field_chain].name) or "?", - pointer_depth = fpd, - } - end - return { - kind = "structure_type", - name = naked.name, - byte_size = naked.byte_size, - fields = fields, - pointer_depth = pointer_depth, - } - end - return { - kind = naked.kind, - name = naked.name, - byte_size = naked.byte_size, - encoding = naked.encoding, - pointer_depth = pointer_depth, - } -end - --- Return a 4-byte little-endian byte string for `value`. --- Caller concatenates with `..` if composing multi-word blobs. --- **Byte weights** written as `0x100` etc. (see `M.read_u32_le` for rationale). @@ -847,10 +529,6 @@ end --- Read ELF symbol addresses by walking the `.symtab` + `.strtab` sections directly (no `nm` subprocess). --- Returns a map `{name -> {addr, size_bytes}}` for every `code_` symbol. --- ---- **Why direct parsing instead of `mipsel-none-elf-nm -S`?** ---- The `nm` subprocess costs ~50ms per spawn on Windows (cmd.exe + mipsel-none-elf-nm.exe). Parsing `.symtab` ourselves is ~0ms. ---- Same return shape, same `code_` prefix filter. ---- --- **Conventions:** --- - ELF32 symtab entry = 16 bytes (`st_name:4 + st_value:4 + st_size:4 + st_info:1 + st_other:1 + st_shndx:2`). 1-indexed for Lua string.sub. --- - We filter on STB_GLOBAL (high nibble of st_info = 1) to match `nm`'s default (external symbols only). STB_WEAK excluded. @@ -963,9 +641,8 @@ function M.uleb128(n) end --- SLEB128 (Signed Little-Endian Base 128) encoder. Returns the byte string for the integer `n` (may be negative). ---- Algorithm differs from ULEB128 by the termination condition: stop when ---- the remaining bits can be inferred from the sign bit in the last byte's ---- 7-bit data payload. +--- Algorithm differs from ULEB128 by the termination condition: +--- stop when the remaining bits can be inferred from the sign bit in the last byte's 7-bit data payload. --- - If `n == 0` (no more value bits) AND bit 6 of the data = 0 → positive terminator (sign bit says "zero-extend"). --- - If `n == -1` (sign-extended all-1s) AND bit 6 of the data = 1 → negative terminator (sign bit says "one-extend"). --- @@ -992,35 +669,26 @@ end -- I/O helpers: atoms source-map + native directory glob -- ════════════════════════════════════════════════════════════════════════════ ---- Parse a FORMAT_VERSION `*.atoms.sourcemap.txt` file. ---- Returns `{name -> {total = N, words = {{pos, line}, ...}}}`. +--- Parse a FORMAT_VERSION atoms-meta file (sourcemap or provenance). +--- Shared by M.parse_source_map_file + M.parse_provenance_file. +--- The two callers differ only in how they parse WORD lines; that's `extract_word(line)`. +--- Returns the standard `{name -> {total, words}}` shape. --- Returns `{}` on format-version mismatch (and logs to stderr). ---- ---- **Wire format** (emitted by `passes/atoms_source_map.lua`): ---- ``` ---- # FORMAT_VERSION ---- ATOM "" ---- WORD LINE TEXT ---- ... ---- ENDATOM ---- ``` ---- ---- **Conventions:** the in-memory shape uses `{pos, line, text}` ---- (`atoms_source_map.lua:142`); the `.txt` file uses `WORD ` so the parser maps `n` → `pos` field name. ---- @param sm_path Path ---- @param expected_version integer -- expected FORMAT_VERSION line +--- @param path string +--- @param expected_version integer +--- @param extract_word fun(line: string): table|nil -- caller-supplied per-line parser --- @return table -function M.parse_source_map_file(sm_path, expected_version) +function M.parse_atom_records(path, expected_version, extract_word) local out = {} local cur_name, cur_words = nil, {} - for raw in io.lines(sm_path) do + for raw in io.lines(path) do local line = raw if line:match("^#") then local ver = line:match("^# FORMAT_VERSION%s+(%d+)") if ver and tonumber(ver) ~= expected_version then io.stderr:write(string.format( - "[elf_dwarf.parse_source_map_file] source-map version mismatch (got %s, expected %d) in %s\n", - ver, expected_version, sm_path)) + "[elf_dwarf.parse_atom_records] version mismatch (got %s, expected %d) in %s\n", + ver, expected_version, path)) return {} end -- skip other comments @@ -1040,16 +708,42 @@ function M.parse_source_map_file(sm_path, expected_version) end cur_name, cur_words = nil, {} elseif line:sub(1, 4) == "WORD" and cur_name then - -- WORD LINE TEXT - local _, n, _, src_line = line:find("WORD%s+(%d+)%s+LINE%s+(%d+)") - if n and src_line then - cur_words[#cur_words + 1] = { pos = tonumber(n), line = tonumber(src_line) } + local field = extract_word(line) + if field then + cur_words[#cur_words + 1] = field end end end return out end +--- Parse a FORMAT_VERSION `*.atoms.sourcemap.txt` file. +--- Returns `{name -> {total = N, words = {{pos, line}, ...}}}`. +--- Returns `{}` on format-version mismatch (and logs to stderr). +--- +--- **Wire format** (emitted by `passes/atoms_source_map.lua`): +--- ``` +--- # FORMAT_VERSION +--- ATOM "" +--- WORD LINE TEXT +--- ... +--- ENDATOM +--- ``` +--- +--- **Conventions:** the in-memory shape uses `{pos, line, text}` +--- (`atoms_source_map.lua:142`); the `.txt` file uses `WORD ` so the parser maps `n` → `pos` field name. +--- @param sm_path Path +--- @param expected_version integer -- expected FORMAT_VERSION line +--- @return table +function M.parse_source_map_file(sm_path, expected_version) + return M.parse_atom_records(sm_path, expected_version, function(line) + local _, n, _, src_line = line:find("WORD%s+(%d+)%s+LINE%s+(%d+)") + if n and src_line then + return { pos = tonumber(n), line = tonumber(src_line) } + end + end) +end + --- Parse a FORMAT_VERSION `*.atoms.provenance.txt` file. --- Returns `{name -> {total = N, words = {{pos, call_file, call_line, comp_name, comp_file, comp_line}, ...}}}`. --- Returns `{}` on format-version mismatch (and logs to stderr). @@ -1072,64 +766,35 @@ end --- @param expected_version integer -- expected FORMAT_VERSION line --- @return table function M.parse_provenance_file(prov_path, expected_version) - local out = {} - local cur_name, cur_words = nil, {} - for raw in io.lines(prov_path) do - local line = raw - if line:match("^#") then - local ver = line:match("^# FORMAT_VERSION%s+(%d+)") - if ver and tonumber(ver) ~= expected_version then - io.stderr:write(string.format( - "[elf_dwarf.parse_provenance_file] provenance version mismatch (got %s, expected %d) in %s\n", - ver, expected_version, prov_path)) - return {} - end - -- skip other comments - elseif line:sub(1, 4) == "ATOM" then - -- ATOM "" - local _, _, name = line:find("ATOM%s+(%S+)%s+\"[^\"]*\"%s+(%d+)") - if name then - cur_name = name - cur_words = {} - out[name] = { total = 0, words = cur_words } - end - elseif line == "ENDATOM" then - if cur_name and out[cur_name] then - out[cur_name].total = #cur_words - end - cur_name, cur_words = nil, {} - elseif line:sub(1, 4) == "WORD" and cur_name then - -- Two accepted shapes: - -- WORD CALL : RAW - -- WORD CALL : MACRO ":" - local pos, call_file, call_line, comp_name, comp_file, comp_line = - line:match('WORD%s+(%d+)%s+CALL%s+(.-):(%d+)%s+MACRO%s+(%S+)%s+"([^"]*):(%d+)"') - if pos then - cur_words[#cur_words + 1] = { - pos = tonumber(pos), - call_file = call_file, - call_line = tonumber(call_line), - comp_name = comp_name, - comp_file = comp_file, - comp_line = tonumber(comp_line), - } - else - -- RAW row. - local raw_pos, raw_file, raw_line = line:match('WORD%s+(%d+)%s+CALL%s+(.-):(%d+)%s+RAW') - if raw_pos then - cur_words[#cur_words + 1] = { - pos = tonumber(raw_pos), - call_file = raw_file, - call_line = tonumber(raw_line), - comp_name = nil, - comp_file = nil, - comp_line = nil, - } - end - end + return M.parse_atom_records(prov_path, expected_version, function(line) + -- Two accepted shapes: + -- WORD CALL : RAW + -- WORD CALL : MACRO ":" + local pos, call_file, call_line, comp_name, comp_file, comp_line = + line:match('WORD%s+(%d+)%s+CALL%s+(.-):(%d+)%s+MACRO%s+(%S+)%s+"([^"]*):(%d+)"') + if pos then + return { + pos = tonumber(pos), + call_file = call_file, + call_line = tonumber(call_line), + comp_name = comp_name, + comp_file = comp_file, + comp_line = tonumber(comp_line), + } end - end - return out + -- RAW row. + local raw_pos, raw_file, raw_line = line:match('WORD%s+(%d+)%s+CALL%s+(.-):(%d+)%s+RAW') + if raw_pos then + return { + pos = tonumber(raw_pos), + call_file = raw_file, + call_line = tonumber(raw_line), + comp_name = nil, + comp_file = nil, + comp_line = nil, + } + end + end) end return M diff --git a/scripts/passes/annotation.lua b/scripts/passes/annotation.lua index 321f7c8..8753825 100644 --- a/scripts/passes/annotation.lua +++ b/scripts/passes/annotation.lua @@ -148,7 +148,7 @@ end --- @param pipe_ctx PipeCtx --- @param findings Findings local function check_binds_struct_exists(a, pipe_ctx, findings) - if not a.binds then return end + if not a.binds then return end if pipe_ctx.binds_index[a.binds] then return end findings.warnings[#findings.warnings + 1] = { line = a.line, @@ -164,7 +164,7 @@ end --- @param wc table -- the shared word-count table (from ctx.shared.word_counts) --- @param findings Findings local function check_macro_word_drift(m, wc, findings) - local declared = wc[m.name] + local declared = wc[m.name] if not declared then findings.errors[#findings.errors + 1] = { line = m.line, @@ -392,8 +392,7 @@ end --- Any source referencing an R_X that's NOT in the registry will trip the new check; a single pass-level info entry --- (emitted only when at least one such rejection lands in this source) tells users where to look. --- ---- Track A Task 13 added the proper `enum_alias_membership` per_source rule; ---- this is the stop-gap until users migrate off raw C-ABI register names. +--- This check is a stop-gap until users migrate off raw C-ABI register names. --- @param _src SourceFile --- @param pipe_ctx PipeCtx --- @param findings Findings @@ -409,7 +408,7 @@ local function check_wave_context_migration(_src, pipe_ctx, findings) line = 0, msg = "wave-context removed; opt in via #define atom_reg in mips.h " .. "(every R_ that should be visible to the annotation pass " - .. "must be enum-declared with the bare atom_reg marker; see Track A Task 21)", + .. "must be enum-declared with the bare atom_reg marker)", } return end diff --git a/scripts/passes/atoms_source_map.lua b/scripts/passes/atoms_source_map.lua index ffd9908..d1fcb15 100644 --- a/scripts/passes/atoms_source_map.lua +++ b/scripts/passes/atoms_source_map.lua @@ -36,9 +36,8 @@ --- --- Marker calls (`atom_label(...)`, `atom_offset(...)`) emit 0 `.word`s. --- They share the same walking convention as `passes/offsets.lua :: scan_atom_body`: ---- markers do NOT advance the word-offset counter, but if a marker is bundled on the same token with a ---- trailing instruction (e.g. `atom_label(foo) load_half_u(...)`), ---- the trailing instruction's word count is added. This matches `offsets.lua :: count_marker_rest`. +--- Markers do NOT advance the word-offset counter, but if a marker is bundled on the same token with a trailing instruction +--- (e.g. `atom_label(foo) load_half_u(...)`), the trailing instruction's word count is added. This matches `offsets.lua :: count_marker_rest`. --- --- **Conventions:** tabs (1/level), EmmyLua annotations, no regex, --- Lua 5.3 compatible. @@ -84,68 +83,6 @@ local OFFSET_MARKER = "atom_offset" -- Helpers -- ════════════════════════════════════════════════════════════════════════════ ---- True iff the leading identifier of `tok` is a marker call (`atom_label` / `atom_offset`). ---- Mirrors `passes/offsets.lua :: is_marker_token` (which is file-local there). ---- @param tok string ---- @return boolean -local function is_marker_token(tok) - local leading = duffle.read_ident(tok, 1) - return leading == LABEL_MARKER or leading == OFFSET_MARKER -end - ---- Count words contributed by the non-marker portion of `tok` (after the marker's closing `)`). ---- Mirrors offsets.lua:182 `count_marker_rest`. ---- Returns 0 if there's no trailing content after the marker call. ---- @param tok string ---- @param wc table ---- @return integer -local function count_marker_rest(tok, wc) - local marker_end = duffle.find_marker_call_end(tok) - if not marker_end or marker_end >= #tok then return 0 end - local rest = duffle.trim(tok:sub(marker_end)) - if rest == "" then return 0 end - return count_token_words(rest, wc) -end - ---- Compute per-word entries for an atom. ---- Shared between the canonical text form and the gdb-runtime form. ---- ---- Returns a list of `{pos, line, text}` entries + the total word count. ---- Markers contribute 0 entries (the marker call emits 0 `.word`s). ---- @param atom table -- one entry of scan.atoms / scan.raw_atoms ---- @param src table -- SourceFile (has .scan with .line_of(), .path) ---- @param wc table -- shared.word_counts ---- @return table[], integer -local function compute_word_entries(atom, src, wc) - local entries = {} - local pos = 0 - for _, t in ipairs(atom.body_tokens) do - local tok = t.tok - local rel = t.rel - - local words - if is_marker_token(tok) then - words = count_marker_rest(tok, wc) - else - words = count_token_words(tok, wc) - end - - if words > 0 then - -- Source line for THIS token = line containing byte offset `atom.body_off + rel`. - -- `src.scan.line_of(...)` is O(log N) via LineIndex. - local line = src.scan.line_of(atom.body_off + rel) - -- Flatten newlines + tabs in TEXT to spaces so each WORD entry fits on one physical line. - -- The gdb Python parser (or our pure-gdb parser) does line-based splits; multi-line TEXT would break it. - local text = duffle.trim(tok):gsub("[\t\r\n]+", " ") - for _ = 1, words do - entries[#entries + 1] = { pos = pos, line = line, text = text } - pos = pos + 1 - end - end - end - return entries, pos -end - -- ════════════════════════════════════════════════════════════════════════════ -- Provenance emission -- ════════════════════════════════════════════════════════════════════════════ @@ -155,7 +92,7 @@ local MAC_PREFIX = "mac_" local MAC_PREFIX_LEN = 4 --- Strip the `mac_` prefix from a token's leading identifier. ---- Returns nil if the identifier doesn't start with `mac_` +--- Returns nil if the identifier doesn't start with `mac_` --- (so non-component tokens like `load_half_u`, `nop2`, `gte_cmdw_*` fall through cleanly). --- @param tok string --- @return string|nil @@ -168,25 +105,48 @@ local function strip_mac_prefix_from_token(tok) return nil end ---- Compute per-word provenance entries for an atom. Mirrors `compute_word_entries` but additionally classifies each emitted `.word` as either: ---- - `RAW` — emitted by a direct instruction token (no component provenance) ---- - `MACRO X` — emitted by a `mac_X(...)` component invocation, with the component's definition file:line resolved from `ctx.shared.components`. ---- ---- Returns a list of `{pos, line, text, comp_name, comp_line, comp_path, body_line}` entries + the total word count. ---- `comp_name` is nil for RAW rows. `body_line` is the line of THIS specific word in the macro body (component source file, not the caller's source); ---- it differs from `comp_line` (= the macro signature line) for every body word whose macro-body token is on a different physical line. ---- `body_line` is `nil` for RAW rows and for component words whose component declaration could not be indexed (older pass combinations / external macros). ---- ---- The per-word body-line lookup mirrors `passes/dwarf_injection.lua :: compute_invocation_body_lines`: ---- walk the component's pre-tokenized body in lockstep with `count_token_words` and attribute the source line via `src.scan.line_of(...)` to each emitted `.word`. ---- Atom labels (`atom_label(...)`) emit 0 `.word`s and are skipped to stay aligned with the macro-side word-counting contract. ---- @param atom table -- one entry of scan.atoms / scan.raw_atoms ---- @param src table -- SourceFile (has .scan with .line_of(), .path) ---- @param wc table -- shared.word_counts ---- @param comp table -- shared.components map: bare_name -> {name=, line=, path=, kind=} ---- @param comp_body_index table -- per-source component body index: bare_name -> {body_off, body_tokens, line_of} +--- Fetch the per-word body lines for a `mac_X(...)` invocation. +--- Walks the component's pre-tokenized body in lockstep with `count_token_words` and attributes each emitted `.word` +--- to a source line via `idx.line_of(...)`. +--- Atom labels (`atom_label(...)`) emit 0 `.word`s and are skipped. +--- @param bare string|nil -- the bare component name (e.g. `gte_load_tri_verts`) +--- @param comp_body_index table +--- @param wc table +--- @return table|nil -- list of source lines, 1-based by word position +local function fetch_body_lines(bare, comp_body_index, wc) + if not (bare and comp_body_index) then return nil end + local idx = comp_body_index[bare] + if not (idx and idx.body_tokens and idx.line_of) then return nil end + local lines = {} + for _, bt in ipairs(idx.body_tokens) do + local bt_tok = duffle.trim(bt.tok or "") + if bt_tok ~= "" then + local leading = duffle.read_ident(bt_tok, 1) + local bt_words + if leading == "atom_label" or leading == "atom_offset" then + bt_words = 0 + else + bt_words = count_token_words(bt_tok, wc) + end + if bt_words > 0 then + local body_line = idx.line_of(idx.body_off + bt.rel) + for _ = 1, bt_words do lines[#lines + 1] = body_line end + end + end + end + return lines +end + +--- Unified per-word entry walker. `mode` is "sourcemap" (3 fields) or "provenance" (8 fields including component + body-line lookup). +--- Returns (entries, total_words). Markers contribute 0 entries. +--- @param atom table +--- @param src table +--- @param wc table +--- @param mode string -- "sourcemap" | "provenance" +--- @param comp table|nil -- shared.components map (provenance only) +--- @param comp_body_index table|nil -- per-source body index (provenance only) --- @return table[], integer -local function compute_provenance_entries(atom, src, wc, comp, comp_body_index) +local function compute_word_entries(atom, src, wc, mode, comp, comp_body_index) local entries = {} local pos = 0 for _, t in ipairs(atom.body_tokens) do @@ -194,70 +154,46 @@ local function compute_provenance_entries(atom, src, wc, comp, comp_body_index) local rel = t.rel local words - if is_marker_token(tok) then - words = count_marker_rest(tok, wc) + if duffle.is_marker_token(tok) then + words = duffle.count_marker_rest(tok, wc, count_token_words) else words = count_token_words(tok, wc) end - -- Resolve component provenance for this token (if any). - local comp_name = nil - local comp_line = nil - local comp_path = nil - local comp_kind = nil - local bare = strip_mac_prefix_from_token(tok) - if bare and comp and comp[bare] then - comp_name = bare - comp_line = comp[bare].line - comp_path = comp[bare].path - comp_kind = comp[bare].kind - end - - -- Per-word body lines: lazily allocate from the indexed component body the first time we see a `mac_X(...)` call to a given component. - -- We allocate ONE full body_lines vector per (call) and consume it sequentially; - -- if a single atom calls the same component more than once, each call refetches its own vector. - -- (Today no atom calls the same `mac_X(...)` twice, but the refetch keeps the semantics correct even if that changes.) - local body_lines = nil - local function fetch_body_lines() - if not (bare and comp_body_index) then return nil end - local idx = comp_body_index[bare] - if not (idx and idx.body_tokens and idx.line_of) then return nil end - local lines = {} - for _, bt in ipairs(idx.body_tokens) do - local bt_tok = duffle.trim(bt.tok or "") - if bt_tok ~= "" then - local leading = duffle.read_ident(bt_tok, 1) - local bt_words - if leading == "atom_label" or leading == "atom_offset" then - bt_words = 0 - else - bt_words = count_token_words(bt_tok, wc) - end - if bt_words > 0 then - local body_line = idx.line_of(idx.body_off + bt.rel) - for _ = 1, bt_words do lines[#lines + 1] = body_line end - end - end + -- Provenance-only: resolve component + body_lines (one fetch per token). + local comp_name, comp_line, comp_path, comp_kind + local body_lines + if mode == "provenance" then + local bare = strip_mac_prefix_from_token(tok) + if bare and comp and comp[bare] then + comp_name = bare + comp_line = comp[bare].line + comp_path = comp[bare].path + comp_kind = comp[bare].kind end - return lines + if comp_name then body_lines = fetch_body_lines(bare, comp_body_index, wc) end end if words > 0 then local line = src.scan.line_of(atom.body_off + rel) local text = duffle.trim(tok):gsub("[\t\r\n]+", " ") - -- Fetch body_lines ONCE per token (one mac_X(...) call exhausts N body words). - if comp_name then body_lines = fetch_body_lines() end for i = 1, words do - entries[#entries + 1] = { - pos = pos, - line = line, - text = text, - comp_name = comp_name, - comp_line = comp_line, - comp_path = comp_path, - comp_kind = comp_kind, - body_line = body_lines and body_lines[i], - } + local entry + if mode == "provenance" then + entry = { + pos = pos, + line = line, + text = text, + comp_name = comp_name, + comp_line = comp_line, + comp_path = comp_path, + comp_kind = comp_kind, + body_line = body_lines and body_lines[i], + } + else -- "sourcemap" (default) + entry = { pos = pos, line = line, text = text } + end + entries[#entries + 1] = entry pos = pos + 1 end end @@ -282,7 +218,7 @@ end local function emit_provenance_stanza(src, atom, wc, comp, comp_body_index) local lines = {} local rel_path = src.path:gsub("\\", "/") - local entries, total = compute_provenance_entries(atom, src, wc, comp, comp_body_index) + local entries, total = compute_word_entries(atom, src, wc, "provenance", comp, comp_body_index) -- ATOM header line with placeholder total (patched after we know it). lines[#lines + 1] = string.format('ATOM %s "%s" 0', atom.raw_name or atom.name, rel_path) @@ -465,7 +401,6 @@ end --- --- Each command is a static sequence of `printf` / `tbreak` / `if ... end` blocks. --- The Lua pass emits N atoms' worth of lines — no runtime iteration. ---- With 7 atoms + ~200 word entries, the runtime file is ~2000 lines, all auto-generated, no human edit ever. --- @param lines table -- output line buffer (mutated in place) --- @param matched table -- list of atom records from `build_atom_table` local function append_gdb_commands(lines, matched) @@ -595,8 +530,7 @@ local function append_gdb_commands(lines, matched) lines[#lines + 1] = "" -- ── show_c2 ── - -- GTE data regs (COP2). pcsx-redux's gdb stub doesn't expose COP2 (only - -- 72 regs: 32 GPR + COP0 + FPR). + -- GTE data regs (COP2). pcsx-redux's gdb stub doesn't expose COP2 (only 72 regs: 32 GPR + COP0 + FPR). -- curl http://localhost:8080/api/v1/lua/gte -- We keep the command definition as a stub that points the user at the plugin. lines[#lines + 1] = "define show_c2" @@ -713,8 +647,9 @@ local M = {} --- but invoked from many source files (every atom body that calls `mac_X(...)`). --- The body_offset + body_tokens + line_of live with the declaration source, so a per-source index would miss invocations from other sources. --- ---- The cross-source index is keyed by the bare component name (`gte_load_tri_verts`, NOT `ac_gte_load_tri_verts`) — `strip_mac_prefix_from_token` strips the `mac_` prefix ---- from call-site identifiers and yields that exact bare name; matching it here keeps the lookup aligned with the `ctx.shared.components` map's keying convention. +--- The cross-source index is keyed by the bare component name (`gte_load_tri_verts`, NOT `ac_gte_load_tri_verts`) +--- `strip_mac_prefix_from_token` strips the `mac_` prefix from call-site identifiers and yields that exact bare name; +--- matching it here keeps the lookup aligned with the `ctx.shared.components` map's keying convention. --- First declaration wins (subsequent redeclarations would collide; today's sources declare each component exactly once). --- @param ctx PassCtx --- @return table -- {[comp_name] = {body_off, body_tokens, line_of}} diff --git a/scripts/passes/components.lua b/scripts/passes/components.lua index ddf43e4..5aa16c8 100644 --- a/scripts/passes/components.lua +++ b/scripts/passes/components.lua @@ -96,16 +96,24 @@ local GEN_SUBDIR = "gen" local M = {} -- ════════════════════════════════════════════════════════════════════════════ --- Function-args extraction (precedes MipsAtomComp_Proc_ invocations) +-- Back-walk helpers (composed into the 2 entry points below: find_function_args_for + preceding_comment_block) -- ════════════════════════════════════════════════════════════════════════════ --- Find the LAST occurrence of `name + "("` in `source[1..before_pos]`. --- Returns the position of the open paren, or nil if not found. --- @param source string --- @param name string --- @param before_pos integer --- @return integer|nil -local function find_last_name_open_paren(source, name, before_pos) +--- Find the args of the function declaration that immediately precedes a `MipsAtomComp_Proc_` invocation of the given name. +--- Returns the args string (e.g., `"U4 off, U4 code, U1 r, U1 g, U1 b"`) or nil if no function declaration is found. +--- +--- Convention: function form is +--- `FI_ MipsAtom ac_X(args) MipsAtomComp_Proc_(ac_X, { body })` +--- We find the LAST occurrence of `"ac_X("` before `before_pos` and extract the args from inside the parens. +--- We then verify the preceding context ends with `MipsAtom` +--- (the function-decl keyword with possible qualifiers between). +--- +--- @param source string +--- @param name string +--- @param before_pos integer +--- @return string|nil +local function find_function_args_for(source, name, before_pos) + -- Find the LAST occurrence of `name + "("` in `source[1..before_pos]`. local name_open = name .. "(" local last_idx = nil local scan_pos = 1 @@ -117,24 +125,6 @@ local function find_last_name_open_paren(source, name, before_pos) last_idx = found scan_pos = found + #name_open end - return last_idx -end - ---- Find the args of the function declaration that immediately precedes a `MipsAtomComp_Proc_` invocation of the given name. ---- Returns the args string (e.g., `"U4 off, U4 code, U1 r, U1 g, U1 b"`) or nil if no function declaration is found. ---- ---- Convention: function form is ---- `FI_ MipsAtom ac_X(args) MipsAtomComp_Proc_(ac_X, { body })` ---- We find the LAST occurrence of `"ac_X("` before `before_pos` and extract the args from inside the parens. ---- We then verify the preceding context ends with `MipsAtom` ---- (the function-decl keyword with possible qualifiers between). ---- ---- @param source string ---- @param name string ---- @param before_pos integer ---- @return string|nil -local function find_function_args_for(source, name, before_pos) - local last_idx = find_last_name_open_paren(source, name, before_pos) if not last_idx then return nil end -- Verify the preceding context ends with "MipsAtom" (with possible qualifiers between). @@ -153,100 +143,11 @@ local function find_function_args_for(source, name, before_pos) return inner end --- ════════════════════════════════════════════════════════════════════════════ --- Preceding-comment-block extraction --- ════════════════════════════════════════════════════════════════════════════ - --- Skip whitespace (space/tab/newline/CR) backward from `pos`, returning the position of the first non-whitespace char. --- @param source string --- @param pos integer --- @return integer -local function skip_ws_backward(source, pos) - local back = pos - 1 - while back > 0 do - local ch = source:sub(back, back) - if ch == " " or ch == "\t" or ch == "\n" or ch == "\r" then - back = back - 1 - else - break - end - end - return back -end - --- Find the opening `/*` for a block comment whose `*/` ends at `close_pos`. --- Returns the position of `/`, or nil if not found. --- @param source string --- @param close_pos integer -- position of the closing `*` of `*/` --- @return integer|nil -local function find_block_comment_open(source, close_pos) - local prefix = source:sub(1, close_pos - 1) - local open_at = nil - for scan = #prefix - 1, 1, -1 do - if prefix:sub(scan, scan + 1) == "/*" then - open_at = scan - break - end - end - return open_at -end - --- Walk back from `open_at` over leading spaces + tabs to include the indentation before the `/*` in the captured comment. --- @param source string --- @param open_at integer --- @return integer -local function extend_left_over_indent(source, open_at) - local start = open_at - while start > 1 do - local ch = source:sub(start - 1, start - 1) - if ch == " " or ch == "\t" then - start = start - 1 - else - break - end - end - return start -end - --- Walk back from `line_end` to the start of the source line (the most recent `\n` or position 1). --- @param source string --- @param line_end integer --- @return integer -local function find_line_start(source, line_end) - local start = line_end - while start > 1 and source:sub(start - 1, start - 1) ~= "\n" do - start = start - 1 - end - return start -end - --- (internal) Capture one `/* ... */` block comment whose closing `*/` --- ends at `close_end_pos`. Returns (block_text, new_scan_pos) where `new_scan_pos` --- is where to continue scanning for more comments, or nil if no block comment was found. -local function capture_block_comment(source, close_end_pos) - local open_at = find_block_comment_open(source, close_end_pos) - if not open_at then return nil end - local block_start = extend_left_over_indent(source, open_at) - return source:sub(block_start, close_end_pos), block_start -end - --- (internal) Capture one `// ...` line comment ending at `line_end_pos`. --- Returns (comment_text, new_scan_pos) or nil if the line is not a `//` comment. -local function capture_line_comment(source, line_end_pos) - local line_start = find_line_start(source, line_end_pos) - local line = source:sub(line_start, line_end_pos) - if line:sub(1, 2) == "//" then - return line, line_start - 1 - end - return nil -end - --- Find the contiguous comment block immediately preceding `pos` in `source`. --- Returns the comment text (with the `/* */` or `//` markers preserved) or an empty string if no comment is adjacent. --- --- Used to copy signature comments from the source declaration (`MipsAtomComp_` / `MipsAtomComp_Proc_` / function decl) --- over to the generated `mac_X` macro, so LSP/IntelliSense displays the args doc. ---- --- @param source string --- @param pos integer --- @return string @@ -254,22 +155,59 @@ local function preceding_comment_block(source, pos) local scan_pos = pos local pieces = {} while true do - local non_ws = skip_ws_backward(source, scan_pos) - if non_ws == 0 then break end + -- Skip whitespace (space/tab/newline/CR) backward from `scan_pos`, + -- returning the position of the first non-whitespace char. + local non_ws = scan_pos - 1 + while non_ws > 0 do + local ch = source:sub(non_ws, non_ws) + if ch == " " or ch == "\t" or ch == "\n" or ch == "\r" then + non_ws = non_ws - 1 + else + break + end + end + if non_ws == 0 then break end local is_block_close = non_ws >= 2 and source:sub(non_ws - 1, non_ws) == "*/" local is_line_end = source:sub(non_ws, non_ws) == "\n" or source:sub(non_ws, non_ws) == "\r" if is_block_close then - local block_text, new_scan_pos = capture_block_comment(source, non_ws) - if not block_text then break end - table.insert(pieces, 1, block_text) - scan_pos = new_scan_pos + -- Find the opening `/*` for a block comment whose `*/` ends at `non_ws`. + -- Walk back from `non_ws` over `/*` candidates. + local prefix = source:sub(1, non_ws - 1) + local open_at = nil + for scan = #prefix - 1, 1, -1 do + if prefix:sub(scan, scan + 1) == "/*" then + open_at = scan + break + end + end + if not open_at then break end + -- Walk back from `open_at` over leading spaces + tabs to include the indentation before the `/*`. + local block_start = open_at + while block_start > 1 do + local ch = source:sub(block_start - 1, block_start - 1) + if ch == " " or ch == "\t" then + block_start = block_start - 1 + else + break + end + end + table.insert(pieces, 1, source:sub(block_start, non_ws)) + scan_pos = block_start elseif is_line_end then - local line_text, new_scan_pos = capture_line_comment(source, non_ws) - if not line_text then break end - table.insert(pieces, 1, line_text) - scan_pos = new_scan_pos + -- Walk back from `non_ws` to the start of the source line (the most recent `\n` or position 1). + local line_start = non_ws + while line_start > 1 and source:sub(line_start - 1, line_start - 1) ~= "\n" do + line_start = line_start - 1 + end + local line = source:sub(line_start, non_ws) + if line:sub(1, 2) == "//" then + table.insert(pieces, 1, line) + scan_pos = line_start - 1 + else + break + end else break end @@ -282,42 +220,6 @@ end -- Argument-name extraction -- ════════════════════════════════════════════════════════════════════════════ --- Walk `trimmed` backward from `pos` over trailing whitespace / asterisks / brackets, --- returning the position of the first non-trailer character (i.e. the end of the identifier). --- @param trimmed string --- @param pos integer --- @return integer -local function trim_trailer_back(trimmed, pos) - local back = pos - while back > 0 do - local ch = trimmed:sub(back, back) - if ch == " " or ch == "\t" or ch == "*" or ch == "]" or ch == "[" then - back = back - 1 - else - break - end - end - return back -end - --- Walk `trimmed` backward from `pos` over identifier chars (alnum + `_`), --- returning the position just before the identifier starts. --- @param trimmed string --- @param pos integer --- @return integer -local function trim_ident_back(trimmed, pos) - local back = pos - while back > 0 do - local ch = trimmed:sub(back, back) - if duffle.is_alnum(ch) or ch == "_" then - back = back - 1 - else - break - end - end - return back -end - --- Extract just the parameter NAMES from a function-args string (stripping type annotations). E.g., --- `"U4 off, U4 code, U1 r, U1 g, U1 b"` -> `{"off", "code", "r", "g", "b"}` --- `"U4 *ptr"` -> `{"ptr"}` @@ -331,9 +233,29 @@ local function extract_arg_names(args_str) for _, tok in ipairs(tokens) do local trimmed = duffle.trim(tok) if trimmed ~= "" then - local ident_end = trim_trailer_back(trimmed, #trimmed) - local ident_start = trim_ident_back(trimmed, ident_end) + 1 - local name = trimmed:sub(ident_start, ident_end) + -- Find the identifier at the end: walk back over trailers (whitespace + `*` + `[]`), + -- then walk back over the identifier chars (alnum + `_`). + -- Plex: inlined the 2 single-caller helpers (no 2-caller rule met). + local ident_end = #trimmed + while ident_end > 0 do + local ch = trimmed:sub(ident_end, ident_end) + if ch == " " or ch == "\t" or ch == "*" or ch == "]" or ch == "[" then + ident_end = ident_end - 1 + else + break + end + end + local ident_start = ident_end + while ident_start > 0 do + local ch = trimmed:sub(ident_start, ident_start) + if duffle.is_alnum(ch) or ch == "_" then + ident_start = ident_start - 1 + else + break + end + end + ident_start = ident_start + 1 + local name = trimmed:sub(ident_start, ident_end) if name ~= "" then names[#names + 1] = name end end end @@ -513,13 +435,6 @@ local function split_comment_lines(s) return out end ---- Split an atom body by top-level commas; drop empty tokens. ---- @param body string ---- @return string[] -local function tokens_from_body(body) - return duffle.tokenize_body_simple(body) -end - --- Determine the macro signature: function-args list (function form) or variadic-ignored (bare form). --- @param args_str string|nil --- @return string @@ -569,7 +484,8 @@ local function build_component_lines(c, counts) end end - local tokens = tokens_from_body(c.body) + local tokens = duffle.split_top_level_commas(c.body) + for i = 1, #tokens do tokens[i] = duffle.trim(tokens[i]) end local sig = signature_from_args(c.args) -- Direct lookup against the per-source precomputed `counts` table (built once by count_all_components). local n = counts[c.name] @@ -605,7 +521,7 @@ local function header_boilerplate(src) "// Component atoms (MipsAtomComp_(ac_*)) -> macro variants (mac_*)", "", -- Self-contained: define WORD_COUNT if not already defined. - -- We use the same definition here so the auto-generated entries below expand + -- We use the same definition here so the auto-generated entries below expand -- to compile-time constants whether the metadata file is included first or not. "#ifndef WORD_COUNT", "#define WORD_COUNT(name, count) enum { words_##name = (count) };", @@ -616,7 +532,7 @@ end -- Compute the output path for one source's `.macs.h` file. -- The pre-rework convention uses the *directory* basename --- (not the source file basename) — e.g. `code/duffle/lottes_tape.h` produces `code/duffle/gen/duffle.macs.h`. +-- (not the source file basename) e.g. `code/duffle/lottes_tape.h` produces `code/duffle/gen/duffle.macs.h`. -- This matches what the C codebase #includes. -- @param src SourceFile -- @return string -- the output directory diff --git a/scripts/passes/dwarf_injection.lua b/scripts/passes/dwarf_injection.lua index ef1c123..c4cc7b8 100644 --- a/scripts/passes/dwarf_injection.lua +++ b/scripts/passes/dwarf_injection.lua @@ -1,12 +1,12 @@ ---- passes/dwarf_injection.lua — Per-atom DWARF injection for tape-atom step-debug (F' + G'). +--- passes/dwarf_injection.lua — Per-atom DWARF injection for tape-atom step-debug. --- --- Reads the post-link ELF directly (lfs + io.open; walks the ELF32 section header table to find --- `.debug_info` + `.debug_abbrev` + `.debug_str` + `.debug_line` + `.debug_aranges` + `.debug_rnglists`), --- APPENDS synthetic DWARF line-program sequences for every `code_` atom, EXTENDS the `.debug_aranges` --- and main-CU range tables with the atom ranges, and APPENDS a new compilation unit to `.debug_info` with ---- per-atom `DW_TAG_subprogram` + per-wave-context-reg `DW_TAG_variable` entries (so `R_PrimCursor` etc. ---- appear as atom-scoped locals in VSCode's Variables pane). Writes the new section data to ---- `/.dwarf_*.bin` (7 blobs total) plus deterministic `/.gdbinit` skip commands. +--- per-atom `DW_TAG_subprogram` + per-wave-context-reg `DW_TAG_variable` entries +--- (so `R_PrimCursor` etc. appear as atom-scoped locals in VSCode's Variables pane). +--- Writes the new section data to `/.dwarf_*.bin` plus deterministic `/.gdbinit` skip commands. --- --- The `build_psyq.ps1` post-link hook then splices those `.bin` files into a copy of the ELF via: --- mipsel-none-elf-objcopy --update-section .debug_info= @@ -20,11 +20,11 @@ --- Splice step runs from PowerShell — no Lua subprocess; no cmd /c parsing issues. --- objcopy's --update-section works fine in PowerShell even though Lua's `os.execute`/`io.popen` would mangle the `=` on Windows.) --- ---- Result: VSCode's source gutter follows per-stepi inside atom bodies (F'), AND the Variables pane shows the wave-context regs as atom-scoped locals (G'). +--- Result: VSCode's source gutter follows per-stepi inside atom bodies, AND the Variables pane shows the wave-context regs as atom-scoped locals. --- Native VSCode UX (gutter arrow + highlighted line + Run to Cursor + conditional BPs by source line + per-atom locals). --- No VSCode plugin, no Python, no pyelftools — pure Lua + objcopy. --- ---- **Conventions:** tabs (1/level), EmmyLua annotations, no regex, Lua 5.3 compatible. +--- **Conventions:** tabs (1/level), EmmyLua annotations, Lua 5.3 compatible. -- ════════════════════════════════════════════════════════════════════════════ -- Bootstrap @@ -43,12 +43,17 @@ local elf_dwarf = require("elf_dwarf") -- word-counting helper shared with passes/atoms_source_map.lua. -- Used here to walk a component's body_tokens in lockstep with their word-count allocation -- when we propagate per-word body lines into each invocation's `body_lines` array. -local word_count_eval = require("word_count_eval") +local word_count_eval = require("word_count_eval") local count_token_words = word_count_eval.count_token_words local lfs = require("lfs") --- File-scope LEB128 aliases; the helpers are available at module-load time so no forward-reference table is needed. +-- File-scope aliases to elf_dwarf helpers; the canonical implementations live in scripts/elf_dwarf.lua. +-- (2-caller lift: these were duplicated file-locals; the canonical is in elf_dwarf.lua, used by parse_abbrev_table + read_form_value.) +local read_uleb128_at = elf_dwarf.read_uleb128_at +local read_sleb128_at = elf_dwarf.read_sleb128_at +local find_abbrev_table_end = elf_dwarf.find_abbrev_table_end +-- Note: uleb128 + sleb128 are encoders; read_uleb128_at + read_sleb128_at are decoders. Different functions. local uleb128 = elf_dwarf.uleb128 local sleb128 = elf_dwarf.sleb128 @@ -58,7 +63,7 @@ local sleb128 = elf_dwarf.sleb128 -- DWARF line-program opcodes + range-list entry encodings (per DWARF5 spec). -- All values lifted from `elf_dwarf.DWARF_LINE_OPS` + `elf_dwarf.DWARF5_RNGLISTS`. --- Local aliases preserve the F' code's readability +-- Local aliases preserve the code's readability -- (e.g. `DW_LNS_copy` reads better than `elf_dwarf.DWARF_LINE_OPS.DW_LNS_copy` in an emitter body). local DWARF_LINE_OPS = elf_dwarf.DWARF_LINE_OPS local DWARF5_RNGLISTS = elf_dwarf.DWARF5_RNGLISTS @@ -109,8 +114,17 @@ local DW_LLE_end_of_list = 0x00 local DW_LLE_start_length = 0x08 local DW_OP_reg0 = 0x50 -- base reg op; regN = 0x50 + N local DW_OP_breg0 = 0x70 -- base breg op; bregN = 0x70 + N (SLEB offset) +local DW_OP_piece = 0x93 local MIPS_LOAD_DELAY_BYTES = 0x08 -- 1 load word + 1 BD-slot word +-- DIE children-list terminator: each DIE that has children ends with a single 0 byte (DWARF5 §7.5.3). +-- This is NOT the same as DW_LLE_end_of_list despite sharing the value 0x00 — different spec sections. +local DIE_CHILDREN_TERMINATOR = 0x00 + +-- Field/piece byte sizes for the typed-view piece chains. +-- All Binds_* struct fields are U4 (sizeof(uint32_t) on MIPS32 = 4 bytes). +local U4_BYTE_SIZE = 4 + -- DWARF3/4/5 opcodes / attributes / forms (for the .debug_info synth). -- DW_TAG values are stable across DWARF3-5 per the standard's Table 7.1; -- gcc emits DW_TAG_structure_type=0x13 + DW_TAG_base_type=0x24 even in DWARF5-versioned CUs, so we match those exact byte values. @@ -171,7 +185,7 @@ local function resolve_provenance_file_index(path) local normalized = path:gsub("\\", "/") -- Take the last path component (the basename). local basename = normalized:match("([^/]+)$") or normalized - local idx = PROVENANCE_BASENAME_TO_FILE_INDEX[basename] + local idx = PROVENANCE_BASENAME_TO_FILE_INDEX[basename] if idx == nil then error(string.format( "[dwarf_injection] resolve_provenance_file_index: unknown provenance basename '%s' (from '%s'). " @@ -198,7 +212,7 @@ local DW_OP_piece = 0x93 -- followed by ULEB128 byte count (per DWA local DW_ATE_unsigned = 0x07 -- DWARF5 §7.8.1: DW_ATE_unsigned (used for U4 base type) local DW_LANG_C99 = 0x0C -- = 12 (C99); use as a safe "C-like" placeholder --- (DW_LANG_Mips_Assembler = 0x8001 was used in the F', but we want this CU to look like a C TU so VSCode's Variables pane treats it as code.) +-- (DW_LANG_Mips_Assembler = 0x8001 was used in the, but we want this CU to look like a C TU so VSCode's Variables pane treats it as code.) -- R_ → MIPS GPR lookups go through the merged register_alias_registry -- (collected by collect_per_source_registries from ctx.sources[*].scan.register_alias_registry). @@ -211,7 +225,6 @@ local DW_LANG_C99 = 0x0C -- = 12 (C99); use as a safe "C-like" plac -- -- Per rbind atom we emit 2 loclist entries (tape → GPR split) using the last field's transition as the boundary. -- This is a conservative approximation that always reads the correct GPR value once the first load has retired (load_pc + 8). --- A future task can refine to per-field transitions. -- -- `tape_alias` (default: "R_TapePtr") names the wave-runtime pointer register whose value is the tape address. -- The GPR integer comes from the merged registry; if the alias is absent, this function fails loud @@ -239,12 +252,16 @@ local function build_debug_loclists_section(atom_table, registries) for _, f in ipairs(fields) do local offset = f.offset or 0 local offset_sleb = elf_dwarf.sleb128(offset) - table.insert(tape_pieces, string.char(DW_OP_breg0 + tape_reg) .. offset_sleb .. string.char(DW_OP_piece) .. uleb128(4)) + -- (DW_OP_bregN, SLEB128(offset), DW_OP_piece, ULEB128(U4_BYTE_SIZE)) + -- 4 = U4_BYTE_SIZE: each piece is sizeof(uint32_t) on MIPS32. + table.insert(tape_pieces, string.char(DW_OP_breg0 + tape_reg) .. offset_sleb .. string.char(DW_OP_piece) .. uleb128(U4_BYTE_SIZE)) end local tape_expr = table.concat(tape_pieces) local gpr_pieces = {} for _, pair in ipairs(regs) do - table.insert(gpr_pieces, string.char(DW_OP_reg0 + pair.reg) .. string.char(DW_OP_piece) .. uleb128(4)) + -- (DW_OP_regN, DW_OP_piece, ULEB128(4)) — one piece per GPR-resident field. + -- The 4 = U4_BYTE_SIZE: each piece is sizeof(uint32_t) on MIPS32. + table.insert(gpr_pieces, string.char(DW_OP_reg0 + pair.reg) .. string.char(DW_OP_piece) .. uleb128(U4_BYTE_SIZE)) end local gpr_expr = table.concat(gpr_pieces) parts[#parts + 1] = string.char(DW_LLE_start_length) @@ -258,13 +275,17 @@ local function build_debug_loclists_section(atom_table, registries) parts[#parts + 1] = string.char(DW_LLE_end_of_list) end end + -- Loclist unit header (DWARF5 §7.7.2): + -- unit_length(4) + version(2) + address_size(1) + segment_size(1) + offset_entry_count(4) = 12 bytes header. + -- version = 5 (DWARF5); address_size = 4 (MIPS32); segment_size = 0; offset_entry_count = 0 (we use DW_LLE_start_length, not offsets). + local LOCLIST_HEADER_SIZE = 12 local body = table.concat(parts) - local unit_length = 2 + 1 + 1 + 4 + #body + local unit_length = LOCLIST_HEADER_SIZE - 4 + #body -- -4 because unit_length excludes itself local header = elf_dwarf.write_u32_le(unit_length) - .. elf_dwarf.write_u16_le(5) - .. string.char(4) - .. string.char(0) - .. elf_dwarf.write_u32_le(0) + .. elf_dwarf.write_u16_le(5) -- DWARF5 + .. string.char(U4_BYTE_SIZE) -- address_size + .. string.char(0) -- segment_size + .. elf_dwarf.write_u32_le(0) -- offset_entry_count return header .. body end @@ -273,7 +294,7 @@ end -- @return table -- {[atom_name] = offset_in_section} local function compute_loclists_offsets(atom_table) local offsets = {} - local cursor = 4 + 2 + 1 + 1 + 4 + local cursor = 4 + 2 + 1 + 1 + 4 for _, atom in ipairs(atom_table) do if atom.rbind then offsets[atom.name] = cursor @@ -290,7 +311,7 @@ local function compute_loclists_offsets(atom_table) end -- Default name for the synthetic CU (so VSCode lists it as a known source). -local DEFAULT_CU_NAME = "tape_atom_locals" +local DEFAULT_CU_NAME = "tape_atom_locals" local DEFAULT_CU_COMP_DIR = "." -- Path templates for the .bin outputs are now in SECTION_WRITERS (see below). @@ -316,16 +337,8 @@ local function normalize_debug_path(path) return duffle.to_absolute_path(path):gsub("\\", "/") end ---- Build a case-insensitive Windows path + exact component-name key. ---- @param path string ---- @param component_name string ---- @return string -local function component_skip_key(path, component_name) - return normalize_debug_path(path):lower() .. "\0" .. component_name -end - --- Consume the per-source scanner associations without naming any atom or component in production. ---- Whole atoms remain symbol-keyed; components are file-qualified internally so a source marker associates +--- Whole atoms remain symbol-keyed; components are file-qualified internally so a source marker associates --- with its exact component definition even though GDB 12 requires function-only skip entries for the resulting synthetic inline frame. --- @param ctx DwarfInjectionCtx --- @return table -- {atoms = {[symbol] = association}, components = {[file|name] = association}} @@ -343,7 +356,9 @@ local function collect_skip_over(ctx) end for component_name, association in pairs(scan_skip.components or {}) do local source_path = normalize_debug_path(src.path) - skip_over.components[component_skip_key(source_path, component_name)] = { + -- component_skip_key (the lower() .. "\0" .. component_name part) inlined at this single call site + -- (no other callers remain after pass 17). + skip_over.components[source_path:lower() .. "\0" .. component_name] = { name = component_name, source_path = source_path, association = association, @@ -361,8 +376,7 @@ end --- Aliases without `atom_reg` adjacent are absent; the absence is treated as "not debug-visible" (see build_inserted_children for the precedence chain). --- --- When two sources register the same key, the last-writer wins (later sources override earlier). ---- Today only one source declares wave-context enums, so collisions are absent; ---- future multi-source declarations would need a name-resolution rule (left as a follow-up). +--- Today only one source declares wave-context enums, so collisions are absent. --- @param ctx DwarfInjectionCtx --- @return table -- { --- register_alias_registry = {[R_Name] = AliasEntry}, @@ -660,7 +674,7 @@ end --- (the MipsAtomComp_/MipsAtomComp_Proc_ declaration). --- Atom labels (`atom_label(...)`) emit 0 `.word`s and are ignored (matching `passes/atoms_source_map.lua :: is_marker_token` + `count_marker_rest`). --- @param ctx DwarfInjectionCtx ---- @param skip_over table -- collect_skip_over(ctx) result +--- @param skip_over table -- {atoms = {[symbol] = association}, components = {[file|name] = association}} --- @return table[] -- list of {name, addr, size_bytes, words, entries, invocations, skip_over?} local function build_atom_table(ctx, skip_over) local basename = ctx.basename or DEFAULT_BASENAME @@ -688,7 +702,7 @@ local function build_atom_table(ctx, skip_over) -- Also read *.atoms.provenance.txt to extract per-component invocations. -- Files are merged by atom name; entries carry the original {pos, call_file, call_line, comp_name, comp_file, comp_line} shape. - local prov_files = duffle.list_dir(ctx.out_root, "%.atoms.provenance%.txt$") + local prov_files = duffle.list_dir(ctx.out_root, "%.atoms.provenance%.txt$") local prov_merged = {} for _, prov_path in ipairs(prov_files) do local prov = elf_dwarf.parse_provenance_file(prov_path, 1) @@ -793,7 +807,9 @@ local function build_atom_table(ctx, skip_over) comp_line = w.comp_line, start_pos = w.pos, end_pos = w.pos, - skip_over = skip_over.components[component_skip_key(w.comp_file, w.comp_name)] ~= nil, + -- component_skip_key: case-insensitive Windows path + "\0" separator + exact component-name. + -- (Inlined from `component_skip_key`; the lookup is in skip_over.components keyed by the result.) + skip_over = skip_over.components[normalize_debug_path(w.comp_file):lower() .. "\0" .. w.comp_name] ~= nil, } -- Capture the per-word body lines for THIS invocation, indexed by 1-based word position within the invocation. -- body_lines[1] is the line of the first body word (= the line of the first macro-body token, NOT the comp def line). @@ -850,7 +866,6 @@ end -- parse_rbind_atoms consumes `scan.binds[i].fields` directly, which is populated by the scan-source pass with the typed-field record ({name, type_name, pointer_depth, offset, byte_size}). --- Find every `load_word(R_, R_TapePtr, O_(Binds_X, FieldName))` call in the atom body and return ordered (reg_index, field_name) pairs. ---- --- Source pattern (single-line, comma-separated at top level): --- load_word(R_PrimCursor, R_TapePtr, O_(Binds_CubeTri,PrimCursor)), --- load_word(R_FaceCursor, R_TapePtr, O_(Binds_CubeTri,FaceCursor)), @@ -859,49 +874,36 @@ end --- The GPR for each `R_` is looked up in the merged register_alias_registry; aliases absent from the registry --- (no `atom_reg` opt-in) are silently skipped — the resulting rbind record will be incomplete and the atom will fail to bind a usable piece chain. --- This is intentional: silently falling back to a hardcoded GPR would mask the missing opt-in. ---- @param body string -- the atom body text (between { ... }) ---- @param binds_name string -- expected Binds_X name (skip pairs with mismatching binds) ---- @param registries table -- merged registries from collect_per_source_registries ---- @return table[] -- list of {reg = , field = } -local function parse_body_load_pairs(body, binds_name, registries) - local pairs = {} - local pos = 1 - local body_len = #body +--- +--- Pre-tokenized: `body_tokens` is the scan-source pass's pre-split list of top-level +--- statements (each entry is a single `load_word(...)` call or other statement). +--- Plex: prepass move — use the pre-computed data instead of re-walking the body text. +--- @param body_tokens table[] -- the atom's pre-tokenized body statements (from atom.body_tokens) +--- @param binds_name string -- expected Binds_X name (skip pairs with mismatching binds) +--- @param registries table -- merged registries from collect_per_source_registries +--- @return table[] -- list of {reg = , field = } +local function parse_body_load_pairs(body_tokens, binds_name, registries) + local pairs = {} local reg_index_by_name = (registries and registries.register_alias_registry) or {} - while pos <= body_len do - -- Find the next "load_word" identifier. - pos = duffle.skip_ws_and_cmt(body, pos) - if pos > body_len then break end - local ident, ident_end = duffle.read_ident(body, pos) - if not ident or ident ~= "load_word" then - -- Either no ident here OR a non-load_word ident (e.g. add_ui_self). - -- Either way, advance by at least 1 byte to avoid infinite loops. - -- ident_end is `pos` when no ident matches (read_ident returns nil, pos). - pos = (ident_end and ident_end > pos) and ident_end or (pos + 1) - else - -- Read the argument list. - local p_open = duffle.skip_ws_and_cmt(body, ident_end) - if body:sub(p_open, p_open) ~= "(" then - pos = p_open + 1 - else - local inner, p_after = duffle.read_parens(body, p_open) - -- Split top-level commas. - local args = duffle.split_top_level_commas(inner) - -- Expected shape: (R_, R_TapePtr, O_(Binds_, FieldName)) - if #args >= 3 then - local reg_name = duffle.trim(args[1]) - local third_arg = duffle.trim(args[3]) - -- Match O_(Binds_, FieldName) - local b, f = third_arg:match("^O_%((Binds_[%w_]+)%s*,%s*(.-)%s*%)$") - local alias_entry = reg_index_by_name[reg_name] - if b and b == binds_name and alias_entry and alias_entry.code then - pairs[#pairs + 1] = { - reg = alias_entry.code, - field = duffle.trim(f), - } - end + for _, t in ipairs(body_tokens or {}) do + local tok = duffle.trim(t.tok or "") + -- Match "load_word(...)" — the entire call is one body_tokens entry. + local inner = tok:match("^load_word%s*%((.*)%)$") + if inner then + local args = duffle.split_top_level_commas(inner) + -- Expected shape: (R_, R_TapePtr, O_(Binds_, FieldName)) + if #args >= 3 then + local reg_name = duffle.trim(args[1]) + local third_arg = duffle.trim(args[3]) + -- Match O_(Binds_, FieldName) + local b, f = third_arg:match("^O_%((Binds_[%w_]+)%s*,%s*(.-)%s*%)$") + local alias_entry = reg_index_by_name[reg_name] + if b and b == binds_name and alias_entry and alias_entry.code then + pairs[#pairs + 1] = { + reg = alias_entry.code, + field = duffle.trim(f), + } end - pos = p_after end end end @@ -955,13 +957,13 @@ local function parse_rbind_atoms(ctx, atom_table, registries) end end - -- Walk every atom_info; if `binds` is set, find the atom body + parse load_word pairs. - local body_by_atom = {} + -- Walk every atom_info; if `binds` is set, find the atom body_tokens + parse load_word pairs. + local body_tokens_by_atom = {} for _, src in ipairs(ctx.sources or {}) do local scan = src.scan if scan then for _, atom in ipairs(scan.atoms or {}) do - body_by_atom[atom.name] = atom.body + body_tokens_by_atom[atom.name] = atom.body_tokens end end end @@ -978,10 +980,10 @@ local function parse_rbind_atoms(ctx, atom_table, registries) for atom_name, ai in pairs(ai_by_atom) do if ai.binds then - local struct = rbind_structs[ai.binds] - local body = body_by_atom[atom_name] - if struct and body then - local pairs = parse_body_load_pairs(body, ai.binds, registries) + local struct = rbind_structs[ai.binds] + local body_toks = body_tokens_by_atom[atom_name] + if struct and body_toks then + local pairs = parse_body_load_pairs(body_toks, ai.binds, registries) if #pairs > 0 then rbind_atoms[atom_name] = { binds = ai.binds, @@ -1182,19 +1184,12 @@ local function build_dwarf_rnglists_section(existing, atom_table) end -- ════════════════════════════════════════════════════════════════════════════ --- G' helpers: per-atom .debug_info synthesis (DW_TAG_subprogram + DW_TAG_variable) +-- Helpers: per-atom .debug_info synthesis (DW_TAG_subprogram + DW_TAG_variable) -- ════════════════════════════════════════════════════════════════════════════ --- Build the location bytes (DW_FORM_exprloc) for a register-resident value. --- DW_OP_reg0..reg31 occupy opcodes 0x50..0x6f. DW_OP_reg15 is 0x5f; --- 0x90 is DW_OP_regx, not the base of the compact register opcode range. ---- @param reg_n integer -- 0..31 (MIPS register number) ---- @return string -- the exprloc byte sequence (length-prefixed) -local function reg_exprloc(reg_n) - local op = string.char(DW_OP_reg0 + reg_n) - return uleb128(#op) .. op -end - --- Build the piece-chain location bytes (DW_FORM_exprloc) for an rbind atom's bind_args. --- --- The location expression is a sequence of (DW_OP_regN, DW_OP_piece, ULEB128(size)) tuples, one per field. @@ -1262,101 +1257,6 @@ end -- Pure-Lua 5.3 LEB decoders (no `bit` library; `2^shift` arithmetic matches elf_dwarf.lua's -- "math.floor(/16) instead of bit.rshift for LuaJIT 2.1 compat" comment at line 352). ---- Read a ULEB128 value starting at 0-based byte offset `pos` in `buf`. ---- Returns (value, next_pos) where `next_pos` is the 0-based offset of the byte AFTER the last byte consumed. ---- Returns (nil, pos) on truncation. ---- @param buf string ---- @param pos integer -- 0-based ---- @return integer|nil, integer -local function read_uleb128_at(buf, pos) - local value = 0 - local shift = 0 - local buf_len = #buf - while pos < buf_len do - local byte = buf:byte(pos + 1) - value = value + (byte % 0x80) * (2 ^ shift) -- extract low 7 bits, shift left - shift = shift + 7 - pos = pos + 1 - if byte < 0x80 then return value, pos end -- continuation bit clear → final byte - end - return nil, pos -end - ---- Read a SLEB128 value starting at 0-based byte offset `pos` in `buf`. ---- Returns (value, next_pos). Returns (nil, pos) on truncation. ---- @param buf string ---- @param pos integer -- 0-based ---- @return integer|nil, integer -local function read_sleb128_at(buf, pos) - local value = 0 - local shift = 0 - local buf_len = #buf - while pos < buf_len do - local byte = buf:byte(pos + 1) - value = value + (byte % 0x80) * (2 ^ shift) - shift = shift + 7 - pos = pos + 1 - if byte < 0x80 then - -- Sign-extend if the sign bit (bit 6) of the last byte is set. - if byte >= 0x40 then - value = value - (2 ^ shift) - end - return value, pos - end - end - return nil, pos -end - ---- Walk a .debug_abbrev table starting at 0-based offset `table_start` and return the 0-based offset of the table-terminator byte ---- (a single 0 byte marking the end of declarations in this table per DWARF5 §7.5.3). ---- ---- The walker consumes each declaration's abbrev code (ULEB) + tag (ULEB) + has_children byte + (attr, form) ---- attr-spec pairs (each pair = ULEB + ULEB, with an extra SLEB constant for DW_FORM_implicit_const per DWARF5 §7.5.6). ---- The declaration's attr-spec list ends at the (0, 0) pair. ---- ---- Returns nil on truncated or malformed input. ---- @param existing string -- the .debug_abbrev section bytes ---- @param table_start integer -- 0-based offset of the table's first byte ---- @return integer|nil -- 0-based offset of the table-terminator byte -local function find_abbrev_table_end(existing, table_start) - local pos = table_start - local buf_len = #existing - -- Empty table = terminator byte at table_start. - if pos >= buf_len or existing:byte(pos + 1) == 0 then return pos end - while pos < buf_len do - -- Each declaration: ULEB code, ULEB tag, 1 byte has_children. - local _code, code_end = read_uleb128_at(existing, pos) - if not _code then return nil end - pos = code_end - local _tag, tag_end = read_uleb128_at(existing, pos) - if not _tag then return nil end - pos = tag_end - -- has_children: 1 byte (DW_CHILDREN_yes=1 / DW_CHILDREN_no=0). - if pos >= buf_len then return nil end - pos = pos + 1 - -- attr/form loop until (0, 0) terminator pair. - while pos < buf_len do - local attr, attr_end = read_uleb128_at(existing, pos) - if not attr then return nil end - pos = attr_end - local form, form_end = read_uleb128_at(existing, pos) - if not form then return nil end - pos = form_end - if attr == 0 and form == 0 then break end - -- DW_FORM_implicit_const: declaration carries a SLEB constant. - if form == DW_FORM_implicit_const then - local _const, const_end = read_sleb128_at(existing, pos) - if not _const then return nil end - pos = const_end - end - end - -- Next declaration starts here; if next byte is 0, the table is done. - if pos >= buf_len then return nil end - if existing:byte(pos + 1) == 0 then return pos end - end - return nil -end - -- DWARF5 compile-unit header constants. local DW_VERSION_5 = 5 local DW_UT_compile = 0x01 @@ -1549,16 +1449,9 @@ local function build_new_abbrev() .. abbrev_struct_type .. abbrev_member .. abbrev_bind_var .. abbrev_base_type .. abbrev_abstract_subprogram .. abbrev_inlined_subroutine .. abbrev_bind_var_loclists - .. string.char(0x00) + .. string.char(DW_LLE_end_of_list) -- section terminator (the loclist-unit end marker per DWARF5 §7.7.3) end ---- Build the new strings to append to .debug_str. Each string is null-terminated. --- The CU name + comp_dir + one entry per RR_ debug-visible alias all go into one new blob. --- Register names are sourced from the merged registry filtered to MIPS GPR 0..31 --- (the same filter that build_inserted_children applies for the RR_ locals, so .debug_str entries stay in sync with .debug_info). --- @param atom_table table[] -- list of {name, addr, size_bytes, ...} --- @param registries table -- merged registries from collect_per_source_registries --- @return string, table -- (new_strings_blob, map_of_name_to_offset_in_new_blob) --- Strip a leading "R_" prefix from an enum alias name. --- The .debug_str/.debug_info consumers display the local without the C-enum prefix (RR_PrimCursor, not RR_R_PrimCursor) @@ -1572,6 +1465,13 @@ local function strip_r_prefix(r_name) return r_name end +--- Build the new strings to append to .debug_str. Each string is null-terminated. +-- The CU name + comp_dir + one entry per RR_ debug-visible alias all go into one new blob. +-- Register names are sourced from the merged registry filtered to MIPS GPR 0..31 +-- (the same filter that build_inserted_children applies for the RR_ locals, so .debug_str entries stay in sync with .debug_info). +-- @param atom_table table[] -- list of {name, addr, size_bytes, ...} +-- @param registries table -- merged registries from collect_per_source_registries +-- @return string, table -- (new_strings_blob, map_of_name_to_offset_in_new_blob) local function build_new_strings(atom_table, registries) registries = registries or {} -- The CU name + comp_dir are the first two strings (offsets 0 and N1). @@ -1601,13 +1501,20 @@ local function build_new_strings(atom_table, registries) -- Register names (one per unique debug-visible R_Name alias from the merged registry, -- filtered to MIPS GPR 0..31 — the same filter that build_inserted_children applies -- for the RR_ locals, so .debug_str entries stay in sync with .debug_info). + -- Plex: Lua's pairs() is non-deterministic; sort the alias names first so the emitted + -- .debug_str bytes are byte-identical across runs. + local sorted_alias_names = {} for r_name, alias in pairs(registries.register_alias_registry or {}) do if alias.code and alias.code >= 0 and alias.code <= 31 then - local rr_name = "RR_" .. strip_r_prefix(r_name) - if not map[rr_name] then - map[rr_name] = #table.concat(strings) - strings[#strings + 1] = rr_name .. "\0" - end + sorted_alias_names[#sorted_alias_names + 1] = r_name + end + end + table.sort(sorted_alias_names) + for _, r_name in ipairs(sorted_alias_names) do + local rr_name = "RR_" .. strip_r_prefix(r_name) + if not map[rr_name] then + map[rr_name] = #table.concat(strings) + strings[#strings + 1] = rr_name .. "\0" end end @@ -1672,12 +1579,19 @@ local function build_inserted_children(main_cu_offset, main_cu_end_excl, atom_ta -- by_alias: the merged register_alias_registry filtered to aliases whose `code` is a valid MIPS GPR 0..31. -- Aliases absent from the merged registry are not debug-visible and are skipped entirely (no fallback GPR). + -- by_alias_order: sorted list of by_alias keys, for deterministic iteration order. + -- (Lua's pairs() order is implementation-defined and may vary between runs; + -- without sorting, the per-atom variable emission order would be non-deterministic + -- and the .debug_info bytes would differ across builds.) local by_alias = {} for r_name, alias in pairs(registries.register_alias_registry or {}) do if alias.code and alias.code >= 0 and alias.code <= 31 then by_alias[r_name] = alias end end + local by_alias_order = {} + for r_name in pairs(by_alias) do by_alias_order[#by_alias_order + 1] = r_name end + table.sort(by_alias_order) -- Build a name->atom lookup for fast rbind_atom resolution during the per-atom phase/ctx propagation. -- Cheap (O(atom_table)) and built once. @@ -1694,18 +1608,31 @@ local function build_inserted_children(main_cu_offset, main_cu_end_excl, atom_ta -- The first emitted byte lives at section offset (main_cu_end_excl - 1). local insertion_start = main_cu_end_excl - 1 - local bytes = {} - local next_offset = insertion_start -- 0-based section offset of the NEXT byte to emit - + -- Closure state: bytes + next_offset + the typed-view/structure/abstract offset caches. + -- All step-emitters mutate this state in place. Plex: Fleury expose structure over interface. + local S = { + bytes = {}, + next_offset = insertion_start, -- 0-based section offset of the NEXT byte to emit + type_chain_offsets = {}, -- {[type_name.."|"..depth] = section_offset for the outermost pointer_type} + struct_section_offsets = {}, -- {[binds_name] = section_offset for the structure_type} + abstract_offsets = {}, -- {[comp_name] = section_offset for the abstract subprogram} + member_base_type_offsets = {}, -- {[tn.."|"..byte_size.."|"..encoding] = section_offset} + base_type_section_offset = nil, -- set by emit_unsigned_int_base_type + } + -- Plex helpers. local function emit(s) - bytes[#bytes + 1] = s - next_offset = next_offset + #s + S.bytes[#S.bytes + 1] = s + S.next_offset = S.next_offset + #s end local function ref4_of(section_offset) return section_offset - main_cu_offset end + -- 1) Emit the base_type DIE first (member ref4s reference it). + -- (Use a local alias for S.next_offset; the rest of the function body still + -- uses the bare name `next_offset` from the previous unrefactored version). + local next_offset = S.next_offset local base_type_section_offset = next_offset emit(uleb128(ABBREV_BASE_TYPE)) emit("unsigned int\0") -- DW_FORM_string (DW_AT_name) @@ -1829,13 +1756,12 @@ local function build_inserted_children(main_cu_offset, main_cu_end_excl, atom_ta local depth = used_typed_views[tn] local type_info = STRUCT_MEMBER_TABLE[tn] if not type_info then - -- Unknown typed view — fall back to a generic base_type to keep the wire valid + -- Unknown typed view — fall back to a generic 4-byte unsigned base_type to keep the wire valid -- (gdb will render as the typename but `print *ptr` will only see the first 4 bytes). - -- This is the pre-Track-A behavior; preserved here for forward compatibility with future typed views not in our table. local innermost_offset = next_offset emit(uleb128(ABBREV_BASE_TYPE)) emit(tn .. "\0") - emit(string.char(4)) -- byte_size = 4 (legacy stand-in) + emit(string.char(U4_BYTE_SIZE)) -- byte_size = 4 (fallback for unknown types) emit(string.char(DW_ATE_unsigned)) -- encoding = unsigned local outermost_offset = next_offset emit(uleb128(9)) -- DW_TAG_pointer_type @@ -1845,8 +1771,8 @@ local function build_inserted_children(main_cu_offset, main_cu_end_excl, atom_ta -- Emit a proper structure_type DIE for this typed view. local struct_offset = next_offset emit(uleb128(ABBREV_STRUCT_TYPE)) - emit(tn .. "\0") -- DW_AT_name (struct_type has children, no name in abbrev 103 [DW_FORM_string only]) - emit(uleb128(type_info.byte_size)) -- DW_AT_byte_size (DW_FORM_udata) + emit(tn .. "\0") -- DW_AT_name (struct_type has children, no name in abbrev 103 [DW_FORM_string only]) + emit(uleb128(type_info.byte_size)) -- DW_AT_byte_size (DW_FORM_udata) -- For each member, emit ABBREV_MEMBER (name + data_member_location + type ref4). for _, m in ipairs(type_info.members) do emit(uleb128(ABBREV_MEMBER)) @@ -1868,7 +1794,7 @@ local function build_inserted_children(main_cu_offset, main_cu_end_excl, atom_ta local member_base_off = ensure_member_base_type(member_type_name, m.byte_size, member_type_encoding) emit(elf_dwarf.write_u32_le(ref4_of(member_base_off))) -- DW_AT_type ref4 → base_type end - emit(string.char(0x00)) -- end of structure_type's children + emit(string.char(DIE_CHILDREN_TERMINATOR)) -- end of structure_type's children (DWARF5 §7.5.3) -- Emit a single DW_TAG_pointer_type (abbrev 9) pointing at the structure_type. -- For depth > 1, we'd chain pointer_type → pointer_type → ... → structure_type; not exercised. if depth == 1 then @@ -1947,7 +1873,7 @@ local function build_inserted_children(main_cu_offset, main_cu_end_excl, atom_ta emit(elf_dwarf.write_u32_le(ref4_of(field_type_offset))) -- DW_FORM_ref4 → type end - emit(string.char(0x00)) -- end of structure_type's children + emit(string.char(DIE_CHILDREN_TERMINATOR)) -- end of structure_type's children (DWARF5 §7.5.3) end -- 3) Emit one abstract DW_TAG_subprogram per unique mac_X component. @@ -2058,51 +1984,72 @@ local function build_inserted_children(main_cu_offset, main_cu_end_excl, atom_ta end end - for r_name, alias in pairs(by_alias) do - local rr_name = "RR_" .. strip_r_prefix(r_name) - local alias_code = alias.code - emit(uleb128(ABBREV_VARIABLE)) - emit(rr_name .. "\0") -- DW_FORM_string (DW_AT_name) - emit(reg_exprloc(alias_code)) -- DW_FORM_exprloc (DW_OP_regN from registry code) - -- Resolve the type for this register via the 6-step precedence chain. - local type_offset = type_chain_offsets["void|1"] -- step (f) fallback: void* (renders as hex) - - -- step (a): per-atom callsite atom_type(R_X, ) - local override = atom_view and atom_view.reg_type_overrides and atom_view.reg_type_overrides[r_name] - if override and override.pointer_depth and override.pointer_depth > 0 then - local chain_offset = type_chain_offsets[override.type_name .. "|" .. override.pointer_depth] - if chain_offset then type_offset = chain_offset end - else - -- step (b): atom_ctx() propagation + -- 5-step precedence chain. Plex: data table; the dispatch loop runs the first step that yields a non-nil offset. + -- Each step returns the type's section offset or nil if it missed. + -- Adding a step = 1 row in the table + 1 function. The 5-level nested if/else is gone. + -- Per-atom precomputed state is captured in upvalues: atom_view, reg_to_field_ctx, atom_view_ctx_fields, + -- reg_to_field_phase, atom_view_phase_fields, field_type_by_name, reg_to_field, alias, type_chain_offsets. + local PRECEDENCE_STEPS = { + -- (a) per-atom callsite atom_type(R_X, ): most specific; user explicit override for THIS atom only. + function(r_name, alias_code) + local override = atom_view and atom_view.reg_type_overrides and atom_view.reg_type_overrides[r_name] + if override and override.pointer_depth and override.pointer_depth > 0 then + return type_chain_offsets[override.type_name .. "|" .. override.pointer_depth] + end + end, + -- (b) atom_ctx() propagation. + function(r_name, alias_code) local ctx_field_name = reg_to_field_ctx and reg_to_field_ctx[alias_code] local ctx_f = ctx_field_name and atom_view_ctx_fields and atom_view_ctx_fields[ctx_field_name] if ctx_f and ctx_f.pointer_depth and ctx_f.pointer_depth > 0 then - local chain_offset = type_chain_offsets[ctx_f.type_name .. "|" .. ctx_f.pointer_depth] - if chain_offset then type_offset = chain_offset end - else - -- step (c): self-bind (this atom is the rbind atom) - local field_name = reg_to_field[alias_code] - local f = field_name and field_type_by_name[field_name] - if f and f.pointer_depth and f.pointer_depth > 0 then - local chain_offset = type_chain_offsets[f.type_name .. "|" .. f.pointer_depth] - if chain_offset then type_offset = chain_offset end - else - -- step (d): atom_phase(