From 9ffd6592bc37fa2defb15ef7c0bce8bf38d4249c Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sat, 25 Jul 2026 04:09:48 -0400 Subject: [PATCH] Better static analysis for C0 <-> C2 data race hazards. --- code/duffle/lottes_tape.h | 1 + code/gte_hello/hello_gte.c | 4 +- code/gte_hello/hello_gte_tape.c | 2 +- scripts/build_psyq.ps1 | 120 +- scripts/duffle.lua | 1574 ++++++++++++++++++++++++--- scripts/passes/annotation.lua | 134 ++- scripts/passes/atoms_source_map.lua | 415 +++---- scripts/passes/components.lua | 216 ++-- scripts/passes/dwarf_injection.lua | 401 +++---- scripts/passes/emission_model.lua | 194 ++++ scripts/passes/offsets.lua | 280 ++--- scripts/passes/report.lua | 55 +- scripts/passes/scan_source.lua | 597 +++++++--- scripts/passes/static_analysis.lua | 1535 +++++++++++++++++++------- scripts/passes/word_count_eval.lua | 134 +-- scripts/ps1_meta.lua | 405 ++++--- scripts/update_deps.ps1 | 17 +- 17 files changed, 4264 insertions(+), 1820 deletions(-) create mode 100644 scripts/passes/emission_model.lua diff --git a/code/duffle/lottes_tape.h b/code/duffle/lottes_tape.h index 7cf69e8..e4757c0 100644 --- a/code/duffle/lottes_tape.h +++ b/code/duffle/lottes_tape.h @@ -104,6 +104,7 @@ FI_ Slice_MipsCode tb_slice(TapeBuilder tb) { return (Sl * ---------------------------------------------------------------------------*/ // The 'Yield' sequence for Tape Atoms (mac_yield). +atom_dbg_skip_over() MipsAtomComp_(ac_yield) { load_word(R_AtomJmp, R_TapePtr, 0), add_ui_self( R_TapePtr, S_(MipsCode)), diff --git a/code/gte_hello/hello_gte.c b/code/gte_hello/hello_gte.c index 4a2e1b1..b43a64b 100644 --- a/code/gte_hello/hello_gte.c +++ b/code/gte_hello/hello_gte.c @@ -1,6 +1,6 @@ -#include "stdio.h" +#include #include -#include "assert.h" +#include // #include "libgpu.h" // #include "libetc.h" // #include "libgte.h" diff --git a/code/gte_hello/hello_gte_tape.c b/code/gte_hello/hello_gte_tape.c index 319e40e..0814de3 100644 --- a/code/gte_hello/hello_gte_tape.c +++ b/code/gte_hello/hello_gte_tape.c @@ -103,8 +103,8 @@ MipsAtom_(rbind_floor_f3_face) atom_info(atom_bind(Binds_FloorTri), atom_phase(f mac_yield() }; -internal // atom_dbg_skip_over() +internal MipsAtom_(floor_f3_face) atom_info(atom_phase(floor_f3) , atom_reads( R_PrimCursor, R_FaceCursor, R_VertBase, R_OtBase) , atom_writes(R_PrimCursor, R_FaceCursr) diff --git a/scripts/build_psyq.ps1 b/scripts/build_psyq.ps1 index 15e557c..352b157 100644 --- a/scripts/build_psyq.ps1 +++ b/scripts/build_psyq.ps1 @@ -81,19 +81,6 @@ $path_psyq = join-path $path_toolchain 'psyq-4_7' $path_psyq_iwyu = join-path $path_toolchain 'psyq_iwyu' $path_psyq_imyu_inc = join-path $path_psyq_iwyu 'include' -function Get-SourceFiles { param([Parameter(Mandatory=$true)] [string[]]$paths, [Parameter(Mandatory=$true)] [string[]]$extensions) - $files = @() - foreach ($p in $paths) { - if (-not (test-path $p)) { continue } - foreach ($ext in $extensions) { - Get-ChildItem -Path $p -File -Recurse -Filter "*$ext" -ErrorAction SilentlyContinue | ForEach-Object { - $files += $_.FullName - } - } - } - return ($files | Sort-Object -Unique) -} - function assemble-unit { param( [string] $unit, [string] $link_module, @@ -243,6 +230,52 @@ function make-binary { param([string]$elf, [string]$exe) if ($LASTEXITCODE -ne 0) { Write-Error "Objcopy failed. Aborting."; exit 1 } } +function ps1-meta { param( + [string]$unity_root, + [string[]]$sources, + [Parameter(Mandatory=$true)][string]$metadata, + [string]$out_root = (join-path $path_build 'gen'), + [string[]]$passes = @('--pre-link'), + [string[]]$extra_args = @() + ) + # `--unity-root` and `--source` are + # mutually exclusive. Exactly one of `$unity_root` / `$sources` must + # be supplied; the other must be absent. + if ($null -ne $unity_root -and $unity_root -ne '') + { + if ($null -ne $sources -and $sources.Count -gt 0) { + write-error 'ps1-meta: -unity_root and -sources are mutually exclusive' + exit 2 + } + } + elseif ($null -eq $sources -or $sources.Count -eq 0) { + write-error 'ps1-meta: either -unity_root or -sources is required' + exit 2 + } + + $script = join-path $path_scripts 'ps1_meta.lua' + $input_summary = if ($null -ne $unity_root -and $unity_root -ne '') { + "unity=$unity_root" + } + else { + "$($sources.Count) source(s)" + } + write-host "ps1-meta $input_summary, passes=$($passes -join ',')" ` -ForegroundColor Magenta + + $arg_list = @($passes) + @('--metadata', $metadata) + @('--out-root', $out_root) + @($extra_args) + if ($null -ne $unity_root -and $unity_root -ne '') { + $arg_list += @('--unity-root', $unity_root) + } + else { + foreach ($s in $sources) { $arg_list += @('--source', $s) } + } + & luajit $script @arg_list + if ($LASTEXITCODE -ne 0) { + write-error "ps1-meta failed (exit $LASTEXITCODE). Aborting." + exit $LASTEXITCODE + } +} + function build-hello_psyqo { $includes += @() @@ -317,34 +350,16 @@ function build-graphis_hello { } # build-graphis_hello -function ps1-meta { param( - [Parameter(Mandatory=$true)][string[]]$sources, - [Parameter(Mandatory=$true)][string]$metadata, - [string]$out_root = (join-path $path_build 'gen'), - [string[]]$passes = @('--pre-link'), - [string[]]$extra_args = @() - ) - $script = join-path $path_scripts 'ps1_meta.lua' - write-host "ps1-meta $($sources.Count) source(s), passes=$($passes -join ',')" ` -ForegroundColor Magenta - $arg_list = @($passes) + @('--metadata', $metadata) + @('--out-root', $out_root) + @($extra_args) - foreach ($s in $sources) { $arg_list += @('--source', $s) } - & luajit $script @arg_list - if ($LASTEXITCODE -ne 0) { - write-error "ps1-meta failed (exit $LASTEXITCODE). Aborting." - exit $LASTEXITCODE - } -} - function build-gte_hello { $includes += @() $path_module = join-path $path_code 'gte_hello' $path_duffle = join-path $path_code 'duffle' $path_atom_metadata = join-path $path_duffle 'word_count.metadata.h' + $path_build_gen = join-path $path_build 'gen' - $source_dirs = @($path_duffle, $path_module) - $atom_sources = Get-SourceFiles -paths $source_dirs -extensions @('.h', '.c') - ps1-meta -sources $atom_sources -metadata $path_atom_metadata -out_root (join-path $path_build 'gen') + $src_c = join-path $path_module 'hello_gte.c' + ps1-meta -unity_root $src_c -metadata $path_atom_metadata -out_root $path_build_gen $assemble_args = @() $assemble_args += $f_debug @@ -360,7 +375,6 @@ function build-gte_hello { # assemble-unit $src_asm $module_asm $includes $assemble_args - $src_c = join-path $path_module 'hello_gte.c' $module_c = join-path $path_build 'hello_gte_c.o' $compile_args = @() @@ -386,27 +400,17 @@ function build-gte_hello { make-binary $elf $exe # Post-link: gdb-runtime + dwarf-injection in a single Lua invocation (one luajit cold start). - ps1-meta -sources $atom_sources -metadata $path_atom_metadata ` - -out_root (join-path $path_build 'gen') ` - -passes @('--post-link') ` - -extra_args @('--elf', $elf) + ps1-meta -unity_root $src_c -metadata $path_atom_metadata -out_root $path_build_gen -passes @('--post-link') ` -extra_args @('--elf', $elf) - # F' + G' splice: collapse 9 objcopy subprocess invocations into 3. - # - 1 call: 3x --update-section for F' (line / aranges / rnglists) - # - 1 call: 3x --update-section for G' (info / abbrev / str) - # - 1 call: 2x --add-section for G' (loc / loclists — these don't exist in the source ELF) - # - 1 call: 1x --set-section-flags (.rodata / .data enable code flag) - # = 4 objcopy calls (was 9; saved 5 spawns). - $dwarfLineBin = join-path (join-path $path_build 'gen') 'hello_gte.dwarf_line.bin' - $dwarfArangesBin = join-path (join-path $path_build 'gen') 'hello_gte.dwarf_aranges.bin' - $dwarfRnglistsBin = join-path (join-path $path_build 'gen') 'hello_gte.dwarf_rnglists.bin' + $dwarfLineBin = join-path $path_build_gen 'hello_gte.dwarf_line.bin' + $dwarfArangesBin = join-path $path_build_gen 'hello_gte.dwarf_aranges.bin' + $dwarfRnglistsBin = join-path $path_build_gen 'hello_gte.dwarf_rnglists.bin' $injectElf = join-path $path_build 'hello_gte.dwarf-injected.elf' if ((Test-Path $dwarfLineBin) -and (Test-Path $dwarfArangesBin) -and (Test-Path $dwarfRnglistsBin)) { Write-Host "[build] DWARF-injecting $elf -> $injectElf" Copy-Item -LiteralPath $elf -Destination $injectElf -Force - - # Single objcopy call: 3x --update-section for F' (line, aranges, rnglists). + # Objcopy call: 3x --update-section for (line, aranges, rnglists). $f_args = @( "--update-section=.debug_line=$dwarfLineBin", "--update-section=.debug_aranges=$dwarfArangesBin", @@ -419,12 +423,11 @@ function build-gte_hello { return; } - # G' 5-section splice: 3 update-section (info / abbrev / str) + 2 add-section (loc / loclists). - $dwarfInfoBin = join-path (join-path $path_build 'gen') 'hello_gte.dwarf_info.bin' - $dwarfAbbrevBin = join-path (join-path $path_build 'gen') 'hello_gte.dwarf_abbrev.bin' - $dwarfStrBin = join-path (join-path $path_build 'gen') 'hello_gte.dwarf_str.bin' - $dwarfLocBin = join-path (join-path $path_build 'gen') 'hello_gte.dwarf_loc.bin' - $dwarfLoclistsBin = join-path (join-path $path_build 'gen') 'hello_gte.dwarf_loclists.bin' + $dwarfInfoBin = join-path $path_build_gen 'hello_gte.dwarf_info.bin' + $dwarfAbbrevBin = join-path $path_build_gen 'hello_gte.dwarf_abbrev.bin' + $dwarfStrBin = join-path $path_build_gen 'hello_gte.dwarf_str.bin' + $dwarfLocBin = join-path $path_build_gen 'hello_gte.dwarf_loc.bin' + $dwarfLoclistsBin = join-path $path_build_gen 'hello_gte.dwarf_loclists.bin' $g_args = @( "--update-section=.debug_info=$dwarfInfoBin", "--update-section=.debug_abbrev=$dwarfAbbrevBin", @@ -441,7 +444,7 @@ function build-gte_hello { # Baked atoms execute from RAM but are emitted as C data arrays, so their ELF sections lack SHF_EXECINSTR. # GDB discards line rows for non-code sections. Mark only the debug-copy sections executable. - # The shipping ELF and PS-EXE remain byte/flag unchanged. + # The original ELF and PS-EXE remain byte/flag unchanged. & $Objcopy ` --set-section-flags ".rodata=alloc,load,readonly,code,contents" ` --set-section-flags ".data=alloc,load,data,code,contents" ` @@ -449,7 +452,8 @@ function build-gte_hello { if ($LASTEXITCODE -ne 0) { Write-Warning "[build] atom-section flag update failed (exit $LASTEXITCODE); removing $injectElf" Remove-Item -LiteralPath $injectElf -ErrorAction SilentlyContinue - } else { + } + else { Write-Host "[build] DWARF-injected ELF: $injectElf" } } diff --git a/scripts/duffle.lua b/scripts/duffle.lua index e0489aa..381c27d 100644 --- a/scripts/duffle.lua +++ b/scripts/duffle.lua @@ -3,8 +3,9 @@ --- --- This module is the source for: --- - **Character classification** (`is_space`, `is_alpha`, `is_alnum`, `is_digit`, plus the byte-fast `_byte` variants). ---- - **String primitives** (`trim`, `dirname`, `basename_no_ext`, `find_byte`). +--- - **String/path primitives** (`trim`, `dirname`, `basename_no_ext`, `normalize_path`, `canonical_path_key`, `find_byte`). --- - **I/O primitives** (`read_file`, `write_file`, `ensure_dir`). +--- - **Canonical corpus resolution** (`parse_direct_quoted_includes`, `resolve_source_corpus`). --- - **C-language scanner** (`skip_ws_and_cmt`, `skip_str_or_cmt`, `read_ident`, `read_parens`, `read_braces`, `read_brackets`, `read_balanced`, `scan_to_char`, `split_top_level_commas`). --- - **Word-count loader** (`load_word_counts` for `WORD_COUNT(...)` metadata files). --- - **Line lookup** (`LineIndex` returns an O(log N) `line_of(pos)` closure for source-mapping). @@ -73,8 +74,8 @@ local BYTE_DIGIT_9 = 0x39 -- '9' -- ════════════════════════════════════════════════════════════════════════════ -- -- Path setup is done by `scripts/duffle_paths.lua`, which derives the repo root from `debug.getinfo(1, "S").source` (NO subprocess, ~0ms) and then calls `require("duffle")`. --- The prior `io.popen("git rev-parse ...")` approach in this section was removed during F'' because: --- 1. Every entry script + every passes script now uses `dofile("duffle_paths.lua")` (14 call sites; verified via grep). +-- Repository paths come from `scripts/duffle_paths.lua` because: +-- 1. Entry and pass scripts load `duffle_paths.lua`. -- The `find_repo_root` / `setup_package_path` defined here was dead code in practice. -- 2. `git rev-parse` costs ~100-180ms per subprocess spawn on Windows. -- `debug.getinfo` is <1ms. There's no reason to keep the slow path even as a "fallback". @@ -88,7 +89,7 @@ local BYTE_DIGIT_9 = 0x39 -- '9' -- -- LPeg is a required dependency (PEG library, no regex). -- It's loaded via `package.cpath` (configured by `duffle_paths.lua` to find `toolchain/lpeg/lpeg.dll`). --- There's no hand-rolled fallback. The original two-tier design added complexity for a 5-10x speedup that's +-- LPeg handles the high-level scanner, while Section 1 handles byte classification. -- only relevant at the high-level scanner stage; the byte-by-byte helpers in Section 1 are sufficient for the classification primitives. -- -- If the require fails, fail loud with an actionable message. The build script (`update_deps.ps1`) builds lpeg.dll into `toolchain/lpeg/`; @@ -185,11 +186,11 @@ function M.trim(s) return s:sub(a, b) end --- Linear-search for a single-byte target in a string. --- @param haystack string --- @param target integer -- byte value --- @param start integer -- optional 1-indexed start (default 1) --- @return integer|nil +--- Linear-search for a single-byte target in a string. +--- @param haystack string +--- @param target integer -- byte value +--- @param start integer -- optional 1-indexed start (default 1) +--- @return integer|nil function M.find_byte(haystack, target, start) for pos = start or 1, #haystack do if haystack:byte(pos) == target then return pos end @@ -223,13 +224,116 @@ function M.basename_no_ext(path) return path:sub(a, last_dot - 1) end +--- Parse the lexical root without changing display spelling. +--- UNC server/share names are part of the immutable root; drive-relative paths remain distinct from drive-absolute paths. +local function parse_path_root(input) + local drive = input:match("^(%a:)") + if drive then + if input:sub(3, 3) == "/" then + local rest = input:sub(4) + while rest:sub(1, 1) == "/" do rest = rest:sub(2) end + return { kind = "drive_absolute", prefix = drive .. "/", rest = rest, anchored = true } + end + return { kind = "drive_relative", prefix = drive, rest = input:sub(3), anchored = false } + end + + if input:sub(1, 2) == "//" then + local server_start = 3 + local server_end = M.find_byte(input, BYTE_SLASH, server_start) + if not server_end or server_end == server_start then + error("UNC path requires //server/share: " .. input, 3) + end + local server = input:sub(server_start, server_end - 1) + local share_start = server_end + 1 + while input:sub(share_start, share_start) == "/" do + share_start = share_start + 1 + end + local share_end = M.find_byte(input, BYTE_SLASH, share_start) or (#input + 1) + if share_end == share_start then + error("UNC path requires //server/share: " .. input, 3) + end + local share = input:sub(share_start, share_end - 1) + local rest = input:sub(share_end + 1) + while rest:sub(1, 1) == "/" do rest = rest:sub(2) end + return { + kind = "unc_absolute", + prefix = "//" .. server .. "/" .. share, + rest = rest, + anchored = true, + } + end + + if input:sub(1, 1) == "/" then + local rest = input:sub(2) + while rest:sub(1, 1) == "/" do rest = rest:sub(2) end + return { kind = "posix_absolute", prefix = "/", rest = rest, anchored = true } + end + return { kind = "relative", prefix = "", rest = input, anchored = false } +end + +--- Normalize path separators and collapse lexical `.` / `..` segments. +--- Display spelling is preserved; case-folding belongs only in `canonical_path_key`. +--- @param path Path +--- @return Path +function M.normalize_path(path) + if type(path) ~= "string" then error("normalize_path requires a string path", 2) end + if path == "" then return "" end + + local root = parse_path_root(path:gsub("\\", "/")) + local segments = {} + for segment in root.rest:gmatch("[^/]+") do + if segment == "." then + -- no-op + elseif segment == ".." then + if #segments > 0 and segments[#segments] ~= ".." then + segments[#segments] = nil + elseif not root.anchored then + segments[#segments + 1] = segment + end + else + segments[#segments + 1] = segment + end + end + + local tail = table.concat(segments, "/") + if root.kind == "relative" then return tail ~= "" and tail or "." end + if root.kind == "drive_relative" then return root.prefix .. tail end + if root.kind == "unc_absolute" then return tail ~= "" and (root.prefix .. "/" .. tail) or root.prefix end + return root.prefix .. tail +end + +local function absolute_normalized_path(path) + local normalized = M.normalize_path(path) + local root = parse_path_root(normalized) + if root.kind == "drive_relative" then + error("drive-relative path cannot be resolved without a per-drive cwd: " .. normalized, 3) + end + if root.anchored then return normalized end + return M.normalize_path(lfs.currentdir() .. "/" .. normalized) +end + +--- Return the normalized absolute, Windows-case-folded comparison key for a path. +--- -Ordinary relative paths resolve against the process cwd. Drive-relative paths are rejected because LuaFileSystem does not expose Windows per-drive current directories. +--- @param path Path +--- @return string +function M.canonical_path_key(path) + local normalized = M.normalize_path(path) + local root = parse_path_root(normalized) + if root.kind == "drive_relative" then + error("canonical_path_key cannot compare drive-relative path: " .. normalized, 2) + end + local key = absolute_normalized_path(normalized):lower() + if #key > 3 and key:sub(-1) == "/" then key = key:sub(1, -2) end + return key +end + -- ════════════════════════════════════════════════════════════════════════════ -- Section 3: I/O primitives -- ════════════════════════════════════════════════════════════════════════════ --- File contents intentionally use io.open below. LuaFileSystem handles path --- metadata, directory iteration, the current directory, and mkdir; it does not --- expose file-content read/write streams. +-- File contents intentionally use io.open below. +-- LuaFileSystem handles path metadata, directory iteration, the current directory, and mkdir; +-- it does not expose file-content read/write streams. function M.read_file(path) local f = io.open(path, "r") if not f then error("Cannot open " .. path) end @@ -243,21 +347,21 @@ function M.write_file(path, content) f:write(content); f:close() end --- Write content to disk in binary mode so LF line endings are preserved on Windows --- (text mode would convert LF -> CRLF, breaking byte-identical diffs against git-tracked gen/*.h files which are stored as LF). --- @param path string --- @param content string +--- Write content to disk in binary mode so LF line endings are preserved on Windows +--- (text mode would convert LF -> CRLF, breaking byte-identical diffs against git-tracked gen/*.h files which are stored as LF). +--- @param path string +--- @param content string function M.write_file_lf(path, content) local f = io.open(path, "wb") if not f then error("Cannot write " .. path) end f:write(content); f:close() end --- Return `{path, ...}` for files in `out_root` whose basename matches `pattern` (Lua pattern, NOT regex — `%.` not `\.`). --- Empty list if `out_root` doesn't exist or matches nothing. --- @param out_root Path --- @param pattern string -- Lua pattern matched against basename only --- @return string[] +--- Return `{path, ...}` for files in `out_root` whose basename matches `pattern` (Lua pattern, NOT regex — `%.` not `\.`). +--- Empty list if `out_root` doesn't exist or matches nothing. +--- @param out_root Path +--- @param pattern string -- Lua pattern matched against basename only +--- @return string[] function M.list_dir(out_root, pattern) local files = {} if lfs.attributes(out_root, "mode") ~= "directory" then return files end @@ -269,15 +373,14 @@ function M.list_dir(out_root, pattern) return files end --- Convert a (possibly relative) path to an absolute path, using CWD if needed. --- Normalizes forward slashes to backslashes on Windows. --- Used for byte-identical emit: the // Source: comment line uses the absolute path. --- --- The CWD is memoized on first call. --- @param path string --- @return string local _absolute_path_cache = {} +--- Convert a (possibly relative) path to an absolute path, using CWD if needed. +--- Normalizes forward slashes to backslashes on Windows. +--- Used for byte-identical emit: the // Source: comment line uses the absolute path. +--- The CWD is memoized on first call. +--- @param path string +--- @return string function M.to_absolute_path(path) if _absolute_path_cache[path] then return _absolute_path_cache[path] end if #path >= 2 and path:sub(2, 2) == ":" then @@ -310,11 +413,11 @@ end -- Not normally needed since Lua state is per-process. function M._reset_ensured_dirs() _ensured_dirs = {} end --- Group a list of `SourceFile`-shaped records by their `dir` field. --- Used by the annotation / static-analysis / report passes to partition sources into per-DIRECTORY (per-module) buckets --- before emitting per-module reports. Insertion order preserved within each bucket (matches source order in `ctx.sources`). --- @param sources table[] -- list of source records (each having a `dir` string field) --- @return table -- map of `dir` -> sources in that dir +--- Group a list of `SourceFile`-shaped records by their `dir` field. +--- Used by the annotation / static-analysis / report passes to partition sources into per-DIRECTORY (per-module) buckets before emitting per-module reports. +--- Insertion order preserved within each bucket (matches source order in `ctx.sources`). +--- @param sources table[] -- list of source records (each having a `dir` string field) +--- @return table -- map of `dir` -> sources in that dir function M.group_sources_by_dir(sources) local by_dir = {} for _, src in ipairs(sources) do @@ -345,8 +448,7 @@ function M.read_ident(s, pos) end -- Read a balanced-delimited group (parens, braces, or brackets) starting at position `pos`. --- Returns the inner content (between the delimiters) + the position --- just past the closing delimiter, or nil + pos if `s[pos]` isn't `open_char`. +-- Returns the inner content (between the delimiters) + the position just past the closing delimiter, or nil + pos if `s[pos]` isn't `open_char`. function M.read_balanced(s, open_char, close_char, pos) local open_byte = open_char:byte() if s:byte(pos) ~= open_byte then return nil, pos end @@ -360,7 +462,7 @@ function M.read_balanced(s, open_char, close_char, pos) local c = s:byte(pos) if c == open_byte then depth = depth + 1 - pos = pos + 1 + pos = pos + 1 -- scan: (depth=depth) elseif c == close_char:byte() then depth = depth - 1 @@ -386,8 +488,7 @@ M.read_parens = function(s, pos) return M.read_balanced(s, "(", ")", pos) end M.read_braces = function(s, pos) return M.read_balanced(s, "{", "}", pos) end M.read_brackets = function(s, pos) return M.read_balanced(s, "[", "]", pos) end --- Scan forward from position `start` until we find a specific single byte `target`, --- transparently stepping over balanced parens/braces/brackets. +-- Scan forward from position `start` until we find a specific single byte `target`, transparently stepping over balanced parens/braces/brackets. -- Returns the position of `target`, or nil if not found. function M.scan_to_char(s, target, start) local target_byte = target:byte() @@ -413,13 +514,303 @@ end function M.skip_preprocessor_line(s, pos) if s:byte(pos) ~= 35 then return nil end -- '#' local scan = pos - local len = #s - while scan <= len and s:byte(scan) ~= BYTE_NEWLINE do - scan = scan + 1 - end + local len = #s + while scan <= len and s:byte(scan) ~= BYTE_NEWLINE do scan = scan + 1 end return scan + 1 end +local function is_horizontal_space(byte) + return byte == BYTE_SPACE or byte == BYTE_TAB or byte == BYTE_CR or byte == BYTE_VT or byte == BYTE_FF +end + +local function segment_has_newline(source, first, after_last) + for pos = first, after_last - 1 do + if source:byte(pos) == BYTE_NEWLINE then return true end + end + return false +end + +local function skip_directive_space(source, pos) + while pos <= #source do + local byte = source:byte(pos) + if is_horizontal_space(byte) then + pos = pos + 1 + elseif byte == BYTE_SLASH and source:byte(pos + 1) == BYTE_STAR then + local after = M.skip_str_or_cmt(source, pos) + if after == pos or segment_has_newline(source, pos, after) then return nil end + pos = after + elseif byte == BYTE_SLASH and source:byte(pos + 1) == BYTE_SLASH then + return nil + else + break + end + end + return pos +end + +--- Apply C line splicing once for the include scanner. +--- Every retained logical byte maps back to its original physical byte offset and one-based physical line so diagnostics preserve source-as-written evidence. +local function splice_c_lines(source) + local logical_bytes = {} + local physical_pos = {} + local physical_line = {} + local pos = 1 + local line = 1 + while pos <= #source do + local byte = source:byte(pos) + local splice_len = nil + if byte == BYTE_BACKSLASH and source:byte(pos + 1) == BYTE_NEWLINE then + splice_len = 2 + elseif byte == BYTE_BACKSLASH and source:byte(pos + 1) == BYTE_CR + and source:byte(pos + 2) == BYTE_NEWLINE then + splice_len = 3 + end + + if splice_len then + pos = pos + splice_len + line = line + 1 + else + local logical_pos = #logical_bytes + 1 + logical_bytes[logical_pos] = source:sub(pos, pos) + physical_pos [logical_pos] = pos + physical_line[logical_pos] = line + if byte == BYTE_NEWLINE then line = line + 1 end + pos = pos + 1 + end + end + return table.concat(logical_bytes), physical_pos, physical_line +end + +--- Parse direct quoted preprocessor includes from one source buffer. +--- Translation Line splicing occurs ahead of comment, string, and directive processing. +--- Interpreted records retain original physical include text and line numbers. +--- Angle includes and include-like text inside comments/strings are ignored. +--- @param source_text string +--- @return table[] -- ordered `{path, include_path, include_text, line}` records +function M.parse_direct_quoted_includes(source_text) + if type(source_text) ~= "string" then + error("parse_direct_quoted_includes requires source text", 2) + end + + local logical_text, physical_pos, physical_line = splice_c_lines(source_text) + local includes = {} + local pos = 1 + local line_leading = true + while pos <= #logical_text do + local byte = logical_text:byte(pos) + if byte == BYTE_NEWLINE then + line_leading = true + pos = pos + 1 + elseif is_horizontal_space(byte) then + pos = pos + 1 + elseif byte == BYTE_SLASH and logical_text:byte(pos + 1) == BYTE_SLASH then + local after = M.skip_str_or_cmt(logical_text, pos) + pos = (after > pos) and after or (pos + 1) + elseif byte == BYTE_SLASH and logical_text:byte(pos + 1) == BYTE_STAR then + local after = M.skip_str_or_cmt(logical_text, pos) + if after == pos then + line_leading = false + pos = pos + 1 + else + for scan = pos, after - 1 do + if logical_text:byte(scan) == BYTE_NEWLINE then line_leading = true end + end + pos = after + end + elseif byte == BYTE_DQUOTE or byte == BYTE_SQUOTE then + line_leading = false + local after = M.skip_str_or_cmt(logical_text, pos) + pos = (after > pos) and after or (pos + 1) + elseif byte == 35 and line_leading then -- '#' + local hash_pos = pos + local directive_line = physical_line[hash_pos] or 1 + local scan = skip_directive_space(logical_text, pos + 1) + local ident, after_ident + if scan then ident, after_ident = M.read_ident(logical_text, scan) end + if ident == "include" then + scan = skip_directive_space(logical_text, after_ident) + end + if ident == "include" and scan and logical_text:byte(scan) == BYTE_DQUOTE then + local after_quote = M.skip_str_or_cmt(logical_text, scan) + if after_quote > scan and logical_text:byte(after_quote - 1) == BYTE_DQUOTE then + local include_path = logical_text:sub(scan + 1, after_quote - 2) + local physical_first = physical_pos[hash_pos] + local physical_last = physical_pos[after_quote - 1] + includes[#includes + 1] = { + path = include_path, + include_path = include_path, + include_text = M.trim(source_text:sub(physical_first, physical_last)), + line = directive_line, + } + pos = after_quote + else + pos = pos + 1 + end + else + pos = pos + 1 + end + line_leading = false + else + line_leading = false + pos = pos + 1 + end + end + return includes +end + +local function path_has_segment(path, wanted) + for segment in M.normalize_path(path):gmatch("[^/]+") do + if segment:lower() == wanted then return true end + end + return false +end + +local function canonical_key_is_within(candidate_key, root_key) + if candidate_key == root_key then return true end + local prefix = root_key .. "/" + return candidate_key:sub(1, #prefix) == prefix +end + +local function load_source_record(path) + local normalized = absolute_normalized_path(path) + return { + path = normalized, + text = M.read_file(normalized), + dir = M.dirname(normalized), + basename = M.basename_no_ext(normalized), + } +end + +--- Resolve a unity source corpus without recursive discovery. +--- The root is loaded first; only its direct quoted includes are considered, in source order. +--- Candidate A is root-directory relative and candidate B is `/code` relative. +--- @param options table -- `{unity_root=Path, project_root=Path}` +--- @return table +function M.resolve_source_corpus(options) + if type(options) ~= "table" then error("resolve_source_corpus requires options", 2) end + if type(options.unity_root) ~= "string" or options.unity_root == "" then + error("resolve_source_corpus requires options.unity_root", 2) + end + if type(options.project_root) ~= "string" or options.project_root == "" then + error("resolve_source_corpus requires options.project_root", 2) + end + + local project_root = absolute_normalized_path(options.project_root) + local code_root = M.normalize_path(project_root .. "/code") + local code_root_key = M.canonical_path_key(code_root) + local root = load_source_record(options.unity_root) + local source_order = { root } + local sources_by_path = { [M.canonical_path_key(root.path)] = root, } + local resolver = { + resolved = { + { + include_path = nil, + include_text = nil, + root_source = root.path, + root_line = 1, + candidate_a = root.path, + candidate_b = nil, + selected_path = root.path, + disposition = "root", + }, + }, + skipped = {}, + shadowed = {}, + } + + for _, include in ipairs(M.parse_direct_quoted_includes(root.text)) do + local candidate_a = absolute_normalized_path(root.dir .. "/" .. include.path) + local candidate_b = absolute_normalized_path(code_root .. "/" .. include.path) + local key_a = M.canonical_path_key(candidate_a) + local key_b = M.canonical_path_key(candidate_b) + local inside_a = canonical_key_is_within(key_a, code_root_key) + local inside_b = canonical_key_is_within(key_b, code_root_key) + local evidence = { + include_path = include.path, + include_text = include.include_text, + root_source = root.path, + root_line = include.line, + candidate_a = candidate_a, + candidate_b = candidate_b, + candidate_a_in_code_root = inside_a, + candidate_b_in_code_root = inside_b, + selected_path = nil, + disposition = nil, + } + + if not inside_a and not inside_b then + evidence.disposition = "skipped" + evidence.reason = "outside_code_root" + resolver.skipped[#resolver.skipped + 1] = evidence + elseif (inside_a and path_has_segment(candidate_a, "gen")) + or (inside_b and path_has_segment(candidate_b, "gen")) then + evidence.disposition = "skipped" + evidence.reason = "gen_segment" + resolver.skipped[#resolver.skipped + 1] = evidence + else + -- Boundary checks above deliberately precede every filesystem probe. + local exists_a = inside_a and lfs.attributes(candidate_a, "mode") == "file" + local exists_b = inside_b and ((key_b == key_a and exists_a) or lfs.attributes(candidate_b, "mode") == "file") + local selected = nil + local selected_key = nil + local disposition = nil + if exists_a then + selected = candidate_a + selected_key = key_a + disposition = "resolved_local" + elseif exists_b then + selected = candidate_b + selected_key = key_b + disposition = "resolved_code" + end + + if exists_a and exists_b and key_a ~= key_b then + resolver.shadowed[#resolver.shadowed + 1] = { + include_path = include.path, + include_text = include.include_text, + root_source = root.path, + root_line = include.line, + candidate_a = candidate_a, + candidate_b = candidate_b, + selected_path = candidate_a, + alternate_path = candidate_b, + disposition = "local_candidate_selected", + } + end + + if not selected then + evidence.disposition = "skipped" + evidence.reason = "unresolved" + resolver.skipped[#resolver.skipped + 1] = evidence + else + evidence.selected_path = selected + if sources_by_path[selected_key] then + evidence.disposition = "duplicate" + evidence.reason = "duplicate" + evidence.duplicate_of = sources_by_path[selected_key].path + resolver.skipped[#resolver.skipped + 1] = evidence + else + local source = load_source_record(selected) + evidence.disposition = disposition + source_order[#source_order + 1] = source + sources_by_path[selected_key] = source + resolver.resolved[#resolver.resolved + 1] = evidence + end + end + end + end + + return { + unity_root = root.path, + project_root = project_root, + code_root = code_root, + source_order = source_order, + sources_by_path = sources_by_path, + sources_by_dir = M.group_sources_by_dir(source_order), + resolver = resolver, + } +end + -- Split a brace-body into top-level comma-separated tokens. Honors nested parens/braces/brackets and skips strings/comments. -- -- FIX (2026-07-09): split at top-level NEWLINES and SEMICOLONS too, AND emit a token break after a top-level comment/string. @@ -432,9 +823,8 @@ function M.split_top_level_commas(body) local body_len = #body local token_start = 1 - -- True iff `chunk` contains any non-whitespace, non-comment, non-string content - -- (i.e., real token material). Walks through ws + comments individually so a chunk like " /* trailing */ shift_lleft(...)" - -- is correctly classified as having real content (the macro call). + -- True iff `chunk` contains any non-whitespace, non-comment, non-string content (i.e., real token material). + -- Walks through ws + comments individually so a chunk like " /* trailing */ shift_lleft(...)" is correctly classified as having real content (the macro call). local function has_real_content(chunk) local scan = 1 local len = #chunk @@ -465,8 +855,7 @@ function M.split_top_level_commas(body) -- can convert `// trailing comment` to `/* */` and emit it with the macro body. -- For word counting, count_token_words only inspects the leading ident, so a trailing comment doesn't affect the count. -- - -- This is the second-half fix to commit 98e27c2: the first fix correctly broke top-level comments - -- off from the NEXT statement (fixing macro-call word counts); + -- This is the second-half fix to commit 98e27c2: the first fix correctly broke top-level comments off from the NEXT statement (fixing macro-call word counts); -- This fix preserves them on the PREVIOUS statement (restoring the comments in the emitted .macs.h output). tokens[#tokens] = tokens[#tokens] .. chunk end @@ -559,7 +948,7 @@ function M.tokenize_body(body) if scan <= len then scan = scan + 1 local w = M.skip_ws_and_cmt(body, scan) - if w > scan then scan = w end + if w > scan then scan = w end end rel = scan end @@ -580,8 +969,8 @@ function M.build_body_line_index(body) if pos > 1 then index[pos] = newline_count + 1 end - -- Newline byte = 0x0A (BYTE_NEWLINE). Counts line boundaries so the - -- index maps each source-byte offset → its 1-based line number. + -- Newline byte = 0x0A (BYTE_NEWLINE). + -- Counts line boundaries so the index maps each source-byte offset → its 1-based line number. if body:byte(pos) == BYTE_NEWLINE then newline_count = newline_count + 1 end @@ -666,7 +1055,7 @@ function M.load_word_counts(metadata_path) end -- ══════════════════════════════════════════════════ --- Section 6: LineIndex (perf fix — replaces the per-call rescan line_of) +-- Section 6: LineIndex (constant-time line lookup) -- ══════════════════════════════════════════════════ function M.LineIndex(source) @@ -705,8 +1094,8 @@ M.TAPE_ATOM_MACROS = { -- GTE command-alias resolution table. -- -- Maps each GTE command macro that may appear in source to its CANONICAL short form. --- Both forms resolve to the same PSX-SPX-documented pipeline semantics; the canonical --- name is the only one that appears in `GTE_COMMAND_INPUTS` and the per-check producer / consumer reports. +-- Both forms resolve to the same PSX-SPX-documented pipeline semantics; +-- The canonical name is the only one that appears in `GTE_COMMAND_INPUTS` and the per-check producer / consumer reports. -- Aliases resolve exactly once; unknown idents (e.g. an MVMVA with a custom `(sf, mx, v, cv, lm)` payload that is not on this list) -- are reported as "command unknown" by the check, not silently treated as 0-cycle. -- @@ -716,26 +1105,24 @@ M.TAPE_ATOM_MACROS = { -- Every alias row maps source ident -> canonical short ident. M.GTE_COMMAND_ALIASES = { -- Canonical -> canonical (identity). - ["gte_cmdw_rtps"] = "gte_cmdw_rtps", - ["gte_cmdw_rtpt"] = "gte_cmdw_rtpt", - ["gte_cmdw_nclip"] = "gte_cmdw_nclip", - ["gte_cmdw_mvmva"] = "gte_cmdw_mvmva", - ["gte_cmdw_op"] = "gte_cmdw_op", - ["gte_cmdw_avsz3"] = "gte_cmdw_avsz3", - ["gte_cmdw_avsz4"] = "gte_cmdw_avsz4", + ["gte_cmdw_rtps"] = "gte_cmdw_rtps", + ["gte_cmdw_rtpt"] = "gte_cmdw_rtpt", + ["gte_cmdw_nclip"] = "gte_cmdw_nclip", + ["gte_cmdw_mvmva"] = "gte_cmdw_mvmva", + ["gte_cmdw_op"] = "gte_cmdw_op", + ["gte_cmdw_avsz3"] = "gte_cmdw_avsz3", + ["gte_cmdw_avsz4"] = "gte_cmdw_avsz4", -- Aliases -> canonical. ["gte_cmdw_rotate_translate_perspective_single"] = "gte_cmdw_rtps", ["gte_cmdw_rotate_translate_perspective_triple"] = "gte_cmdw_rtpt", - ["gte_cmdw_avg_sort_z3"] = "gte_cmdw_avsz3", - ["gte_cmdw_avg_sort_z4"] = "gte_cmdw_avsz4", - ["gte_cmdw_outer_product"] = "gte_cmdw_op", - ["gte_cmdw_wedge"] = "gte_cmdw_op", + ["gte_cmdw_avg_sort_z3"] = "gte_cmdw_avsz3", + ["gte_cmdw_avg_sort_z4"] = "gte_cmdw_avsz4", + ["gte_cmdw_outer_product"] = "gte_cmdw_op", + ["gte_cmdw_wedge"] = "gte_cmdw_op", -- Bare-name aliases (no `gte_cmdw_` prefix; used in atom bodies directly): -- gte_avg_sort_z3 / gte_avg_sort_z4 are the duffle-side aliases for AVSZ3/4. - -- alias-to-canonical resolution lives in `check_gte_write_retire` (via - -- `M.GTE_COMMAND_ALIASES`); see static_analysis.lua :: check_gte_write_retire. - ["gte_avg_sort_z3"] = "gte_cmdw_avsz3", - ["gte_avg_sort_z4"] = "gte_cmdw_avsz4", + ["gte_avg_sort_z3"] = "gte_cmdw_avsz3", + ["gte_avg_sort_z4"] = "gte_cmdw_avsz4", } -- GTE command input-set table. @@ -822,22 +1209,172 @@ M.GTE_COMMAND_INPUTS = { }, } --- COP2 write-retire slot table. +-- GTE command output-set + semantic role table. -- --- Number of cached instruction slots required for a recent CPU-to-COP2 write to retire before the next dependent command can issue. --- Per PSX-SPX `cpuspecifications.md:407-419` and `gtepipelinetimings.md` --- (every per-input N for a recent mtc2/ctc2, with two cached instructions being the common case and three for IRGB / ORGB writes). +-- For each canonical command, the SET of C2 data registers the command writes as outputs, paired with the SEMANTIC ROLE of each output. +-- The semantic role is the basis for the `_post_` contract validation: +-- The contract says "after , the latest screen-XY is C2_SXY2" (NOT C2_SXY0. The FIFO side effects do NOT make SXY0 the newest result). -- --- The model is intentionally small: --- cpu_to_cop2 -- default for any mtc2 / ctc2 / lwc2 / swc2 to a COP2 register --- cpu_to_irgb -- override for writes to the IRGB / ORGB control registers (the documented 3-cycle fan-out) +-- Per PSX-SPX `docs/psx-spx/docs/geometrytransformationenginegte.md`: +-- * RTPS: writes VXY/VZ -> MAC results; the SINGLE projected screen coordinate is written to C2_SXY2 (the IRGB -> SXY2 path via the perspective divide). +-- C2_SXY0 and C2_SXY1 are NOT written. +-- * RTPT: writes three projected screen coordinates into SXY0, SXY1, SXY2 in pipeline order. +-- The LAST projection is in C2_SXY2; a reader that wants "the last RTPT result" must read C2_SXY2. +-- * NCLIP: writes a single MAC result into C2_SZ3 (the inner-product sum); no screen XY output. +-- * AVSZ3 / AVSZ4: write average Z into C2_OTZ (single output). +-- * OP: writes C2_IR1, C2_IR2, C2_IR3 (cross-product result; no projection). +-- * MVMVA: writes C2_IR1, C2_IR2, C2_IR3 (single MAC result; same shape as OP from the role perspective). -- --- The downstream check (`check_gte_write_retire`) resolves the per-event class from the C2 destination --- and the write kind (gte_mv_to_data_r vs. gte_mv_to_ctrl_r + control-register index). --- A future pass can introduce command-specific per-input-N tables; the structural place to add them is here. -M.COP2_WRITE_RETIRE_SLOTS = { - cpu_to_cop2 = 2, - cpu_to_irgb = 3, +-- Role taxonomy (closed set): +-- * "latest_screen_xy" : newest projected screen X/Y pair +-- * "latest_screen_z" : newest projected screen Z +-- * "latest_color" : newest IRGB / IR fan-out result +-- * "screen_xy[N]" : Nth projection in a batched sequence +-- * "screen_z" : Z projection (avsz / otz) +-- * "otz" : ordered-table Z (avsz output) +-- * "mac_result" : generic MAC output (nclip, op, mvmva) +-- +-- Consumers: +-- * passes/static_analysis.lua::analyze_hardware_relations (the walker consults this table after a GTE command to update `forward_state.post_command_roles` for `gte_result_position`). +-- * passes/static_analysis.lua::check_gte_result_position (per-atom CHECK_RULES reader; renders role mismatches). +-- This table is consumed by the hardware-relation analyzer and result-position check. +M.GTE_COMMAND_OUTPUTS = { + -- RTPS: writes ONE screen coordinate (the perspective-divide result) + -- into C2_SXY2; the FIFO side effects do NOT make SXY0 / SXY1 newest. + -- `latest_screen_xy` is C2_SXY2. + ["gte_cmdw_rtps"] = { + { register = "C2_SXY2", role = "latest_screen_xy" }, + { register = "C2_SZ2", role = "latest_screen_z" }, + { register = "C2_OTZ", role = "otz" }, + { register = "C2_IR0", role = "latest_color" }, + }, + -- RTPT: writes THREE screen coordinates; the LAST projection lands in + -- C2_SXY2. `latest_screen_xy` is C2_SXY2; C2_SXY0 / C2_SXY1 are the + -- earlier projections of the batched triple. + ["gte_cmdw_rtpt"] = { + { register = "C2_SXY0", role = "screen_xy[0]" }, + { register = "C2_SXY1", role = "screen_xy[1]" }, + { register = "C2_SXY2", role = "latest_screen_xy" }, + { register = "C2_SZ3", role = "latest_screen_z" }, + { register = "C2_OTZ", role = "otz" }, + }, + -- NCLIP: single MAC result; written to C2_SZ3 (the inner-product sum). + -- No screen XY output. + ["gte_cmdw_nclip"] = { + { register = "C2_SZ3", role = "mac_result" }, + }, + -- AVSZ3 / AVSZ4: average Z written to C2_OTZ. + ["gte_cmdw_avsz3"] = { + { register = "C2_OTZ", role = "otz" }, + }, + ["gte_cmdw_avsz4"] = { + { register = "C2_OTZ", role = "otz" }, + }, + -- OP (outer product): writes IR1/IR2/IR3 (color-conversion fan-out). + ["gte_cmdw_op"] = { + { register = "C2_IR1", role = "latest_color" }, + { register = "C2_IR2", role = "latest_color" }, + { register = "C2_IR3", role = "latest_color" }, + }, + -- MVMVA: same shape as OP from the role perspective; the single + -- MAC result is written to C2_IR1/IR2/IR3. + ["gte_cmdw_mvmva"] = { + { register = "C2_IR1", role = "latest_color" }, + { register = "C2_IR2", role = "latest_color" }, + { register = "C2_IR3", role = "latest_color" }, + }, +} + +-- GTE command/post-command latch-window table. +-- +-- Per PSX-SPX `docs/psx-spx/docs/gtepipelinetimings.md`, a GTE command emits outputs that latch into the pipeline for a measured number of emitted words. +-- A subsequent MTC2/CTC2 OVERWRITE of one of those outputs BEFORE the latch window expires is a hazard +-- (the latched value in the pipeline is overwritten by the CPU before the pipeline consumes it). +-- +-- This relation is the COMMAND -> REGISTER direction (the command is the producer, the MTC2/CTC2 is the consumer). +-- It is NOT the same relation as the preceding MTC2 -> command input propagation +-- (which is the REGISTER -> COMMAND direction and is staged by the producer step of `analyze_hardware_relations`). +-- +-- The schema mirrors the producer-side relations (`direction`, `evidence`, `violation_kind`); +-- `required` is the number of emitted words strictly between the command's last output word and the overwrite. +-- `N=0` permits the immediately following overwrite instruction; `N=4` permits an overwrite that occurs after 4 intervening words. +-- +-- Per PSX-SPX `gtepipelinetimings.md` +-- The per-command input latching measurements are the SAME number, just inverted: +-- They describe when a recent MTC2/CTC2 must retire before the command issues; +-- here we describe when a recent command's outputs latch into the pipeline before a later MTC2/CTC2 may overwrite them. +-- +-- Consumers: +-- * passes/static_analysis.lua::analyze_hardware_relations (the walker consults this table after a GTE command to stage post-command latch relations in `pending`). +-- * passes/static_analysis.lua::check_gte_input_latch (per-atom CHECK_RULES reader; renders the over-the-boundary findings). +-- This table is consumed by the hardware-relation analyzer and input-latch check. +M.GTE_COMMAND_LATCH_WINDOWS = { + -- RTPS output latches: a subsequent MTC2 to SXY0 within 4 emitted words overwrites the latched result. + -- (PSX-SPX §"RTPS" lists the measured boundary; the exact number is from `gtepipelinetimings.md`.) + ["gte_cmdw_rtps"] = { + { register = "C2_SXY2", required = 4 }, + { register = "C2_SZ2", required = 4 }, + { register = "C2_OTZ", required = 4 }, + { register = "C2_IR0", required = 4 }, + }, + -- RTPT: same latching as RTPS (the LAST projection in SXY2 is the + -- newest one; the earlier SXY0 / SXY1 entries are part of the + -- batched triple). + ["gte_cmdw_rtpt"] = { + { register = "C2_SXY0", required = 4 }, + { register = "C2_SXY1", required = 4 }, + { register = "C2_SXY2", required = 4 }, + { register = "C2_SZ3", required = 4 }, + { register = "C2_OTZ", required = 4 }, + }, + -- NCLIP output (SZ3): latches for 4 emitted words. + ["gte_cmdw_nclip"] = { + { register = "C2_SZ3", required = 4 }, + }, + -- AVSZ3/4: OTZ output latches for 4 emitted words. + ["gte_cmdw_avsz3"] = { + { register = "C2_OTZ", required = 4 }, + }, + ["gte_cmdw_avsz4"] = { + { register = "C2_OTZ", required = 4 }, + }, + -- OP / MVMVA: IR1/IR2/IR3 latch for 4 emitted words. + ["gte_cmdw_op"] = { + { register = "C2_IR1", required = 4 }, + { register = "C2_IR2", required = 4 }, + { register = "C2_IR3", required = 4 }, + }, + ["gte_cmdw_mvmva"] = { + { register = "C2_IR1", required = 4 }, + { register = "C2_IR2", required = 4 }, + { register = "C2_IR3", required = 4 }, + }, +} + +-- GTE component result contracts (immutable; keyed by bare component name). +-- +-- Register-role claims that cannot be inferred from the `_post_` suffix alone live here. +-- The bare name (without the `_post_` suffix) is the key; the row carries the expected command, the expected role, and the expected C2 register. +-- +-- Known rows: +-- * `gte_store_g4_p3_post_rtps`: post-RTPS polygon-emit slot reads the newest projected screen coordinate from C2_SXY2 +-- (NOT C2_SXY0; the FIFO side effects do not make SXY0 the newest result). +-- +-- Unknown `_post_` components (a `_post_` suffixed component name whose bare `` is not a row key) +-- emit ONE `table_gap` info finding so downstream consumers can detect when the canonical contract table is incomplete for an authored atom body. +-- +-- Consumers: +-- * passes/static_analysis.lua::check_gte_result_position (per-atom CHECK_RULES reader; renders result-position findings). +-- * passes/static_analysis.lua::emit_table_gap_warning (called once per atom body; surfaces the missing-row diagnostic). +-- This table is consumed by the result-position check. +M.GTE_COMPONENT_RESULT_CONTRACTS = { + -- Post-RTPS g4 p3 store contract: writes the latest screen XY (C2_SXY2) into the primitive's p3 slot. + -- Reads from C2_SXY0 would be a semantic mismatch (C2_SXY0 is the OLDEST post-RTPS SXY, not the newest one). + ["gte_store_g4_p3_post_rtps"] = { + command = "gte_cmdw_rtps", + role = "latest_screen_xy", + register = "C2_SXY2", + }, } -- Operand-class table for the COP2->GPR load-delay check. @@ -847,8 +1384,7 @@ M.COP2_WRITE_RETIRE_SLOTS = { -- expand by adding rows here as new encoders land. -- -- Semantics: --- * A "GPR operand position" is the textual slot in the macro's argument list, --- 1-based; e.g. `load_word(rt, base, off)` has positional operands 1 (rt), 2 (base), 3 (off); +-- * A "GPR operand position" is the textual slot in the macro's argument list, 1-based; e.g. `load_word(rt, base, off)` has positional operands 1 (rt), 2 (base), 3 (off); -- The table reads operands 1 + 2 + 3 to find what GPRs the macro touches. -- * The check tracks one entry per destination GPR per MFC2/CFC2 event. -- A subsequent event is considered a "use" iff any of its READ operand positions reference that destination GPR's ident (e.g. `R_T0`). @@ -1008,12 +1544,11 @@ M.GP0_MACRO_CONTRIB = { -- mvmva = 8 + 2 nops = 10 total -- op = 6 (no pre-cmd nops required; atomic) -- --- Note: the "total" above is the pre-fill nops + the GTE intrinsic cycles. PSX-SPX documents the GTE --- intrinsic cycles as the total execution time of the command itself (rtpt=23, rtps=15, nclip=8, etc.). --- The pre-fill nops are a codebase convention for retiring preceding C2 writes, not part of the GTE's --- own execution time. See `docs/psx-spx/docs/geometrytransformationenginegte.md` for the canonical --- per-command cycle counts and `docs/psx-spx/docs/gtepipelinetimings.md` for the hardware-verified --- input-latch boundaries (which show most inputs are safe to clobber after just 0-4 cycles). +-- Note: the "total" above is the pre-fill nops + the GTE intrinsic cycles. +-- PSX-SPX documents the GTE intrinsic cycles as the total execution time of the command itself (rtpt=23, rtps=15, nclip=8, etc.). +-- The pre-fill nops are a codebase convention for retiring preceding C2 writes, not part of the GTE's own execution time. +-- See `docs/psx-spx/docs/geometrytransformationenginegte.md` for the canonical per-command cycle counts and `docs/psx-spx/docs/gtepipelinetimings.md` +-- for the hardware-verified input-latch boundaries (which show most inputs are safe to clobber after just 0-4 cycles). M.INSTRUCTION_LATENCY = { -- CPU ALU (single-cycle R3000A ops) ["nop"] = 1, @@ -1136,12 +1671,376 @@ M.INSTRUCTION_LATENCY = { -- advisory so the cycle budget stays accurate as the codebase grows. M.UNKNOWN_INSTRUCTION_CYCLES = 1 +-- Hardware-relation policy table. +-- +-- The single forward-analyzer in `passes/static_analysis.lua::analyze_hardware_relations` +-- reads every emitted word_event, matches its `encoder` against `row.token`, and: +-- * stages the event as a producer in `atom.paths.forward_state`; or +-- * matches it as a consumer against pending producers and records a hazard on `atom.paths.hazards` when the gap is below `visibility.required`. +-- +-- Each row is the contract for one CPU-to-coprocessor transfer semantic +-- (the coprocessor-to-CPU path mirrors the same shape). The `reads` / `writes` sub-tables carry the argument positions the analyzer inspects: +-- * `writes.arg` is the destination operand (the producer's effect); the analyzer stages this register as a pending producer. +-- * `reads` (when present) lists the operand positions the SAME token reads back from hardware; for MTC2 / CTC2 the producer reads the GPR source it is loading from. +-- The `fanout_to` field (MTC2-IRGB row only) tells the consumer-match logic which downstream COP2 registers are transitively updated by the write. +-- +-- Visibility semantics: +-- * `kind = "post_producer_words"` means the consumer must observe the producer's effect after `required` independent emitted words that are strictly between the producer and the consumer. +-- The producer's own emitted slot does NOT retire the relation (per the canonical PSX-SPX rule: "Store delays are counted in numbers of clock cycles (not in numbers of opcodes). +-- For 3 cycle delay, one must usuallys insert 3 cached opcodes (or one uncached opcode)."). +-- * `required` is the minimum count of intervening emitted words between producer and consumer. +-- `required = 0` is permitted (the consumer may sit on the very next slot); +-- `required < 0` would mean the consumer may sit on the same slot as the producer and is reserved for future "self-retires" relations. +-- +-- Evidence: +-- * `evidence.confidence` is one of `"exact"`, `"conservative"`, `"unknown"`. The severity comes from `violation_kind`; +-- a hardware measurement that the vendor caveats may still be `"conservative"` even when the underlying timing is numerically known. +-- * `evidence.source` is the canonical upstream reference (file + line range) the row is sourced from. Doc-edits that add new rows must add the source citation here. +-- +-- Consumers: +-- * passes/static_analysis.lua::analyze_hardware_relations (forward walker). +-- * passes/static_analysis.lua::transfer_hazards CHECK_RULES reader (renders hazards onto `findings`). +-- This table is consumed by the hardware-relation analyzer and hazard renderer. +M.HARDWARE_RELATIONS = { + -- CPU → COP2 data register (MTC2). The ordinary default is 2 cached words between producer and consumer (cpuspecifications.md:407-419). + { + id = "mtc2_gpr_visibility", + semantic = "MTC2", + token = "gte_mv_to_data_r", + direction = "gpr_to_cop2_data", + reads = { domain = "gpr", arg = 1 }, + writes = { domain = "cop2.data", arg = 2 }, + visibility = { kind = "post_producer_words", required = 2 }, + evidence = { + confidence = "exact", + source = "cpuspecifications.md:407-419", + }, + violation_kind = "error", + }, + -- CPU → COP2 data register when the destination is C2_IRGB (data 28). + -- C2_IRGB drives the IR1/IR2/IR3 color-conversion fan-out, which extends the propagation delay to 3 cached words. + -- `destination_match = "C2_IRGB"` is the row's filter; the analyzer consults this when the producer's destination operand equals "C2_IRGB". + -- C2_ORGB (data 29) is read-only and is never classified as a writable fan-out destination. + { + id = "mtc2_irgb_visibility", + semantic = "MTC2", + token = "gte_mv_to_data_r", + direction = "gpr_to_cop2_data", + reads = { domain = "gpr", arg = 1 }, + writes = { domain = "cop2.data", arg = 2 }, + destination_match = "C2_IRGB", + fanout_to = { "C2_IR1", "C2_IR2", "C2_IR3" }, + visibility = { kind = "post_producer_words", required = 3 }, + evidence = { + confidence = "exact", + source = "cpuspecifications.md:407-419", + }, + violation_kind = "error", + }, + -- CPU → COP2 control register (CTC2). Ordinary minimum 2; + -- no IRGB-style fan-out exists for control registers (per spec §3.6: only C2_IRGB has the 3-cycle fan-out on the data side). + { + id = "ctc2_gpr_visibility", + semantic = "CTC2", + token = "gte_mv_to_ctrl_r", + direction = "gpr_to_cop2_control", + reads = { domain = "gpr", arg = 1 }, + writes = { domain = "cop2.ctrl", arg = 2 }, + visibility = { kind = "post_producer_words", required = 2 }, + evidence = { + confidence = "exact", + source = "cpuspecifications.md:407-419", + }, + violation_kind = "error", + }, + -- COP2 data → GPR (MFC2). One cached slot between the transfer and the first GPR consumer; + -- the GPR is not updated until the instruction AFTER the MFC2 completes (geometrytransformationenginegte.md:29-32). + { + id = "mfc2_gpr_visibility", + semantic = "MFC2", + token = "gte_mv_from_data_r", + direction = "cop2_data_to_gpr", + reads = { domain = "cop2.data", arg = 2 }, + writes = { domain = "gpr", arg = 1 }, + visibility = { kind = "post_producer_words", required = 1 }, + evidence = { + confidence = "exact", + source = "geometrytransformationenginegte.md:29-32", + }, + violation_kind = "error", + }, + -- COP2 control → GPR (CFC2). Same delay as MFC2 (cpuspecifications.md treats the two load-from-COP2 paths symmetrically). + { + id = "cfc2_gpr_visibility", + semantic = "CFC2", + token = "gte_mv_from_ctrl_r", + direction = "cop2_control_to_gpr", + reads = { domain = "cop2.ctrl", arg = 2 }, + writes = { domain = "gpr", arg = 1 }, + visibility = { kind = "post_producer_words", required = 1 }, + evidence = { + confidence = "exact", + source = "cpuspecifications.md:382-419", + }, + violation_kind = "error", + }, + -- COP0 control → GPR (MFC0). + -- One cached slot; the analyzer treats `sys_mov_from_cop0(rt, 12)` (the SR/CU2 transfer) as the same shape as the COP2 load-delay path. + -- The semantic-level SR/CU2 transition models the load delay; + -- SR.CU2 bounded-value propagation is modeled separately). + { + id = "mfc0_gpr_visibility", + semantic = "MFC0", + token = "sys_mov_from_cop0", + direction = "cop0_control_to_gpr", + reads = { domain = "cop0.ctrl", arg = 2 }, + writes = { domain = "gpr", arg = 1 }, + visibility = { kind = "post_producer_words", required = 1 }, + evidence = { + confidence = "exact", + source = "cpuspecifications.md:171-178", + }, + violation_kind = "error", + }, + -- Memory -> COP2 data register (LWC2). + -- The memory-side timing is not measured by the vendored GTE latch experiment, so this relation has no numeric retirement threshold. + -- The forward walker emits one info edge at the first command-input consumer and then clears the pending relation. + { + id = "lwc2_unknown_visibility", + semantic = "LWC2", + token = "gte_lw", + direction = "memory_to_cop2_data", + reads = { domain = "memory", arg = 2 }, + writes = { domain = "cop2.data", arg = 1 }, + visibility = { kind = "unknown_consumer", required = nil }, + evidence = { + confidence = "unknown", + source = "gtepipelinetimings.md:271-274", + }, + violation_kind = "info", + clear_on_consumer = true, + }, + -- COP2 data register -> memory (SWC2). This is a read of C2 state, not a CPU-to-COP2 write. + -- Keep the policy row for direction/provenance, but do not stage it as a later command-input producer. + { + id = "swc2_memory_write", + semantic = "SWC2", + token = "gte_sw", + direction = "cop2_data_to_memory", + reads = { domain = "cop2.data", arg = 1 }, + writes = { domain = "memory", arg = 2 }, + visibility = { kind = "none", required = 0 }, + evidence = { + confidence = "exact", + source = "cpuspecifications.md:79", + }, + violation_kind = "info", + stage = false, + }, + -- MTC0 Status/SR.CU2. The ordinary COP0 store has no general store-delay relation; + -- This row is consumed by the dedicated CU2 transition logic in the same forward walk and is therefore not staged in `pending`. + { + id = "mtc0_cu2_visibility", + semantic = "MTC0", + token = "sys_mov_to_cop0", + direction = "gpr_to_cop0_status", + reads = { domain = "gpr", arg = 1 }, + writes = { domain = "cop0.status", arg = 2 }, + status_register = 12, + visibility = { kind = "post_producer_words", required = 2 }, + evidence = { + confidence = "conservative", + source = "cpuspecifications.md:543,625-628", + }, + violation_kind = "warning", + stage = false, + cu2_transition = true, + }, + } + +-- Bounded Status/SR.CU2 transition policy. +-- The value lattice and the transition consumer both read this immutable row; no second value pass is permitted. +-- The source says the enable/disable transition takes "2 clock cycles or so", so the boundary is conservative rather than exact. +M.CU2_TRANSITION_POLICY = { + status_register = 12, + enable_bit = 0x40000000, + required = 2, + visibility_kind = "post_producer_words", + evidence = { + confidence = "conservative", + source = "cpuspecifications.md:543,625-628", + }, +} + +-- Instruction GPR read/write effects table. +-- +-- Maps every CPU/GTE encoder used in production atoms and the focused transfer-hazard tests to its actual GPR operand effects. +-- The analyzer applies this table to `atom.paths.forward_state.gpr_values`: +-- * a write to a GPR invalidates its constant; +-- * a constant-producing transform re-establishes a constant when its inputs are constant (the lattice for `gpr_values` is closed: +-- `{kind="unknown"}` and `{kind="constant", value=}`). +-- +-- The schema is: +-- reads = {pos1, pos2, ...} -- 1-based argument positions that are GPR reads. +-- writes = {pos1, pos2, ...} -- 1-based argument positions that are GPR writes. +-- The argument positions refer to `word_event.args` (the top-level comma-split args of the emitting token, parsed by `tokenize_body`). +-- Operands that are numeric literals, `0x` hex literals, or `U4`/`S4` type keywords are not GPR operand positions and are not listed. +-- +-- Encoders not listed here are treated as "unknown writers" for any GPR they touch; +-- Wknown writers invalidate `forward_state.gpr_values` for every GPR operand they touch (the analyzer cannot assume the result is a constant). +-- This is deliberately conservative: a row missing for a writer means "we do not know what value the GPR now holds" rather than "the GPR keeps its previous constant". +-- +-- Consumers: +-- * passes/static_analysis.lua::analyze_hardware_relations (forward walker). +-- This table is consumed by the hardware-relation analyzer. +M.INSTRUCTION_GPR_EFFECTS = { + -- CPU ALU with one or two GPR operands. Reads every GPR operand position. + add_ui = { reads = {1, 2}, writes = {1} }, + add_ui_self = { reads = {1}, writes = {1} }, + add_si = { reads = {1, 2}, writes = {1} }, + add_u = { reads = {2, 3}, writes = {1} }, + add_u_self = { reads = {1, 2}, writes = {1} }, + sub_s = { reads = {2, 3}, writes = {1} }, + sub_u = { reads = {2, 3}, writes = {1} }, + and_i = { reads = {1, 2}, writes = {1} }, + and_u = { reads = {2, 3}, writes = {1} }, + or_i = { reads = {1, 2}, writes = {1} }, + or_i_self = { reads = {1}, writes = {1} }, + or_u = { reads = {2, 3}, writes = {1} }, + or_u_self = { reads = {1, 2}, writes = {1} }, + xor_i = { reads = {1, 2}, writes = {1} }, + xor_u = { reads = {2, 3}, writes = {1} }, + slt_s = { reads = {2, 3}, writes = {1} }, + slt_u = { reads = {2, 3}, writes = {1} }, + slt_si = { reads = {1, 2}, writes = {1} }, + slt_ui = { reads = {1, 2}, writes = {1} }, + mult_s = { reads = {1, 2}, writes = {} }, + mult_u = { reads = {1, 2}, writes = {} }, + div_s = { reads = {1, 2}, writes = {} }, + div_u = { reads = {1, 2}, writes = {} }, + -- Shifts: shift_lleft(rd, rt, shamt). rd is destination (write); rt is source (read). + shift_lleft = { reads = {2}, writes = {1} }, + shift_lleft_self = { reads = {1}, writes = {1} }, + shift_lright = { reads = {2}, writes = {1} }, + shift_aright = { reads = {2}, writes = {1} }, + -- Loads: load_word(rt, base, off). rt is destination (write); base is source (read). + load_word = { reads = {2}, writes = {1} }, + load_half_u = { reads = {2}, writes = {1} }, + load_byte_u = { reads = {2}, writes = {1} }, + load_half = { reads = {2}, writes = {1} }, + load_byte = { reads = {2}, writes = {1} }, + -- 2-word loads for > 16-bit immediates. + load_upper_i = { reads = {}, writes = {1} }, + load_ui = { reads = {}, writes = {1} }, + load_imm = { reads = {}, writes = {1} }, + load_imm_1w = { reads = {}, writes = {1} }, + load_imm_1w_s0 = { reads = {}, writes = {1} }, + load_imm_2w = { reads = {}, writes = {1} }, + load_imm_2w_addi_forced = { reads = {}, writes = {1} }, + load_imm_2w_ori_forced = { reads = {}, writes = {1} }, + -- Stores: store_word(base, rt, off). base + rt are both GPR reads. + store_word = { reads = {1, 2}, writes = {} }, + store_half = { reads = {1, 2}, writes = {} }, + store_byte = { reads = {1, 2}, writes = {} }, + -- Branches: branch_equal(rs, rt, label). rs + rt are GPR reads. + branch_equal = { reads = {1, 2}, writes = {} }, + branch_ne = { reads = {1, 2}, writes = {} }, + branch_le_zero = { reads = {1}, writes = {} }, + branch_lt_zero = { reads = {1}, writes = {} }, + branch_ge_zero = { reads = {1}, writes = {} }, + branch_gt_zero = { reads = {1}, writes = {} }, + -- Jumps / link / call: jump_reg(rs) reads rs. RD is the destination link. + jump = { reads = {}, writes = {} }, + jump_reg = { reads = {1}, writes = {} }, + jump_link = { reads = {1}, writes = {2} }, + call_reg = { reads = {1}, writes = {2} }, + call_addr = { reads = {}, writes = {1} }, + -- mask_upper is a 2-word macro: shift_lleft then shift_lright. First reads rt. + mask_upper = { reads = {1, 2}, writes = {1} }, + -- move from/to HI/LO. + mov_from_high = { reads = {}, writes = {1} }, + mov_from_low = { reads = {}, writes = {1} }, + mov_to_high = { reads = {1}, writes = {} }, + mov_to_low = { reads = {1}, writes = {} }, + -- Set-on-condition (SLT family). + set_lt_u = { reads = {2, 3}, writes = {1} }, + set_lt_ui = { reads = {1, 2}, writes = {1} }, + set_lt_s = { reads = {2, 3}, writes = {1} }, + set_lt_si = { reads = {1, 2}, writes = {1} }, + -- COP2 transfers: gte_mv_*_r(rt, c2reg). + -- to_data_r / to_ctrl_r: rt is the GPR source (read); c2reg is the COP2 destination (hardware, not a GPR). + -- from_data_r / from_ctrl_r: rt is the GPR destination (write); c2reg is the COP2 source (hardware, not a GPR). + gte_mv_to_data_r = { reads = {1}, writes = {} }, + gte_mv_to_ctrl_r = { reads = {1}, writes = {} }, + gte_mv_from_data_r = { reads = {}, writes = {1} }, + gte_mv_from_ctrl_r = { reads = {}, writes = {1} }, + -- COP2 lw/sw: gte_lw(c2reg, base, off) / gte_sw(c2reg, base, off). base is GPR source; c2reg is COP2 hardware. + -- LWC2 uses an unknown dependency edge; the GPR effects are unchanged. + gte_lw = { reads = {2}, writes = {} }, + gte_sw = { reads = {2}, writes = {} }, + -- COP0 transfers: sys_mov_from_cop0(rt, creg) / sys_mov_to_cop0(rt, creg). + -- from_cop0: rt is the GPR destination (write); creg is the COP0 source. + -- to_cop0: rt is the GPR source (read); creg is the COP0 destination. + -- The SR.CU2 transition uses bounded-value rules. + sys_mov_from_cop0 = { reads = {}, writes = {1} }, + sys_mov_to_cop0 = { reads = {1}, writes = {} }, + -- GTE commands / aliases: the encoder is atomic from the CPU's POV once it + -- issues (the CPU holds until the command completes). No GPR reads/writes. + gte_cmdw_rtps = { reads = {}, writes = {} }, + gte_cmdw_rtpt = { reads = {}, writes = {} }, + gte_cmdw_nclip = { reads = {}, writes = {} }, + gte_cmdw_avsz3 = { reads = {}, writes = {} }, + gte_cmdw_avsz4 = { reads = {}, writes = {} }, + gte_cmdw_mvmva = { reads = {}, writes = {} }, + gte_cmdw_op = { reads = {}, writes = {} }, + -- High-level GTE helpers (CPU-side load/store wrappers around gte_lw/gte_sw). + gte_stotz = { reads = {}, writes = {} }, + gte_stsxy3 = { reads = {}, writes = {} }, + gte_load_v0 = { reads = {2}, writes = {} }, + gte_load_v1 = { reads = {2}, writes = {} }, + gte_load_v2 = { reads = {2}, writes = {} }, + gte_load_v0v1v2 = { reads = {2}, writes = {} }, + -- nop / nop2: zero GPR effects (nop2 = two nop halves in emission-model). + nop = { reads = {}, writes = {} }, + nop2 = { reads = {}, writes = {} }, + -- Annotation markers: zero GPR effects; pure metaprogram hints. + atom_label = { reads = {}, writes = {} }, + atom_offset = { reads = {}, writes = {} }, + atom_info = { reads = {}, writes = {} }, + atom_bind = { reads = {}, writes = {} }, + atom_reads = { reads = {}, writes = {} }, + atom_writes = { reads = {}, writes = {} }, + -- mac_yield transfers control to the next atom; zero GPR effects. + mac_yield = { reads = {}, writes = {} }, +} + +-- Bounded GPR-value rules consumed by the same forward event walk as `INSTRUCTION_GPR_EFFECTS`. +-- A rule describes a literal/constant-producing transform; if its required inputs are not constant, the destination is invalidated rather than carrying a stale value. +-- The lattice is deliberately closed to `{kind = "unknown"}` and `{kind = "constant", value = }`. +-- +-- Consumers: +-- * passes/static_analysis.lua::apply_gpr_effects +-- No second `bounded_value_pass` is permitted. +M.GPR_VALUE_RULES = { + load_upper_i = { op = "load_upper_i", dest = 1, immediate = 2, }, + add_ui = { op = "add_ui", dest = 1, source = 2, immediate = 3, }, + or_i = { op = "or_i", dest = 1, source = 2, immediate = 3, }, + and_i = { op = "and_i", dest = 1, source = 2, immediate = 3, }, + xor_i = { op = "xor_i", dest = 1, source = 2, immediate = 3, }, + add_ui_self = { op = "add_ui", dest = 1, source = 1, immediate = 2, }, + or_i_self = { op = "or_i", dest = 1, source = 1, immediate = 2, }, + -- Present register-form self variants. They are included here so a + -- known value is not needlessly lost when these encoders are used. + add_u_self = { op = "add_u", dest = 1, sources = {1, 2}, }, + or_u_self = { op = "or_u", dest = 1, sources = {1, 2}, }, + shift_lleft_self = { op = "shift_lleft", dest = 1, source = 1, immediate = 2, }, +} + -- Control-transfer (branch/jump/call) delay-slot policy table. -- --- Used by the emitted-word delay-slot check to identify which emitted machine-word idents --- are control transfers whose next emitted word is the hardware delay slot. +-- Used by the emitted-word delay-slot check to identify which emitted machine-word idents are control transfers whose next emitted word is the hardware delay slot. -- One table row per emitted encoder; the `family` field is informational (informational only; --- the check matches by `event.ident` against the row keys). +-- The check matches by `event.ident` against the row keys). -- `suppress_arg1` (when present) lists first-arg values that should NOT emit a finding even when the next emitted word is -- `nop` or absent — e.g. the fixed `mac_yield()` handshake uses `jump_reg(R_AtomJmp), nop` and is intentionally suppressed. -- @@ -1197,49 +2096,10 @@ M.CONTROL_TRANSFER_DELAY_SLOT_POLICIES = { --- @field source string -- path of the source containing the offending token --- @field line integer -- 1-based line of the offending token within `source` ---- Build (and memoize) the cross-source component-body index keyed by the BARE component name (`gte_load_tri_verts`, NOT `ac_gte_load_tri_verts`). ---- ---- Components are declared in one source (the header holding `MipsAtomComp_(ac_X)` or `MipsAtomComp_Proc_(ac_X, { ... })`) but invoked from any source that calls `mac_X(...)`. ---- Body-offsets + body_tokens + line_of live with the declaration, so a per-source index misses invocations from other sources ---- (this is the bug class that motivated moving the index to duffle). ---- ---- Only `comp_bare` / `comp_proc` declarations contribute (a `mac_X(...)` invocation can only resolve to one of those). ---- First declaration wins; subsequent redeclarations would collide, but today's sources declare each component exactly once. ---- ---- The memoized table is stored at `ctx.shared.component_body_index` so callers can detect "already built" without re-scanning every source. Idempotent: ---- safe to call from multiple passes within the same build. ---- @param ctx table -- the PassCtx; reads `ctx.sources` + writes `ctx.shared.component_body_index` ---- @return table -function M.get_component_body_index(ctx) - local shared = ctx.shared or {} - if shared.component_body_index ~= nil then return shared.component_body_index end - local index = {} - for _, src in ipairs(ctx.sources or {}) do - if src.scan and src.scan.atoms then - local line_of = src.scan.line_of - for _, atom in ipairs(src.scan.atoms) do - if atom.kind == "comp_bare" or atom.kind == "comp_proc" then - -- Prefer `atom.name` (the bare identifier, stripped of `ac_`); fall back to `raw_name` - -- only if the stripped name is absent (defensive — current scan-source always sets both). - local name = atom.name or atom.raw_name - if name and not index[name] then - index[name] = { - body_tokens = atom.body_tokens, - body_off = atom.body_off, - line_of = line_of, - source = src.path, - declaration = atom.line, - kind = atom.kind, - } - end - end - end - end - end - shared.component_body_index = index - ctx.shared = shared - return index -end +-- The cross-source component-body index is owned by the canonical corpus +-- (`corpus.component_body_index`, populated by `passes/components.lua`). +-- Consumers (`passes/static_analysis.lua`, `passes/emission_model.lua`) read it directly; +-- No per-pass memoization helper is needed. -- ASCII byte constants used by split_top_level_args (kept local to keep Section 8 self-contained). local E_BYTE_OPEN_PAREN = 0x28 @@ -1256,12 +2116,12 @@ local E_OPEN_CLOSE = { [E_BYTE_OPEN_BRACK] = "]", } --- Split the INSIDE of a `f(...)` call on top-level commas. --- Honors nested parens / braces / brackets and skips strings / comments. --- Returns a list of trimmed argument strings in source order. --- (Mirrors split_top_level_commas but for paren-body args; intentionally distinct so a caller's brace-body split isn't confused with an arg list.) --- @param inner string --- @return string[] +--- Split the INSIDE of a `f(...)` call on top-level commas. +--- Honors nested parens / braces / brackets and skips strings / comments. +--- Returns a list of trimmed argument strings in source order. +--- (Mirrors split_top_level_commas but for paren-body args; intentionally distinct so a caller's brace-body split isn't confused with an arg list.) +--- @param inner string +--- @return string[] local function split_top_level_args(inner) local args = {} if not inner or inner == "" then return args end @@ -1288,10 +2148,10 @@ local function split_top_level_args(inner) return args end --- Extract the leading identifier + top-level args list from a token string. --- Returns (ident, args). For tokens without a `(...)` call, args is `{}`. --- @param tok string --- @return string, string[] +--- Extract the leading identifier + top-level args list from a token string. +--- Returns (ident, args). For tokens without a `(...)` call, args is `{}`. +--- @param tok string +--- @return string, string[] local function token_ident_and_args(tok) local ident, after = M.read_ident(tok, 1) if not ident then return "?", {} end @@ -1325,6 +2185,429 @@ local E_MAC_PREFIX_LEN = 4 --- @param component_index table -- the bare-name → ComponentBodyEntry map from M.get_component_body_index --- @param word_counts table -- macro name → emitted-word count (from `ctx.shared.word_counts`) --- @return WordEvent[], WordEventError[] + +-- ════════════════════════════════════════════════════════════════════════════ +-- Section 11: project_emission (canonical per-atom emission projection) +-- ════════════════════════════════════════════════════════════════════════════ +-- +-- Canonical per-atom emission projection is owned by `passes/emission_model.lua`. +-- The projection is built from the root atom body only (no nested component expansion at this stage); +-- Invocation ancestry recursively expands nested components. +-- The items stream is the single ordered source of truth; `word_events` and `markers` are dense views over it (never a separate walk). +-- +-- The helper below operates on a body string (not a body_entry) so the canonical pass can call it without depending on the older SourceScan / body_off conventions. +-- component_index argument is accepted for parity with `expand_word_events` and is reserved for recursive component expansion. +-- word_counts table is the canonical authored-metadata + current-component count table. + +--- @class EmissionProjection +--- @field items table[] -- ordered stream of word|label|offset|invoke_begin|invoke_end +--- @field word_events table[] -- dense view of items where kind == "word" +--- @field markers table[] -- dense view of items where kind == "label"|"offset" +--- @field invocations table[] -- dense view of items where kind == "invoke_begin"|"invoke_end" +--- @field errors table[] -- token-resolution failures surfaced without fail-loud +--- @field warnings table[] -- opaque warnings (e.g. unknown uncounted macro) + +-- Internal recursive walker. Single source of truth for the items stream; +-- `word_events`, `markers`, `invocations`, `errors`, `warnings` are dense views / side outputs derived while appending `items`. +-- +-- Output rules: +-- * `word` items record: `invocation_ids` (innermost last) and `outermost_invocation_id` (0 if no invocation is open). +-- * `invoke_begin` / `invoke_end` items are zero-width at the current word index; the same `word_index` is recorded on both. +-- * `root_call_text` is the outermost `mac_X(...)` token text for every word emitted inside a component expansion; +-- it is `nil` for direct words emitted from the root atom body. +-- * `call_text` is the IMMEDIATE top-level token spelling for the word (for nested words this is the inner `mac_X(...)` token; +-- for direct words it is the trimmed encoder token). +-- * `def_path` / `def_line` are the definition site of the current body (component source for nested words; root atom source for direct words, filled in by the pass caller). +-- * Unknown uncounted macros emit one opaque word + one warning. Unknown metadata-backed macros (entry in `word_counts`) emit the declared word count, no warning. +-- * Cycle detection uses an active DFS stack (`visiting`); a cycle appends a construction error to BOTH the projection errors and the cycle invocation's own errors, +-- then breaks out without recursing (the cycle entry still receives an invocation ID + paired `invoke_begin` / `invoke_end` items, so the boundary invariant is preserved). +-- * Component declared-count mismatch (declared vs. measured) is a construction error (kind = "count_mismatch"); it is recorded on the invocation record and the pass-level errors list. +-- * Final boundary check: if any invocation is still open at end of walk, surface a "unbalanced" construction error. +local function _project_emission_inner(root_body_entry, ctx_table) + local items = {} + local word_events = {} + local markers = {} + local invocations = {} + local errors = {} + local warnings = {} + + local word_idx = 0 + local invocation_stack = {} -- stack of currently-open invocation records + local next_inv_id = 0 + + local function open_invocation_ids_snapshot() + local ids = {} + for _, inv in ipairs(invocation_stack) do + ids[#ids + 1] = inv.id + end + return ids + end + + local function emit_word(encoder, args, line, word_call_text, + def_source_now, def_line_now, + immediate_call_text, root_call_text_w) + local inv_ids = open_invocation_ids_snapshot() + local outermost = inv_ids[1] or 0 + -- For words emitted at the root atom body, `immediate_call_text` is nil and the walker's `word_call_text` (the word's own token, e.g. "nop") becomes the effective call_text. + -- For words emitted inside a component expansion, `immediate_call_text` is the immediate outer `mac_X(...)` token text; + -- The call that triggered the body expansion we're currently walking. + local eff_call_text = immediate_call_text or word_call_text + local eff_root_call_text = root_call_text_w + items[#items + 1] = { + kind = "word", + encoder = encoder, + args = args, + i = word_idx, + word_count = 1, + line = line, + call_text = eff_call_text, + root_call_text = eff_root_call_text, + invocation_ids = inv_ids, + outermost_invocation_id = outermost, + } + word_events[#word_events + 1] = { + i = word_idx, + encoder = encoder, + args = args, + def_path = def_source_now or "", + def_line = def_line_now or 0, + call_text = eff_call_text, + root_call_text = eff_root_call_text, + invocation_ids = inv_ids, + outermost_invocation_id = outermost, + word_count = 1, + } + word_idx = word_idx + 1 + end + + local function emit_marker(kind, name, target, line, + immediate_call_text, root_call_text_w) + local inv_ids = open_invocation_ids_snapshot() + local outermost = inv_ids[1] or 0 + -- Markers carry the open invocation stack snapshot but do NOT record `call_text` / `root_call_text` — + -- markers are zero-width and never participate in the per-word call-site attribution. + local it = { + kind = kind, + name = name, + line = line, + word_index = word_idx, + invocation_ids = inv_ids, + outermost_invocation_id = outermost, + } + if target ~= nil then it.target = target end + items[#items + 1] = it + markers[#markers + 1] = { + kind = kind, + name = name, + line = line, + word_index = word_idx, + target = target, + } + end + + local function emit_embedded_markers(tok, tok_line) + local pos = 1 + while pos <= #tok do + pos = M.skip_ws_and_cmt(tok, pos) + if pos > #tok then break end + local ident, after = M.read_ident(tok, pos) + if ident then + if ident == "atom_label" or ident == "atom_offset" then + local open = M.skip_ws_and_cmt(tok, after) + local inner, after_paren = M.read_parens(tok, open) + if inner then + local args = split_top_level_args(inner) + if ident == "atom_label" then + emit_marker("label", args[1] or "", nil, tok_line) + else + emit_marker("offset", args[1] or "", args[2] or "", tok_line) + end + pos = after_paren + else + pos = after + end + else + pos = after + end + else + local next_pos = M.skip_str_or_cmt(tok, pos) + pos = (next_pos > pos) and next_pos or (pos + 1) + end + end + end + + local function emit_invoke_begin(inv_kind, component_name, call_text, + root_call_text, call_path, call_line) + next_inv_id = next_inv_id + 1 + local inv = { + id = next_inv_id, + parent_id = 0, -- patched below by caller + kind = inv_kind, + component_name = component_name, + call_text = call_text, + root_call_text = root_call_text, + call_path = call_path, + call_line = call_line, + def_path = nil, -- patched below after component lookup + def_line = nil, + start_word = #items + 1, -- 1-based items index of invoke_begin + end_word = nil, -- patched by emit_invoke_end + word_count = 0, + errors = {}, + } + invocations[#invocations + 1] = inv + items[#items + 1] = { + kind = "invoke_begin", + invocation_id = inv.id, + word_index = word_idx, + invocation_ids = open_invocation_ids_snapshot(), + } + invocation_stack[#invocation_stack + 1] = inv + return inv + end + + local function emit_invoke_end(inv) + inv.end_word = #items + 1 -- 1-based items index of invoke_end + items[#items + 1] = { + kind = "invoke_end", + invocation_id = inv.id, + word_index = word_idx, + invocation_ids = open_invocation_ids_snapshot(), + } + for i = #invocation_stack, 1, -1 do + if invocation_stack[i] == inv then + table.remove(invocation_stack, i) + break + end + end + end + + -- Resolve the per-token word count. If unresolved, surface ONE warning + -- (NOT an error; the build does not fail-loud on an uncounted opaque word) + -- and fall back to 1 opaque word so the cycle budget still accounts for the slot. + local function resolve_count(ident, tok_line) + local wc = ctx_table.word_counts + if wc and wc[ident] then return wc[ident] end + if M.GTE_COMMAND_ALIASES then + local seen = { [ident] = true } + local target = M.GTE_COMMAND_ALIASES[ident] + while target and not seen[target] do + seen[target] = true + if wc and wc[target] then return wc[target] end + target = M.GTE_COMMAND_ALIASES[target] + end + end + warnings[#warnings + 1] = { + kind = "uncounted", + line = tok_line, + msg = string.format("project_emission: opaque word emitted for %q (no entry in word_counts or component_index)", + ident), + } + return 1 + end + + -- Recursive walker: walk one body entry, possibly descending into components. + -- `walk_parent_inv_id` is the invocation ID of the enclosing call (0 for the root call). + -- `walk_root_call_text` is the outermost `mac_X(...)` token text (preserved across recursion). + -- `walk_immediate_call_text` is the IMMEDIATE outer `mac_X(...)` token text for words emitted in this body — nil for the root atom body. + -- The two trackers are propagated as separate parameters so words deep inside nested expansions correctly identify both their immediate call site and the outermost call site. + local function walk_body_entry(body_entry, walk_parent_inv_id, + walk_root_call_text, walk_immediate_call_text) + local tokens = body_entry.body_tokens or {} + local body_off = body_entry.body_off or 0 + local line_of = body_entry.line_of or M.LineIndex("") + local def_source = body_entry.source or "" + local def_line = body_entry.declaration or 0 + + for _, bt in ipairs(tokens) do + local tok = M.trim(bt.tok or "") + if tok ~= "" then + local ident = M.read_ident(tok, 1) or "?" + local _, args = token_ident_and_args(tok) + local tok_line = line_of(body_off + bt.rel) or 0 + + if ident ~= "atom_label" and ident ~= "atom_offset" then emit_embedded_markers(tok, tok_line) end + + if ident == "atom_label" then emit_marker("label", args[1] or "", nil, tok_line) + elseif ident == "atom_offset" then emit_marker("offset", args[1] or "", args[2] or "", tok_line) + elseif ident:sub(1, 4) == "mac_" then + local bare = ident:sub(5) + local comp = ctx_table.component_index[bare] + if comp then + local invocation_root_call_text = walk_root_call_text or tok + if ctx_table.visiting[bare] then + -- Cycle: this component is already on the expansion stack. + -- Still allocate an ID, still emit invoke_begin / invoke_end (zero-width), still record the cycle error — but do NOT recurse. + local inv = emit_invoke_begin(comp.kind or "comp_bare", bare, tok, invocation_root_call_text, def_source, tok_line) + inv.parent_id = walk_parent_inv_id + inv.call_text = tok + local err = { + kind = "cycle", + msg = string.format( + "project_emission: component cycle detected: %q", bare), + source = def_source, + line = tok_line, + } + inv.errors[#inv.errors + 1] = err + errors[#errors + 1] = err + emit_invoke_end(inv) + else + ctx_table.visiting[bare] = true + local inv = emit_invoke_begin(comp.kind or "comp_bare", bare, tok, invocation_root_call_text, def_source, tok_line) + inv.parent_id = walk_parent_inv_id + inv.call_text = tok + inv.def_path = comp.source + inv.def_line = comp.declaration + -- Propagate trackers into the component body: + -- * `immediate_call_text` for the recursive walk is THIS call's token text (the immediate outer call for any word emitted inside this body). + -- * `root_call_text` is the OUTERMOST call. Immutable value computed before this invocation was opened. + local inner_immediate_call_text = tok + walk_body_entry({ + body_tokens = comp.body_tokens or {}, + body_off = comp.body_off or 0, + line_of = comp.line_of, + source = comp.source, + declaration = comp.declaration, + } + , inv.id + , invocation_root_call_text + , inner_immediate_call_text + ) + ctx_table.visiting[bare] = nil + emit_invoke_end(inv) + + -- Count `word` items inside [start_word, end_word]. + local wc_inside = 0 + for i = inv.start_word, inv.end_word do + local it = items[i] + if it and it.kind == "word" then + wc_inside = wc_inside + 1 + end + end + inv.word_count = wc_inside + + -- Declared-vs-measured mismatch is a construction error. + -- `word_counts["mac_X"]` is the declared count populated by the components pass; + -- we compare against the measured word count. + local declared = ctx_table.word_counts["mac_" .. bare] + if declared and wc_inside ~= declared then + local err = { + kind = "count_mismatch", + msg = string.format( + "project_emission: mac_%s declared=%d measured=%d", + bare, declared, wc_inside), + source = def_source, + line = tok_line, + } + inv.errors[#inv.errors + 1] = err + errors [#errors + 1] = err + end + end + else + -- mac_X is NOT a known component. Resolve the count from `word_counts`; if unresolved, emit 1 opaque word + one warning. + local n = resolve_count(ident, tok_line) + local out_ident = (ident == "nop2") and "nop" or ident + for _ = 1, n do + emit_word(out_ident, args, tok_line, tok, def_source, def_line, walk_immediate_call_text, walk_root_call_text) + end + end + else + -- Direct encoder (non-mac_X). Resolve count; emit N words. + local n = resolve_count(ident, tok_line) + local out_ident = (ident == "nop2") and "nop" or ident + for _ = 1, n do + emit_word(out_ident, args, tok_line, tok, def_source, def_line, walk_immediate_call_text, walk_root_call_text) + end + end + end + end + end + + -- Initialize the per-walk mutable context. + -- `visiting` is the active DFS component stack; `root_call_path` / `root_call_line` are preserved across recursion so nested words always point at the + -- ORIGINAL root atom call site. + ctx_table.visiting = ctx_table.visiting or {} + ctx_table.root_call_path = ctx_table.root_call_path or "" + ctx_table.root_call_line = ctx_table.root_call_line or 0 + + -- Walk first; the pass caller stamps the root call site for direct words after the projection returns. + -- For nested words the def_path / def_line already point at the component source and MUST be preserved (the stamping helper checks for that). + walk_body_entry(root_body_entry, 0, nil, nil) + + -- Boundary check: every invoke_begin must have a matching invoke_end. + -- If anything is still open, surface a hard error. + if #invocation_stack > 0 then + errors[#errors + 1] = { + kind = "unbalanced", + msg = string.format("project_emission: invocation boundaries not balanced (%d unclosed invocation(s) at end of walk)", #invocation_stack), + } + end + + return { + items = items, + word_events = word_events, + markers = markers, + invocations = invocations, + errors = errors, + warnings = warnings, + } +end + +--- Project a body string into the canonical per-atom emission projection. +--- +--- Semantics: +--- * Direct one-word tokens (`nop`, `add_ui`, ...): one `word` item, encoder = ident, word_count = 1. +--- * Metadata-backed N-word tokens (`nop2`, `mask_upper`, ...): N `word` items, all sharing the same encoder + word_count = 1. +--- `nop2` is normalized to encoder `nop` (per the spec). +--- * `atom_label(F)` markers: one `label` item with `name = "F"`, `word_index = current word_idx`; zero-width (does NOT advance word_idx). +--- * `atom_offset(B, T)` markers: one `offset` item with `name = "B"`, `target = "T"`, `word_index = current word_idx`; zero-width. +--- * `mac_X(...)` calls: emit `invoke_begin` (zero-width), recurse into the component body, emit `invoke_end` (zero-width). +--- The component body's words land between the begin/end pair; one invocation record is allocated per call (monotonic ID per atom). +--- * Unknown uncounted macros emit 1 opaque word + one warning per occurrence. +--- * Tokens whose count cannot be resolved (e.g. `mac_unknown` not in word_counts and not in component_index) surface one +--- warning; cycle + count-mismatch + boundary violations are construction errors on `pass.errors`. +--- +--- Every emitted `word` carries: `i` (0-based word index), `encoder`, `args` (top-level args), `def_path`, `def_line`, +--- `call_text` (the immediate token spelling), `root_call_text` (outermost `mac_X(...)` text), `word_count` (always 1), +--- `invocation_ids` (innermost last), `outermost_invocation_id`. +--- Markers carry: `kind`, `name`, `line`, `word_index`, `target` (only for offset kind), plus `invocation_ids` / `outermost_invocation_id` for the open invocation stack at that word. +--- +--- @param body_text string -- the raw atom body string +--- @param component_index table -- bare-name → component record (corpus.component_body_index) +--- @param word_counts table -- macro name → emitted word count +--- @return EmissionProjection +function M.project_emission(body_text, component_index, word_counts) + -- Project nested invocation ancestry and construction failures. + -- The public surface remains `M.project_emission(body_text, ...)`; + -- the recursive walk is delegated to `_project_emission_inner` so that component bodies (which arrive as `{body_tokens, body_off, + -- line_of, source, declaration}` records from `corpus.component_body_index`) re-enter the same walker with the same shared output state. + + if type(body_text) ~= "string" or body_text == "" then + -- Empty body: still return a valid (empty) projection. + return { + items = {}, + word_events = {}, + markers = {}, + invocations = {}, + errors = {}, + warnings = {}, + } + end + + local tokens = M.tokenize_body(body_text) + local line_of = M.LineIndex(body_text) + return _project_emission_inner({ + body_tokens = tokens, + body_off = 0, + line_of = line_of, + source = "", + declaration = 0, + }, { + component_index = component_index or {}, + word_counts = word_counts or {}, + }) +end + function M.expand_word_events(body_entry, component_index, word_counts) local events = {} local errors = {} @@ -1394,6 +2677,7 @@ function M.expand_word_events(body_entry, component_index, word_counts) end -- Initial call: the root atom body. `def_source` and `call_source` both start at the atom's source; + -- `call_line` starts at the atom's declaration line (every event from the root body inherits this). expand( body_entry.body_tokens, diff --git a/scripts/passes/annotation.lua b/scripts/passes/annotation.lua index 2b6ce92..b407e84 100644 --- a/scripts/passes/annotation.lua +++ b/scripts/passes/annotation.lua @@ -21,7 +21,7 @@ local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua") local write_file = duffle.write_file local ensure_dir = duffle.ensure_dir --- The annotation pass now consults the source-derived registries built by scan_source: +-- The annotation pass reads the source-derived registries from scan_source: -- * pipe_ctx.register_alias_registry — for atom_dbg_reg_default(R_X, ...) and atom_reg_types(R_X, ...) member-identity checks -- * pipe_ctx.type_name_registry — for atom_dbg_reg_default(, ...) and atom_reg_types(, ...) type-identity checks @@ -108,12 +108,11 @@ local ensure_dir = duffle.ensure_dir -- -- Each check has a uniform `append_to_findings` shape (errors[] / warnings[] / info[]). -- The dispatcher in `validate()` decides which findings list each check writes to — by convention, --- "existence" checks (declaration must exist, struct must exist) write errors[]; "shape" checks --- (writes/reads must be wave-context) write warnings[]. +-- "existence" checks (declaration must exist, struct must exist) write errors[]; "shape" checks (writes/reads must be wave-context) write warnings[]. -- The `macro_word_drift` check writes both errors[] (missing/mismatch) and info[] (match). --- Check: every annotated atom must have a matching MipsAtom_(name) declaration. ---- @param a AtomAnnotation +--- @param a AtomAnnotation --- @param pipe_ctx PipeCtx --- @param findings Findings local function check_atom_decl_exists(a, pipe_ctx, findings) @@ -144,7 +143,7 @@ end --- Emitting a warning here keeps the annotation pass from being stop-on-error for the common test-fixture case, --- while still surfacing the issue in the report. --- The static-analysis report remains the source of truth for build-stopping errors. ---- @param a AtomAnnotation +--- @param a AtomAnnotation --- @param pipe_ctx PipeCtx --- @param findings Findings local function check_binds_struct_exists(a, pipe_ctx, findings) @@ -160,7 +159,7 @@ end --- Check: TAPE_WORDS(mac_X, N) ↔ WORD_COUNT(mac_X, N) drift. --- Three outcomes: missing (error), mismatch (error), match (info). ---- @param m MacroEntry +--- @param m MacroEntry --- @param wc table -- the shared word-count table (from ctx.shared.word_counts) --- @param findings Findings local function check_macro_word_drift(m, wc, findings) @@ -188,7 +187,7 @@ end --- Check: atom_dbg_reg_default(R_X, ) must target a register declared as a debug-visible alias in `pipe_ctx.register_alias_registry`, --- with a type name found in `pipe_ctx.type_name_registry`. --- Pointer depth is still bounded to 0 or 1. Duplicate defaults are still detected. ---- @param _src SourceFile -- unused (kept for the per_source shape) +--- @param _src SourceFile -- unused (kept for the per_source shape) --- @param pipe_ctx PipeCtx --- @param findings Findings local function check_semantic_reg_defaults(_src, pipe_ctx, findings) @@ -240,7 +239,7 @@ end --- The alias ident `R_` now encodes the GPR identity only for entries that are explicitly opted in via the bare `atom_reg` marker. --- R_T0..R_T3 are intentionally NOT auto-included (per the prototype principle: no auto-include of wave-context; explicit opt-in only). --- The check fires for any R_T0..R_T3 reference that hasn't been opted in via `#define atom_reg`. ---- @param _src SourceFile +--- @param _src SourceFile --- @param pipe_ctx PipeCtx --- @param findings Findings local function check_atom_reg_types(_src, pipe_ctx, findings) @@ -271,7 +270,7 @@ local function check_atom_reg_types(_src, pipe_ctx, findings) end --- Check: atom_view(Binds_X) entries must reference a real Binds_* struct and that struct must declare at least one field. ---- @param _src SourceFile +--- @param _src SourceFile --- @param pipe_ctx PipeCtx --- @param findings Findings local function check_atom_view_layout(_src, pipe_ctx, findings) @@ -385,14 +384,14 @@ local function check_skip_marker(marker, _pipe_ctx, findings) end end ---- Migration warning emitted alongside the new registry-membership check. +--- Warn when a source references an unregistered alias. --- --- R_TapePtr / R_AtomJmp / R_PrimCursor / R_FaceCursor / R_VertBase / R_OtBase are the context aliases opted in via `#define atom_reg` in lottes_tape.h. ---- Any source referencing an R_X that's NOT in the registry will trip the new check; a single pass-level info entry +--- A source referencing an unregistered R_X emits one pass-level info entry --- (emitted only when at least one such rejection lands in this source) tells users where to look. --- ---- This check is a stop-gap until users migrate off raw C-ABI register names. ---- @param _src SourceFile +--- This check directs raw C-ABI register names to explicit alias registration. +--- @param _src SourceFile --- @param pipe_ctx PipeCtx --- @param findings Findings local function check_wave_context_migration(_src, pipe_ctx, findings) @@ -445,14 +444,65 @@ local CHECK_RULES = { -- Validation -- ════════════════════════════════════════════════════════════════════════════ -- --- Pure check: read from src.scan, run validations, emit findings. --- No source walking; no parsing. The scan was done once upstream. +-- Pure check: read from src.scan, run validations, emit findings. The scan was done once upstream. ---- Validate one source against its pre-scanned SourceScan payload. +--- Build the corpus-wide pipe_ctx ONCE per pass run. +--- Reads the merged `corpus.*` registries (canonical cross-source lookups), +--- and the corpus-wide `atom_infos` list (preserving source order + duplicates). +--- The corpus is the source of truth; per-source scans retain body / declaration +--- ownership via `src.scan` and the per-source `atoms` / `atom_infos` projections. +--- +--- Canonical ownership: a context without `ctx.shared.corpus` is rejected with an explicit canonical-corpus message. +--- No per-source fallback synthesis is performed; callers MUST construct a canonical ctx through `build_ctx`. --- @param ctx PassCtx ---- @param src SourceFile +--- @return PipeCtx +local function build_corpus_pipe_ctx(ctx) + local corpus = ctx.shared and ctx.shared.corpus + if not corpus then + error("annotation requires ctx.shared.corpus " + .. "(the canonical corpus is the source of truth; " + .. "no per-source fallback is supported)", 0) + end + + -- Corpus atom_infos preserves source-order + duplicates; + -- the per-check `check_unique_annotation` post-rule still flags duplicate annotation + -- names within this list. We pre-compute the annot_counts map here so the per_source checks can iterate it without re-walking. + local annot_counts = {} + for _, info in ipairs(corpus.atom_infos or {}) do + if info and info.atom_name then + annot_counts[info.atom_name] = (annot_counts[info.atom_name] or 0) + 1 + end + end + + -- The pipe_ctx views REFERENCE the corpus tables directly (no copies). + -- Every consumer of these fields observes mutations via the canonical corpus without independently mutable registry construction. + return { + -- Cross-source lookup tables (canonical corpus projections). + register_alias_registry = corpus.register_alias_registry or {}, + type_name_registry = corpus.type_name_registry or {}, + atom_views = corpus.atom_views or {}, + atom_ctxs = corpus.atom_ctxs or {}, + atom_phases = corpus.atom_phases or {}, + binds_by_name = corpus.binds_by_name or {}, + atoms_by_name = corpus.atoms_by_name or {}, + -- Corpus-wide ordered list of atom_info records (source-order + duplicates). + atom_infos_list = corpus.atom_infos or {}, + -- Corpus-wide annotation count aggregation (post-rule consumes this). + annot_counts = annot_counts, + -- Corpus-wide collisions (recorded by scan_source.merge_corpus_registries). + collisions = corpus.collisions or {}, + -- wc still consumed by check_macro_word_drift; reads from the canonical + -- `corpus.word_counts` table (built by word_count_eval.run). + word_counts = corpus.word_counts or {}, + } +end + +--- Validate one source against its pre-scanned SourceScan payload + the corpus-wide pipe_ctx. +--- @param ctx PassCtx +--- @param src SourceFile +--- @param corpus_pipe_ctx PipeCtx -- built once per pass from corpus registries --- @return AnnotatedResult -local function validate(ctx, src) +local function validate(ctx, src, corpus_pipe_ctx) local scan = src.scan -- Project the pre-scanned atoms to the AtomEntry shape this pass needs. @@ -478,9 +528,11 @@ local function validate(ctx, src) } end - -- Build pipe_ctx (Fleury: expose structure). Pre-compute everything the per-check functions need. - -- Single source of truth for atom / binds / annotation-count lookups. - -- pipe_ctx.types / pipe_ctx.atom_views / pipe_ctx.seen_defaults are projected from the scan payload so per_source check rules can iterate. + -- Build the per-source pipe_ctx (Fleury: expose structure). + -- Cross-source visibility comes from `corpus_pipe_ctx`; + -- per-source declaration / body ownership comes from `src.scan`. + -- pipe_ctx.types / pipe_ctx.atom_views / pipe_ctx.seen_defaults / pipe_ctx.type_occurrences + -- are projected from the per-source scan so the per_source check rules can iterate the source-local occurrences. local seen_defaults = {} for reg, _ in pairs(scan.types or {}) do seen_defaults[reg] = (seen_defaults[reg] or 0) + 1 @@ -493,25 +545,20 @@ local function validate(ctx, src) local pipe_ctx = { atom_index = {}, binds_index = {}, - annot_counts = {}, + annot_counts = corpus_pipe_ctx.annot_counts, types = scan.types or {}, type_occurrences = scan.type_occurrences or {}, atom_views = scan.atom_views or {}, seen_defaults = seen_defaults, atom_infos_list = atom_infos_list, binds_list = scan.binds or {}, - -- Project the source-derived registries from the scan payload so per_source checks consult them instead of the deleted - -- SEMANTIC_DEFAULT_REGS / KNOWN_REG_DEFAULT_TYPES / etc. - register_alias_registry = scan.register_alias_registry or {}, - type_name_registry = scan.type_name_registry or {}, + -- Source-derived registries: still populated from the scan payload as a convenience for callers that want source-local visibility. + -- The canonical cross-source lookup tables live in corpus_pipe_ctx. + register_alias_registry = corpus_pipe_ctx.register_alias_registry, + type_name_registry = corpus_pipe_ctx.type_name_registry, } for _, a in ipairs(atoms) do pipe_ctx.atom_index [a.name] = a end for _, b in ipairs(scan.binds) do pipe_ctx.binds_index[b.name] = b end - for _, a in ipairs(annots) do - if a.name then - pipe_ctx.annot_counts[a.name] = (pipe_ctx.annot_counts[a.name] or 0) + 1 - end - end -- Findings live in a single struct with three lists (errors / warnings / info). -- Each check writes to the list appropriate for its severity. @@ -543,8 +590,8 @@ local function validate(ctx, src) if rule.post then rule.post(pipe_ctx, findings) end end - -- Per-skip-marker rules. - -- Each raw marker recorded by scan_source (in scan.skip_over.markers) is validated independently; + -- Per-skip-marker rules. + -- Each raw marker recorded by scan_source (in scan.skip_over.markers) is validated independently; -- the check emits at most one error per marker. -- Valid markers stay attached to scan.skip_over.atoms /.components for dwarf_injection.lua consumer. local skip_markers = scan.skip_over and scan.skip_over.markers or {} @@ -555,7 +602,7 @@ local function validate(ctx, src) end -- Per-macro rules (TAPE_WORDS vs WORD_COUNT drift). - local wc = ctx.shared.word_counts + local wc = corpus_pipe_ctx.word_counts for _, m in ipairs(scan.macros) do for _, rule in ipairs(CHECK_RULES) do if rule.per_macro then rule.per_macro(m, wc, findings) end @@ -571,8 +618,8 @@ local function validate(ctx, src) -- Information summary (always emitted). findings.info[#findings.info + 1] = { line = 0, - msg = string.format("scanned: %d atom(s), %d annotation(s), %d macro-word-decl(s), %d binds struct(s)", - #atoms, #annots, #scan.macros, #scan.binds), + msg = string.format("scanned: %d atom(s), %d annotation(s), %d macro-word-decl(s), %d binds struct(s)" + , #atoms, #annots, #scan.macros, #scan.binds), } return { @@ -650,9 +697,16 @@ function M.run(ctx) local errors = {} local warnings = {} - -- Per-DIRECTORY (per-module) aggregation. Group sources by `src.dir`, validate every source in the dir, then emit ONE errors.h per dir. - -- `ctx.by_dir` is pre-computed in build_ctx (shared across all passes). - local by_dir = ctx.by_dir or duffle.group_sources_by_dir(ctx.sources) + -- Build the corpus-wide pipe_ctx ONCE per pass run. + -- The corpus owns the canonical cross-source registries; per-source scans retain body / declaration ownership. + -- The pipe_ctx is shared across every validate() invocation in this M.run so cross-source visibility is constant. + local corpus_pipe_ctx = build_corpus_pipe_ctx(ctx) + local corpus = ctx.shared.corpus + + -- Per-DIRECTORY (per-module) aggregation. + -- Group sources by `src.dir`, validate every source in the dir, then emit ONE errors.h per dir. + -- The corpus owns `sources_by_dir`; this pass reads the corpus bucket directly. + local by_dir = (corpus and corpus.sources_by_dir) or {} for dir, dir_sources in pairs(by_dir) do local dir_basename = dir:match("([^/\\]+)$") or dir @@ -663,7 +717,7 @@ function M.run(ctx) ctx.flags = ctx.flags or {} ctx.flags._annot_source_results = ctx.flags._annot_source_results or {} for _, src in ipairs(dir_sources) do - local result = validate(ctx, src) + local result = validate(ctx, src, corpus_pipe_ctx) result.source = src.path -- tag for downstream rendering ctx.flags._annot_source_results[src.path] = result -- stash so report.lua reads from cache instead of re-running validate() dir_atoms = dir_atoms + #result.atoms diff --git a/scripts/passes/atoms_source_map.lua b/scripts/passes/atoms_source_map.lua index 375f153..e9178e1 100644 --- a/scripts/passes/atoms_source_map.lua +++ b/scripts/passes/atoms_source_map.lua @@ -1,11 +1,8 @@ --- passes/atoms_source_map.lua — Per-.word source-line map emitter for tape atoms. --- ---- Reads the pre-scanned SourceScan payload (produced once upstream by `duffle.scan_source`) ---- for `MipsAtom_(name)` (kind="atom"), `MipsAtomComp_` / `MipsAtomComp_Proc_` (kind="comp_*"), ---- and `MipsCode code_` (kind="raw_atom") declarations. ---- Walks each atom's pre-tokenized body (`{{tok=string, rel=integer}, ...}` from `duffle.tokenize_body`), ---- counts per-token word contributions via `ctx.shared.word_counts`, and emits one ---- `WORD N LINE L TEXT T` line per `.word` to `/.atoms.sourcemap.txt`. +--- Reads the canonical `atom.paths` projection produced by the upstream `emission_model` pass. +--- The ordered `items` stream, dense `word_events`, and `invocations` views are the only semantic inputs to this pass; +--- it emits one `WORD N LINE L TEXT T` line per emitted `.word`. --- --- **Two output forms** (per the workspace's per-emission-form pattern from --- `guide_metaprogram_ssdl.md`): @@ -34,10 +31,8 @@ --- ENDATOM --- ``` --- ---- Marker calls (`atom_label(...)`, `atom_offset(...)`) emit 0 `.word`s. ---- They share the same walking convention as `passes/offsets.lua :: scan_atom_body`: ---- Markers do NOT advance the word-offset counter, but if a marker is bundled on the same token with a trailing instruction ---- (e.g. `atom_label(foo) load_half_u(...)`), the trailing instruction's word count is added. This matches `offsets.lua :: count_marker_rest`. +--- Marker records are zero-width in `atom.paths.items`; they do not appear in +--- the dense word view and therefore emit no WORD rows. --- --- **Conventions:** tabs (1/level), EmmyLua annotations, no regex, --- Lua 5.3 compatible. @@ -50,10 +45,8 @@ -- (works both standalone + when require'd). `duffle_paths.lua` sets package.path then returns `require("duffle")` -- at the bottom, so the dofile value IS the duffle module. local _bootstrap_dir = debug.getinfo(1, "S").source:match("^@?(.*[/\\])") or "./" -local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua") -local elf_dwarf = require("elf_dwarf") -local word_count_eval = require("word_count_eval") -local count_token_words = word_count_eval.count_token_words +local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua") +local elf_dwarf = require("elf_dwarf") -- ════════════════════════════════════════════════════════════════════════════ -- Constants @@ -63,199 +56,90 @@ local count_token_words = word_count_eval.count_token_words -- the gdb runtime loader rejects mismatches (E2). local FORMAT_VERSION = 1 --- Marker-call identifiers (mirrors offsets.lua:33-34). -local LABEL_MARKER = "atom_label" -local OFFSET_MARKER = "atom_offset" - -- ════════════════════════════════════════════════════════════════════════════ -- Type declarations -- ════════════════════════════════════════════════════════════════════════════ --- @class AtomSourceMapCtx ---- @field sources table[] -- SourceScan payload per source (from `ctx.sources`) --- @field shared table -- `ctx.shared` ---- @field shared.word_counts table -- macro name -> word count (populated by word-counts + components passes) +--- @field shared.corpus table -- canonical source-order corpus +--- @field shared.word_counts table -- identity alias of `corpus.word_counts` --- @field out_root string -- output root (e.g. "build/gen") --- @field dry_run boolean -- if true, compute but don't write --- @field flags table -- `ctx.flags`; reads `flags.gdb_runtime` + `flags.elf_path` -- ════════════════════════════════════════════════════════════════════════════ --- Helpers +-- Canonical atom-path renderers -- ════════════════════════════════════════════════════════════════════════════ --- ════════════════════════════════════════════════════════════════════════════ --- Provenance emission --- ════════════════════════════════════════════════════════════════════════════ - --- Component-macro invocation prefix (mirrors components.lua's MAC_PREFIX). -local MAC_PREFIX = "mac_" -local MAC_PREFIX_LEN = 4 - ---- Strip the `mac_` prefix from a token's leading identifier. ---- Returns nil if the identifier doesn't start with `mac_` ---- (so non-component tokens like `load_half_u`, `nop2`, `gte_cmdw_*` fall through cleanly). ---- @param tok string ---- @return string|nil -local function strip_mac_prefix_from_token(tok) - local leading = duffle.read_ident(tok, 1) - if not leading then return nil end - if leading:sub(1, MAC_PREFIX_LEN) == MAC_PREFIX then - return leading:sub(MAC_PREFIX_LEN + 1) - end - return nil -end - ---- Fetch the per-word body lines for a `mac_X(...)` invocation. ---- Walks the component's pre-tokenized body in lockstep with `count_token_words` and attributes each emitted `.word` ---- to a source line via `idx.line_of(...)`. ---- Atom labels (`atom_label(...)`) emit 0 `.word`s and are skipped. ---- @param bare string|nil -- the bare component name (e.g. `gte_load_tri_verts`) ---- @param comp_body_index table ---- @param wc table ---- @return table|nil -- list of source lines, 1-based by word position -local function fetch_body_lines(bare, comp_body_index, wc) - if not (bare and comp_body_index) then return nil end - local idx = comp_body_index[bare] - if not (idx and idx.body_tokens and idx.line_of) then return nil end - local lines = {} - for _, bt in ipairs(idx.body_tokens) do - local bt_tok = duffle.trim(bt.tok or "") - if bt_tok ~= "" then - local leading = duffle.read_ident(bt_tok, 1) - local bt_words - if leading == "atom_label" or leading == "atom_offset" then - bt_words = 0 - else - bt_words = count_token_words(bt_tok, wc) - end - if bt_words > 0 then - local body_line = idx.line_of(idx.body_off + bt.rel) - for _ = 1, bt_words do lines[#lines + 1] = body_line end - end - end - end - return lines -end - ---- Unified per-word entry walker. `mode` is "sourcemap" (3 fields) or "provenance" (8 fields including component + body-line lookup). ---- Returns (entries, total_words). Markers contribute 0 entries. +--- Join canonical words to canonical word items. `items` supplies the ordered +--- word boundaries, while `word_events` supplies call text and source lines. --- @param atom table ---- @param src table ---- @param wc table ---- @param mode string -- "sourcemap" | "provenance" ---- @param comp table|nil -- shared.components map (provenance only) ---- @param comp_body_index table|nil -- per-source body index (provenance only) --- @return table[], integer -local function compute_word_entries(atom, src, wc, mode, comp, comp_body_index) - local entries = {} - local pos = 0 - for _, t in ipairs(atom.body_tokens) do - local tok = t.tok - local rel = t.rel - - local words - if duffle.is_marker_token(tok) then - words = duffle.count_marker_rest(tok, wc, count_token_words) - else - words = count_token_words(tok, wc) - end - - -- Provenance-only: resolve component + body_lines (one fetch per token). - local comp_name, comp_line, comp_path, comp_kind - local body_lines - if mode == "provenance" then - local bare = strip_mac_prefix_from_token(tok) - if bare and comp and comp[bare] then - comp_name = bare - comp_line = comp[bare].line - comp_path = comp[bare].path - comp_kind = comp[bare].kind - end - if comp_name then body_lines = fetch_body_lines(bare, comp_body_index, wc) end - end - - if words > 0 then - local line = src.scan.line_of(atom.body_off + rel) - local text = duffle.trim(tok):gsub("[\t\r\n]+", " ") - for i = 1, words do - local entry - if mode == "provenance" then - entry = { - pos = pos, - line = line, - text = text, - comp_name = comp_name, - comp_line = comp_line, - comp_path = comp_path, - comp_kind = comp_kind, - body_line = body_lines and body_lines[i], - } - else -- "sourcemap" (default) - entry = { pos = pos, line = line, text = text } - end - entries[#entries + 1] = entry - pos = pos + 1 - end - end +local function canonical_word_entries(atom) + local paths = atom.paths or {} + local events = paths.word_events or {} + local word_items = {} + for _, item in ipairs(paths.items or {}) do + if item.kind == "word" then word_items[#word_items + 1] = item end end - return entries, pos + + local entries = {} + for index, event in ipairs(events) do + local item = word_items[index] or {} + entries[#entries + 1] = { + pos = event.i or (index - 1), + line = event.call_line or item.line or 0, + text = event.call_text or item.call_text or "", + body_line = event.body_line or item.body_line or item.line or 0, + invocation = (event.outermost_invocation_id + and paths.invocations + and paths.invocations[event.outermost_invocation_id]) or nil, + } + end + return entries, #events end ---- Render one atom's provenance stanza. Format: ---- `WORD N CALL : MACRO ":" [BODY ]` (for component words) ---- `WORD N CALL : RAW` (for direct instructions) ---- `BODY ` is the source line of THIS specific word within the macro body ---- (lottes_tape.h:N where N is the per-word body line). ---- Absent for RAW rows and for component rows whose component declaration could not be indexed (older pass combinations / external macros). ---- Downstream consumers (dwarf_injection, tests) fall back to DefLine / comp_line when BODY is absent. ---- Returns (lines, total_words). ---- @param src table ---- @param atom table ---- @param wc table ---- @param comp table -- shared.components map ---- @param comp_body_index table -- per-source component body index: bare_name -> {body_off, body_tokens, line_of} +--- Render one atom's provenance stanza. Format 1 remains: +--- `WORD N CALL : MACRO ":" BODY ` +--- `WORD N CALL : RAW` +--- Component identity comes from the canonical outermost invocation record; +--- the count-table lookup is the canonical component declaration witness. +--- @param src table +--- @param atom table +--- @param wc table -- identity alias of corpus.word_counts --- @return string[], integer -local function emit_provenance_stanza(src, atom, wc, comp, comp_body_index) +local function emit_provenance_stanza(src, atom, wc) local lines = {} - local rel_path = src.path:gsub("\\", "/") - local entries, total = compute_word_entries(atom, src, wc, "provenance", comp, comp_body_index) + local rel_path = src.path:gsub("\\\\", "/") + local entries, total = canonical_word_entries(atom) - -- ATOM header line with placeholder total (patched after we know it). lines[#lines + 1] = string.format('ATOM %s "%s" 0', atom.raw_name or atom.name, rel_path) - for _, pe in ipairs(entries) do - if pe.comp_name then - local body_suffix = "" - if pe.body_line then - body_suffix = " BODY " .. tostring(pe.body_line) - end - lines[#lines + 1] = string.format('WORD %d CALL %s:%d MACRO %s "%s:%d"%s', - pe.pos, rel_path, pe.line, pe.comp_name, pe.comp_path, pe.comp_line, body_suffix) + for _, entry in ipairs(entries) do + local inv = entry.invocation + local macro_count = inv and wc["mac_" .. inv.component_name] + if inv and macro_count ~= nil then + lines[#lines + 1] = string.format( + 'WORD %d CALL %s:%d MACRO %s "%s:%d" BODY %d', + entry.pos, rel_path, entry.line, inv.component_name, + inv.def_path or "", inv.def_line or 0, entry.body_line) else - lines[#lines + 1] = string.format("WORD %d CALL %s:%d RAW", pe.pos, rel_path, pe.line) + lines[#lines + 1] = string.format( + "WORD %d CALL %s:%d RAW", entry.pos, rel_path, entry.line) end end - -- Patch the placeholder total in the ATOM header line. lines[1] = lines[1]:gsub(" 0$", " " .. tostring(total)) lines[#lines + 1] = "ENDATOM" return lines, total end ---- Build a per-source component body index keyed by the bare component name (e.g. `gte_load_tri_verts`). ---- Each entry holds the data we need to map each emitted `.word` to its actual source line within the macro body: ---- body_off -- byte offset of the `{` (start of body) in the component's source file. ---- body_tokens -- list of {tok, rel} pairs; `rel` is the byte offset within the body. ---- line_of -- closure resolving byte offsets in the component's source file to lines. ---- Only `comp_bare` + `comp_proc` declarations contribute (a macro invocation can only resolve to one of those). ---- First declaration wins (subsequent redeclarations would collide; today's sources declare each component exactly once). ---- Render the full provenance file content for one source (one `.atoms.provenance.txt` per source). +--- Render the full provenance file content for one source. --- @param src table --- @param wc table ---- @param comp table -- shared.components map ---- @param comp_body_index table -- cross-source component body index (built once in M.run; may be empty) --- @return string -local function render_provenance(src, wc, comp, comp_body_index) +local function render_provenance(src, wc) local lines = {} lines[#lines + 1] = "# FORMAT_VERSION 1" lines[#lines + 1] = "# auto-generated by ps1_meta.lua (passes/atoms_source_map.lua) — DO NOT EDIT" @@ -265,16 +149,15 @@ local function render_provenance(src, wc, comp, comp_body_index) lines[#lines + 1] = "# dwarf_injection to synthesize DW_TAG_inlined_subroutine instances + per-word" lines[#lines + 1] = "# line program rows for native source-level step into component bodies." - -- The cross-source component body index is passed in from M.run (one global lookup shared across every source's provenance file). - -- A per-source lookup would miss every component whose declaration is in another source (e.g. `gte_load_tri_verts` is declared in `lottes_tape.h` but invoked from `hello_gte_tape.c`). - - for _, atom in ipairs(src.scan.atoms or {}) do - local stanza = emit_provenance_stanza(src, atom, wc, comp, comp_body_index) + local function append(atom) + local stanza = emit_provenance_stanza(src, atom, wc) for _, line in ipairs(stanza) do lines[#lines + 1] = line end end + for _, atom in ipairs(src.scan.atoms or {}) do + if atom.paths then append(atom) end + end for _, atom in ipairs(src.scan.raw_atoms or {}) do - local stanza = emit_provenance_stanza(src, atom, wc, comp, comp_body_index) - for _, line in ipairs(stanza) do lines[#lines + 1] = line end + if atom.paths then append(atom) end end return table.concat(lines, "\n") .. "\n" @@ -286,19 +169,16 @@ end --- @param atom table --- @param wc table --- @return string[], integer -local function emit_atom_stanza(src, atom, wc) - local lines = {} - local rel_path = src.path:gsub("\\", "/") - local entries, total = compute_word_entries(atom, src, wc) +local function emit_atom_stanza(src, atom) + local lines = {} + local rel_path = src.path:gsub("\\\\", "/") + local entries, total = canonical_word_entries(atom) - -- ATOM header line with placeholder total (patched after we know it). lines[#lines + 1] = string.format('ATOM %s "%s" 0', atom.raw_name or atom.name, rel_path) - for _, we in ipairs(entries) do + for _, entry in ipairs(entries) do lines[#lines + 1] = string.format("WORD %d LINE %d TEXT %s", - we.pos, we.line, we.text) + entry.pos, entry.line, entry.text) end - - -- Patch the placeholder total in the ATOM header line. lines[1] = lines[1]:gsub(" 0$", " " .. tostring(total)) lines[#lines + 1] = "ENDATOM" return lines, total @@ -309,18 +189,20 @@ end --- @param src table --- @param wc table --- @return string -local function render_source_map(src, wc) +local function render_source_map(src) local lines = {} lines[#lines + 1] = "# FORMAT_VERSION " .. FORMAT_VERSION lines[#lines + 1] = "# auto-generated by ps1_meta.lua (passes/atoms_source_map.lua) — DO NOT EDIT" - for _, atom in ipairs(src.scan.atoms or {}) do - local stanza = emit_atom_stanza(src, atom, wc) + local function append(atom) + local stanza = emit_atom_stanza(src, atom) for _, line in ipairs(stanza) do lines[#lines + 1] = line end end - for _, atom in ipairs(src.scan.raw_atoms or {}) do - local stanza = emit_atom_stanza(src, atom, wc) - for _, line in ipairs(stanza) do lines[#lines + 1] = line end + for _, atom in ipairs(src.scan.atoms or {}) do + if atom.paths then append(atom) end + end + for _, atom in ipairs(src.scan.raw_atoms or {}) do + if atom.paths then append(atom) end end return table.concat(lines, "\n") .. "\n" @@ -343,55 +225,35 @@ end --- @param ctx PassCtx --- @return table[] -- list of {idx, name, src_path, file_base, addr, size_bytes, words, entries} local function build_atom_table(ctx) - local wc = (ctx.shared and ctx.shared.word_counts) or {} local addrs = elf_dwarf.read_nm(ctx.flags.elf_path) - + local corpus = ctx.shared and ctx.shared.corpus local matched = {} - for _, src in ipairs(ctx.sources) do - if src.scan then - local file_base = src.path:match("([^/\\]+)$") or src.path - for _, atom in ipairs(src.scan.atoms or {}) do - if atom.kind == nil or atom.kind == "atom" then - local name = atom.raw_name or atom.name - local info = addrs[name] - if info then - local entries, total = compute_word_entries(atom, src, wc) - matched[#matched + 1] = { - name = name, - src_path = src.path, - file_base = file_base, - addr = info[1], - size_bytes = info[2], - words = total, - entries = entries, - } - end - end - end - for _, atom in ipairs(src.scan.raw_atoms or {}) do - local name = atom.name - local info = addrs[name] - if info then - local entries, total = compute_word_entries(atom, src, wc) - matched[#matched + 1] = { - name = name, - src_path = src.path, - file_base = file_base, - addr = info[1], - size_bytes = info[2], - words = total, - entries = entries, - } - end - end + + for _, src in ipairs(corpus.source_order or {}) do + local file_base = src.path:match("([^/\\\\]+)$") or src.path + local function append(atom) + if not atom.paths then return end + local name = atom.raw_name or atom.name + local info = addrs[name] + if not info then return end + local entries, total = canonical_word_entries(atom) + matched[#matched + 1] = { + name = name, + src_path = src.path, + file_base = file_base, + addr = info[1], + size_bytes = info[2], + words = total, + entries = entries, + } end + for _, atom in ipairs((src.scan or {}).atoms or {}) do append(atom) end + for _, atom in ipairs((src.scan or {}).raw_atoms or {}) do append(atom) end end -- Deterministic order: sort by address (matches `nm` output ordering). table.sort(matched, function(a, b) return a.addr < b.addr end) - for i, a in ipairs(matched) do - a.idx = i - 1 - end + for i, a in ipairs(matched) do a.idx = i - 1 end return matched end @@ -652,57 +514,52 @@ function M.run(ctx) local errors = {} local warnings = {} - -- word-counts + components passes must have populated shared.word_counts. - -- If absent, the orchestrator wired the deps wrong — fail loud. - local wc = (ctx.shared and ctx.shared.word_counts) or {} - if not wc or not next(wc) then + local corpus = ctx.shared and ctx.shared.corpus + if type(corpus) ~= "table" or type(corpus.source_order) ~= "table" then + error("atoms_source_map.run requires ctx.shared.corpus.source_order (canonical corpus).", 0) + end + + -- Word counts are owned by `corpus.word_counts`. + -- The canonical owner is `corpus.word_counts` (populated by `passes/word_count_eval.lua` + `passes/components.lua`). + local wc = corpus.word_counts or {} + if not next(wc) then warnings[#warnings + 1] = { line = 0, - msg = "atoms_source_map: ctx.shared.word_counts is empty; the word-counts + components passes may not have populated it. Check the PASSES dep edges.", + msg = "atoms_source_map: corpus.word_counts is empty; the word-counts + components passes may not have populated it. Check the PASSES dep edges.", } end - -- shared.components map is populated by `passes/components.lua`. - -- Used to attribute each emitted `.word` to either a component macro or the enclosing atom body. - -- If absent, all words fall through as RAW (correct behavior — provenance is additive). - local comp = (ctx.shared and ctx.shared.components) or {} - - -- Cross-source component body index. - -- Built ONCE (and memoized at `ctx.shared.component_body_index`) so every source's provenance writer can resolve `mac_X(...)` - -- invocations back to the macro's body tokens (regardless of which source declared the component). - -- The atom file (`hello_gte_tape.c`) does not contain the `MipsAtomComp_(...)` declarations, - -- so the body data would be missing for every component invocation the atom file emitted. - -- Superseded by `duffle.get_component_body_index` so the same index is shared with `static_analysis.lua` - -- and any future dependency-check pass (sourcemap/provenance byte-identical because the new entry only ADDS fields). - local comp_body_index = duffle.get_component_body_index(ctx) - -- Always emit the canonical text form (per-source). - for _, src in ipairs(ctx.sources) do - if src.scan then - local n_atoms = src.scan.atoms and #src.scan.atoms or 0 - local n_raw_atoms = src.scan.raw_atoms and #src.scan.raw_atoms or 0 - if n_atoms + n_raw_atoms > 0 then - local basename = duffle.basename_no_ext(src.path) - - -- (1) atoms.sourcemap.txt — per-.word line map (unchanged contract). - local sourcemap_path = ctx.out_root .. "/" .. basename .. ".atoms.sourcemap.txt" - local sourcemap_body = render_source_map(src, wc) - - -- (2) atoms.provenance.txt — per-.word provenance with `mac_X(...)` component resolution back to the component's definition file:line. - -- Consumed by `passes/dwarf_injection.lua` to synthesize `DW_TAG_inlined_subroutine` instances for source-level Step Into on component invocations. - local prov_path = ctx.out_root .. "/" .. basename .. ".atoms.provenance.txt" - local prov_body = render_provenance(src, wc, comp, comp_body_index) - - if not ctx.dry_run then - duffle.ensure_dir(duffle.dirname(sourcemap_path)) - duffle.write_file_lf(sourcemap_path, sourcemap_body) - duffle.write_file_lf(prov_path, prov_body) - end - - outputs[#outputs + 1] = { kind = "report", path = sourcemap_path } - outputs[#outputs + 1] = { kind = "report", path = prov_path } + for _, src in ipairs(corpus.source_order) do + local has_projection = false + for _, atom in ipairs((src.scan or {}).atoms or {}) do + if atom.paths then has_projection = true; break end + end + if not has_projection then + for _, atom in ipairs((src.scan or {}).raw_atoms or {}) do + if atom.paths then has_projection = true; break end end end + if has_projection then + local basename = duffle.basename_no_ext(src.path) + + -- (1) atoms.sourcemap.txt — format-1 per-word call-site map. + local sourcemap_path = ctx.out_root .. "/" .. basename .. ".atoms.sourcemap.txt" + local sourcemap_body = render_source_map(src) + + -- (2) atoms.provenance.txt — format-1 per-word definition/body map. + local prov_path = ctx.out_root .. "/" .. basename .. ".atoms.provenance.txt" + local prov_body = render_provenance(src, wc) + + if not ctx.dry_run then + duffle.ensure_dir(duffle.dirname(sourcemap_path)) + duffle.write_file_lf(sourcemap_path, sourcemap_body) + duffle.write_file_lf(prov_path, prov_body) + end + + outputs[#outputs + 1] = { kind = "report", path = sourcemap_path } + outputs[#outputs + 1] = { kind = "report", path = prov_path } + end end -- Optionally emit the gdb-runtime form (post-link, one file per build). diff --git a/scripts/passes/components.lua b/scripts/passes/components.lua index ffc8d8b..90eb9d4 100644 --- a/scripts/passes/components.lua +++ b/scripts/passes/components.lua @@ -108,8 +108,8 @@ local M = {} --- We then verify the preceding context ends with `MipsAtom` --- (the function-decl keyword with possible qualifiers between). --- ---- @param source string ---- @param name string +--- @param source string +--- @param name string --- @param before_pos integer --- @return string|nil local function find_function_args_for(source, name, before_pos) @@ -149,7 +149,7 @@ end --- Used to copy signature comments from the source declaration (`MipsAtomComp_` / `MipsAtomComp_Proc_` / function decl) --- over to the generated `mac_X` macro, so LSP/IntelliSense displays the args doc. --- @param source string ---- @param pos integer +--- @param pos integer --- @return string local function preceding_comment_block(source, pos) local scan_pos = pos @@ -266,12 +266,12 @@ end -- Component projection (read from pre-scanned SourceScan) -- ════════════════════════════════════════════════════════════════════════════ --- Project pre-scanned MipsAtomComp_ / MipsAtomComp_Proc_ entries into Component shape. --- Does per-source backward lookups for args (preceding function decl) and comment (preceding comment block). --- Carries `body_tokens` forward from scan-source so word_count_rec reads from the precomputed table instead of calling duffle.tokenize_body again. --- @param source string -- the full source text (needed for backward lookups) --- @param scan table -- SourceScan from duffle.scan_source --- @return Component[] +--- Project pre-scanned MipsAtomComp_ / MipsAtomComp_Proc_ entries into Component shape. +--- Does per-source backward lookups for args (preceding function decl) and comment (preceding comment block). +--- Carries `body_tokens` forward from scan-source so word_count_rec reads from the precomputed table instead of calling duffle.tokenize_body again. +--- @param source string -- the full source text (needed for backward lookups) +--- @param scan table -- SourceScan from duffle.scan_source +--- @return Component[] local function project_components(source, scan) local out = {} for _, a in ipairs(scan.atoms) do @@ -282,6 +282,7 @@ local function project_components(source, scan) line = a.line, name = a.name, body = a.body, + body_off = a.body_off, body_tokens = a.body_tokens, args = args, comment = comment, @@ -339,11 +340,11 @@ end -- Word-count computation (memoized recursive lookup) -- ════════════════════════════════════════════════════════════════════════════ --- Strip the `mac_` prefix from a component-call ident so we can look it up against the components-by-name table. --- Returns the ident unchanged if it doesn't start with the prefix --- (so a non-component ident like `mask_upper` falls through to the wc-table branch). --- @param ident string|nil --- @return string|nil +--- Strip the `mac_` prefix from a component-call ident so we can look it up against the components-by-name table. +--- Returns the ident unchanged if it doesn't start with the prefix +--- (so a non-component ident like `mask_upper` falls through to the wc-table branch). +--- @param ident string|nil +--- @return string|nil local function strip_mac_prefix(ident) if not ident then return nil end if ident:sub(1, MAC_PREFIX_LEN) == MAC_PREFIX then @@ -352,13 +353,13 @@ local function strip_mac_prefix(ident) return ident end --- (internal) Recursive word-count lookup. `cache` is the memoization table shared across all components --- in a single source's `count_all_components` pass; the in-progress -1 sentinel detects cycles (A -> B -> A). --- @param name string -- the component name (without `mac_`) --- @param comp_by_name table --- @param wc table --- @param cache table --- @return integer +--- (internal) Recursive word-count lookup. `cache` is the memoization table shared across all components +--- in a single source's `count_all_components` pass; the in-progress -1 sentinel detects cycles (A -> B -> A). +--- @param name string -- the component name (without `mac_`) +--- @param comp_by_name table +--- @param wc table +--- @param cache table +--- @return integer local function word_count_rec(name, comp_by_name, wc, cache) if cache[name] ~= nil then return cache[name] end cache[name] = -1 -- mark in-progress (cycle detection) @@ -397,7 +398,7 @@ end --- references hit memoized values instead of re-walking the body. --- Cycle detection (A -> B -> A) is preserved via the in-progress `-1` sentinel in `cache`. --- @param components Component[] ---- @param wc table +--- @param wc table --- @return table -- map of component name (without `mac_`) -> word count local function count_all_components(components, wc) local comp_by_name = {} @@ -470,9 +471,9 @@ end --- Build the list of lines for one component --- (signature comment, `#define mac_X(...)` line with backslash-continued tokens, then `WORD_COUNT(mac_X, N)` entry). ---- @param c Component +--- @param c Component --- @param components Component[] ---- @param wc table +--- @param wc table --- @return string[] -- list of lines for this component local function build_component_lines(c, counts) local lines = {} @@ -504,10 +505,10 @@ end -- Per-source emit logic -- ════════════════════════════════════════════════════════════════════════════ --- Build the boilerplate header lines (the `#ifdef INTELLISENSE_DIRECTIVES` block, --- the `// Auto-generated` comment, the `// Source:` line, and the self-contained `WORD_COUNT` macro definition). --- @param src SourceFile --- @return string[] +--- Build the boilerplate header lines (the `#ifdef INTELLISENSE_DIRECTIVES` block, +--- the `// Auto-generated` comment, the `// Source:` line, and the self-contained `WORD_COUNT` macro definition). +--- @param src SourceFile +--- @return string[] local function header_boilerplate(src) return { -- #pragma once wrapped in #ifdef INTELLISENSE_DIRECTIVES, matching the convention in lottes_tape.h. @@ -529,13 +530,13 @@ local function header_boilerplate(src) } end --- Compute the output path for one source's `.macs.h` file. --- The pre-rework convention uses the *directory* basename --- (not the source file basename) e.g. `code/duffle/lottes_tape.h` produces `code/duffle/gen/duffle.macs.h`. --- This matches what the C codebase #includes. --- @param src SourceFile --- @return string -- the output directory --- @return string -- the full output path +--- Compute the output path for one source's `.macs.h` file. +--- The pre-rework convention uses the *directory* basename (not the source file basename) +--- e.g. `code/duffle/lottes_tape.h` produces `code/duffle/gen/duffle.macs.h`. +--- This matches what the C codebase #includes. +--- @param src SourceFile +--- @return string -- the output directory +--- @return string -- the full output path local function compute_macs_h_path(src) local out_dir = src.dir .. "/" .. GEN_SUBDIR local out_path = out_dir .. "/" .. duffle.basename_no_ext(src.dir) .. ".macs.h" @@ -545,10 +546,10 @@ end --- Emit a per-source `.macs.h` header with the `mac_X` macros + `WORD_COUNT` entries. --- Writes in BINARY mode so LF line endings are preserved (the git blob is LF; Windows text-mode would emit CRLF and break the byte-identical diff). --- Honors `ctx.dry_run`: prints the intended path but does not write the file. ---- @param ctx PassCtx ---- @param src SourceFile +--- @param ctx PassCtx +--- @param src SourceFile --- @param components Component[] ---- @param counts table -- precomputed word counts (from count_all_components) +--- @param counts table -- precomputed word counts (from count_all_components) --- @return string|nil -- path to the written file (nil if no components) local function emit_component_macros_h(ctx, src, components, counts) if #components == 0 then return nil end @@ -577,41 +578,86 @@ end -- Pass entry -- ════════════════════════════════════════════════════════════════════════════ --- (internal) Extend `ctx.shared.word_counts` with this source's component macros so offsets sees them without re-reading the file. --- @param ctx PassCtx --- @param components Component[] --- @param counts table -- precomputed word counts (from count_all_components) -local function update_shared_word_counts(ctx, components, counts) - local wc = ctx.shared.word_counts +--- (internal) Extend the canonical `corpus.word_counts` with this source's component macros so offsets sees them without re-reading the file. +--- First declaration wins: a later caller's count is dropped (the existing entry from the first source is preserved). +--- @param corpus table -- the canonical corpus +--- @param components Component[] +--- @param counts table -- precomputed word counts (from count_all_components) +local function update_canonical_word_counts(corpus, components, counts) + local wc = corpus.word_counts for _, c in ipairs(components) do - wc["mac_" .. c.name] = counts[c.name] + local key = "mac_" .. c.name + if wc[key] == nil then + wc[key] = counts[c.name] + end end end --- @class ComponentDef ---- @field name string -- bare name (without ac_/mac_ prefix) ---- @field line integer -- definition source line (line of `MipsAtomComp_(ac_X)` / `MipsAtomComp_Proc_(ac_X, ...)`) ---- @field path string -- absolute source path of the definition ---- @field kind string -- "comp_bare" | "comp_proc" +--- @field name string -- bare name (without ac_/mac_ prefix) +--- @field line integer -- definition source line (line of `MipsAtomComp_(ac_X)` / `MipsAtomComp_Proc_(ac_X, ...)`) +--- @field path string -- absolute source path of the definition +--- @field kind string -- "comp_bare" | "comp_proc" ---- (internal) Extend `ctx.shared.components` with this source's components-by-name map so downstream passes ---- (atoms_source_map, dwarf_injection) can resolve `mac_X(...)` invocations back to their component definition file:line. ---- provenance emission uses this to attribute each emitted `.word` to either a component macro or the enclosing atom body. --- @param ctx PassCtx --- @param src SourceFile --- @param components Component[] -local function update_shared_components(ctx, src, components) - ctx.shared.components = ctx.shared.components or {} +--- (internal) Populate the canonical `corpus.components` projection with this source's components-by-name map. +--- First declaration wins; later declarations of the same bare name are dropped and recorded as a collision via `corpus.collisions` (kind = "component"). +--- The pass does NOT write to `ctx.shared.components` (ownership follows the canonical contract). +--- @param corpus table -- the canonical corpus +--- @param src SourceFile +--- @param components Component[] +local function update_canonical_components(corpus, src, components) local rel_path = src.path:gsub("\\", "/") for _, c in ipairs(components) do - -- Keyed by bare name (e.g. `yield`, `load_tri_indices`). - -- The atoms_source_map pass strips the `mac_` prefix from the call site identifier before lookup. - ctx.shared.components[c.name] = { - name = c.name, - line = c.line, - path = rel_path, - kind = c.kind or "comp_bare", - } + -- Keyed by bare name (e.g. `yield`, `load_tri_indices`). + -- The atoms_source_map pass looks up components by bare name from the canonical corpus; + -- `mac_` prefix lives at the call-site identifier and is stripped before lookup. + if corpus.components[c.name] == nil then + corpus.components[c.name] = { + name = c.name, + line = c.line, + path = rel_path, + kind = c.kind or "comp_bare", + } + else + -- A second declaration of the same bare name: record a typed collision so static-analysis + the report can surface it. + -- Identical-shape declarations (same path + line) do NOT record a collision (the first-wins entry already covers the case). + local existing = corpus.components[c.name] + if existing.path ~= rel_path or existing.line ~= c.line then + local kind = c.kind or "comp_bare" + local first_kind = existing.kind or "comp_bare" + corpus.collisions[#corpus.collisions + 1] = { + kind = "component", + name = c.name, + first_site = { path = existing.path, line = existing.line }, + conflicting_site = { path = rel_path, line = c.line }, + first_shape = "kind=" .. first_kind, + conflicting_shape = "kind=" .. kind, + } + end + end + end +end + +--- (internal) Populate the canonical `corpus.component_body_index` projection with this source's body index entries. +--- First declaration wins; later declarations are dropped (no separate collision record: the components collision is already surfaced by `update_canonical_components`). +--- The pass does NOT write to `ctx.shared.component_body_index` (the legacy corpus owns this projection). +--- @param corpus table -- the canonical corpus +--- @param src SourceFile +--- @param components Component[] +--- @param scan table -- the SourceScan payload (for line_of) +local function update_canonical_component_body_index(corpus, src, components, scan) + local line_of = scan and scan.line_of + for _, c in ipairs(components) do + if corpus.component_body_index[c.name] == nil then + corpus.component_body_index[c.name] = { + body_tokens = c.body_tokens, + body_off = c.body_off, + line_of = line_of, + source = src.path, + declaration = c.line, + kind = c.kind, + } + end end end @@ -622,24 +668,42 @@ function M.run(ctx) local errors = {} local warnings = {} - -- Initialize shared component map. - -- The atoms_source_map and dwarf_injection passes consume `ctx.shared.components` to resolve `mac_X(...)` - -- invocations back to the component's definition file:line. - ctx.shared.components = ctx.shared.components or {} + -- Canonical-corpus ownership gate. + local corpus = ctx.shared and ctx.shared.corpus + if type(corpus) ~= "table" then + error("components.run requires ctx.shared.corpus (canonical corpus).", 0) + end + if type(corpus.source_order) ~= "table" then + error("components.run requires ctx.shared.corpus.source_order (canonical corpus).", 0) + end + if type(corpus.word_counts) ~= "table" then + error("components.run requires ctx.shared.corpus.word_counts; " + .. "word_count_eval.run must run before components.run " + .. "(see PASSES deps).", 0) + end - for _, src in ipairs(ctx.sources) do + -- Canonical projection ownership: + -- * `corpus.word_counts["mac_"..name]` — current component count + -- * `corpus.components[name]` — bare-name component definition + -- * `corpus.component_body_index[name]` — body / line_of / source index + -- The pass does NOT mutate `ctx.shared.components` or `ctx.shared.component_body_index` + -- (ownership follows the canonical corpus; consumers read from the corpus directly). + + for _, src in ipairs(corpus.source_order) do -- project_components reads from src.scan + does backward lookups on src.text local components = project_components(src.text, src.scan) if #components > 0 then - -- Compute word counts for ALL components once (was: rebuilt per call inside the helpers). - local counts = count_all_components(components, ctx.shared.word_counts) + -- Compute all component word counts once per source. + -- Use `corpus.word_counts` (the canonical count table) so the recursive lookup sees both authored-metadata entries + -- (loaded by word_count_eval.run) AND same-source component entries (populated earlier in this loop by `update_canonical_word_counts`). + local counts = count_all_components(components, corpus.word_counts) local macs_path = emit_component_macros_h(ctx, src, components, counts) if macs_path then outputs[#outputs + 1] = { macs_h = macs_path } - update_shared_word_counts(ctx, components, counts) - -- share component definitions with downstream passes. - -- `mac_X(...)` invocations in atom bodies resolve back to (path, line) via this map. - update_shared_components(ctx, src, components) + -- Populate the canonical projections AFTER disk emission (so the byte-identical `.macs.h` contract is preserved before any current-count mutation). + update_canonical_word_counts(corpus, components, counts) + update_canonical_components(corpus, src, components) + update_canonical_component_body_index(corpus, src, components, src.scan) end end end diff --git a/scripts/passes/dwarf_injection.lua b/scripts/passes/dwarf_injection.lua index cbec377..6b6ae54 100644 --- a/scripts/passes/dwarf_injection.lua +++ b/scripts/passes/dwarf_injection.lua @@ -20,8 +20,8 @@ --- Splice step runs from PowerShell — no Lua subprocess; no cmd /c parsing issues. --- objcopy's --update-section works fine in PowerShell even though Lua's `os.execute`/`io.popen` would mangle the `=` on Windows.) --- ---- Result: VSCode's source gutter follows per-stepi inside atom bodies, AND the Variables pane shows the wave-context regs as atom-scoped locals. ---- Native VSCode UX (gutter arrow + highlighted line + Run to Cursor + conditional BPs by source line + per-atom locals). +--- Result: source stepping follows atom-body lines, and wave-context registers appear as atom-scoped locals. +--- Native VSCode stepping, line highlighting, run-to-cursor, conditional breakpoints, and per-atom locals. --- No VSCode plugin, no Python, no pyelftools — pure Lua + objcopy. --- --- **Conventions:** tabs (1/level), EmmyLua annotations, Lua 5.3 compatible. @@ -37,19 +37,15 @@ local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua") -- ELF32 / DWARF / atoms-source-map utilities (post-link debug-info injection). -- Sister module to duffle.lua — contains the format-constant tables (ELF32 byte offsets, DWARF opcodes, etc.) and the I/O helpers --- (read_elf_sections, nm, source-map parser, LE byte r/w). `list_dir` lives in duffle.lua as a general I/O primitive (lifted out during F''). +-- (read_elf_sections, nm, source-map parser, LE byte r/w). `list_dir` is the general directory primitive in duffle.lua. local elf_dwarf = require("elf_dwarf") --- word-counting helper shared with passes/atoms_source_map.lua. --- Used here to walk a component's body_tokens in lockstep with their word-count allocation --- when we propagate per-word body lines into each invocation's `body_lines` array. -local word_count_eval = require("word_count_eval") -local count_token_words = word_count_eval.count_token_words +-- Per-word body lines come from the canonical `atom.paths` projection. local lfs = require("lfs") -- File-scope aliases to elf_dwarf helpers; the canonical implementations live in scripts/elf_dwarf.lua. --- (2-caller lift: these were duplicated file-locals; the canonical is in elf_dwarf.lua, used by parse_abbrev_table + read_form_value.) +-- ELF decoding helpers come from `elf_dwarf.lua`. local read_uleb128_at = elf_dwarf.read_uleb128_at local read_sleb128_at = elf_dwarf.read_sleb128_at local find_abbrev_table_end = elf_dwarf.find_abbrev_table_end @@ -97,7 +93,7 @@ local ATOM_SOURCE_FILE_INDEX = 11 -- New abbreviation codes (100+ to avoid collision with gcc's existing 1-60+ codes). local ABBREV_CU = 0x64 -- 100: DW_TAG_compile_unit local ABBREV_SUBPROGRAM = 0x65 -- 101: DW_TAG_subprogram -local ABBREV_VARIABLE = 0x66 -- 102: DW_TAG_variable (DW_AT_type = ref4 to U4; was missing pre-2026-07-13 → gdb resolved R_PrimCursor against the C-level enum, not the register) +local ABBREV_VARIABLE = 0x66 -- 102: DW_TAG_variable with DW_AT_type = ref4 to U4 local ABBREV_STRUCT_TYPE = 0x67 -- 103: DW_TAG_structure_type with children (Binds_X mirror) local ABBREV_MEMBER = 0x68 -- 104: DW_TAG_member no children (DW_AT_type = ref4 to U4 base) local ABBREV_BIND_VAR = 0x69 -- 105: DW_TAG_variable no children + DW_AT_type = ref4 (the bind_args variable) @@ -150,10 +146,10 @@ local DW_AT_language = 0x13 local DW_AT_location = 0x02 local DW_AT_comp_dir = 0x1B local DW_AT_byte_size = 0x0B -local DW_AT_encoding = 0x3E -- DWARF5 §7.7.1: DW_AT_encoding (for DW_ATE_unsigned base type; was 0x13 = DW_AT_language in prior slice - semantically wrong) +local DW_AT_encoding = 0x3E -- DWARF5 §7.7.1: DW_AT_encoding for the DW_ATE_unsigned base type local DW_AT_data_member_location = 0x38 local DW_AT_type = 0x49 -local DW_AT_linkage_name = 0x6E -- DWARF5 §7.7.1: DW_AT_linkage_name (standard form; 0x200027 was the GNU extension form - wrong vs DW_FORM_string abbrev) +local DW_AT_linkage_name = 0x6E -- DWARF5 §7.7.1: DW_AT_linkage_name with DW_FORM_string local DW_AT_external = 0x3F -- marks a variable/function as externally visible -- Inlined_subroutine + abstract_origin attributes. local DW_AT_abstract_origin = 0x31 @@ -341,7 +337,7 @@ end local DEFAULT_CU_NAME = "tape_atom_locals" local DEFAULT_CU_COMP_DIR = "." --- Path templates for the .bin outputs are now in SECTION_WRITERS (see below). +-- SECTION_WRITERS owns the .bin output path templates. -- Default basename if not provided via ctx. local DEFAULT_BASENAME = "hello_gte" @@ -367,11 +363,12 @@ end --- Consume the per-source scanner associations without naming any atom or component in production. --- Whole atoms remain symbol-keyed; components are file-qualified internally so a source marker associates --- with its exact component definition even though GDB 12 requires function-only skip entries for the resulting synthetic inline frame. ---- @param ctx DwarfInjectionCtx +--- Iterates `corpus.source_order` (the canonical corpus projection). +--- @param corpus table -- the canonical corpus from `ctx.shared.corpus` --- @return table -- {atoms = {[symbol] = association}, components = {[file|name] = association}} -local function collect_skip_over(ctx) +local function collect_skip_over(corpus) local skip_over = { atoms = {}, components = {} } - for _, src in ipairs(ctx.sources or {}) do + for _, src in ipairs((corpus and corpus.source_order) or {}) do local scan_skip = src.scan and src.scan.skip_over if scan_skip then for atom_name, association in pairs(scan_skip.atoms or {}) do @@ -396,53 +393,50 @@ local function collect_skip_over(ctx) return skip_over end ---- Merge the per-source scanner registries (register_alias_registry, type_name_registry, atom_views) ---- into a single set of tables that downstream consumers can read from without re-iterating ctx.sources. +--- Project the canonical corpus registries into the shape the section builders expect. +--- The corpus already owns the merged `register_alias_registry`, `type_name_registry`, `atom_views`, `atom_ctxs`, `atom_phases`, and `atom_infos` projections (populated by `passes.scan_source.lua`). +--- This helper just references them so the rest of `dwarf_injection.lua` keeps the same `registries.` access shape it has always used. --- --- Every `R_*` lookup and per-atom type override resolution in this file goes through this merged table. --- Aliases without `atom_reg` adjacent are absent; the absence is treated as "not debug-visible" (see build_inserted_children for the precedence chain). --- --- When two sources register the same key, the last-writer wins (later sources override earlier). --- Today only one source declares wave-context enums, so collisions are absent. ---- @param ctx DwarfInjectionCtx +--- @param corpus table -- the canonical corpus from `ctx.shared.corpus` --- @return table -- { --- register_alias_registry = {[R_Name] = AliasEntry}, --- type_name_registry = {[T] = TypeEntry}, --- atom_views = {[atom_name] = AtomViewEntry}, --- } -local function collect_per_source_registries(ctx) - local merged = { - register_alias_registry = {}, - type_name_registry = {}, - atom_views = {}, +local function collect_per_source_registries(corpus) + -- The corpus already holds the merged registries; reference them directly. + -- No per-source iteration is needed because `passes.scan_source.lua` has already folded every per-source scan into the canonical tables. + -- `atom_infos` is preserved byte-for-byte with no filtering; consumers consult `corpus.atoms_by_name` + -- themselves when they need to know whether a particular atom_info corresponds to an actual atom record. + local atom_infos_list = {} + for _, ai in ipairs((corpus and corpus.atom_infos) or {}) do + atom_infos_list[#atom_infos_list + 1] = ai + end + return { + register_alias_registry = (corpus and corpus.register_alias_registry) or {}, + type_name_registry = (corpus and corpus.type_name_registry) or {}, + atom_views = (corpus and corpus.atom_views) or {}, -- Per-atom atom_ctx declarations: atom_name -> {rbind_atom, ...} -- (populated by scan_source from `atom_ctx()` sub-calls inside `atom_info`) - atom_ctxs = {}, + atom_ctxs = (corpus and corpus.atom_ctxs) or {}, -- Per-phase atom groups: phase_label -> {atoms = {atom_name1, ...}} -- (populated by scan_source from `atom_phase(