diff --git a/scripts/audit_lua_nesting.lua b/scripts/audit_lua_nesting.lua new file mode 100644 index 0000000..14bcaa6 --- /dev/null +++ b/scripts/audit_lua_nesting.lua @@ -0,0 +1,241 @@ +--- audit_lua_nesting.lua — Walk Lua source files and flag any block +--- nesting deeper than 5 levels. +--- +--- Usage: +--- luajit scripts/audit_lua_nesting.lua scripts/duffle.lua scripts/ps1_meta.lua +--- luajit scripts/audit_lua_nesting.lua scripts/passes/ +--- +--- Output: for each file, a list of {line, depth} entries where depth > 5. +--- Returns exit code 1 if any violations found, 0 if clean. +--- +--- **Implementation**: a hand-rolled depth tracker that counts: +--- - `do`, `function`, `if`, `for`, `while`, `repeat` -> depth +1 +--- - `end`, `until` -> depth -1 +--- - `else`, `elseif` -> depth unchanged +--- +--- **Caveats**: doesn't handle string/comment state (will miscount +--- braces inside strings). For our metaprogram files (no embedded +--- code generation), this is acceptable. + +local M = {} + +local BLOCK_OPEN = { + ["do"] = true, + ["function"] = true, + ["if"] = true, + ["for"] = true, + ["while"] = true, + ["repeat"] = true, +} + +local function is_block_close(token) + return token == "end" or token == "until" +end + +-- (internal) Walk one source file and return a list of +-- {line, depth, token} entries where depth > MAX_NESTING. +local function audit_file(path, max_nesting) + local f = io.open(path, "r") + if not f then + error("Cannot open " .. path) + end + local content = f:read("*a") + f:close() + + local violations = {} + local depth = 0 + local line = 1 + local pos = 1 + local len = #content + local token_start = 0 + + local function read_ident_at(p) + -- Lua ident: [a-zA-Z_][a-zA-Z0-9_]* + local start = p + if start > len then return nil end + local ch = content:sub(start, start) + if not (ch:match("[%a_]")) then return nil end + p = p + 1 + while p <= len do + local c = content:sub(p, p) + if not (c:match("[%w_]")) then break end + p = p + 1 + end + return content:sub(start, p - 1), p + end + + local function skip_string_or_comment(p) + local ch = content:sub(p, p) + if ch == '"' or ch == "'" then + -- String literal: skip to matching end-quote. + p = p + 1 + while p <= len do + local c = content:sub(p, p) + if c == "\\" then + p = p + 2 + elseif c == ch then + p = p + 1 + break + else + p = p + 1 + end + end + return p + elseif ch == "-" and content:sub(p + 1, p + 1) == "-" then + -- Lua comment: -- to end of line. + p = p + 2 + if content:sub(p, p + 1) == "[[" and content:sub(p + 2, p + 3) == "[" then + -- Long bracket comment [==[ ... ]==] + p = p + 2 + local eq = "" + while content:sub(p, p) == "=" do + eq = eq .. "=" + p = p + 1 + end + local close_marker = "]" .. eq .. "]" + local close_pos = content:find(close_marker, p, true) + if close_pos then + p = close_pos + #close_marker + else + p = len + 1 + end + else + while p <= len and content:sub(p, p) ~= "\n" do p = p + 1 end + end + return p + elseif ch == "[" and content:sub(p + 1, p + 1) == "[" then + -- Long bracket string: [==[ ... ]==] + p = p + 2 + local eq = "" + while content:sub(p, p) == "=" do + eq = eq .. "=" + p = p + 1 + end + local close_marker = "]" .. eq .. "]" + local close_pos = content:find(close_marker, p, true) + if close_pos then + p = close_pos + #close_marker + else + p = len + 1 + end + return p + end + return nil + end + + local token_count = 0 + while pos <= len do + local ch = content:sub(pos, pos) + if ch == "\n" then line = line + 1 end + + local skip_to = skip_string_or_comment(pos) + if skip_to then + for i = pos, skip_to - 1 do + if content:sub(i, i) == "\n" then line = line + 1 end + end + pos = skip_to + elseif ch:match("[%a_]") then + local tok, next_pos = read_ident_at(pos) + token_count = token_count + 1 + if BLOCK_OPEN[tok] then + depth = depth + 1 + if depth > max_nesting then + violations[#violations + 1] = { + line = line, + depth = depth, + token = tok, + } + end + elseif is_block_close(tok) then + depth = depth - 1 + end + pos = next_pos + else + pos = pos + 1 + end + end + + return violations +end + +--- Audit one file. Returns nil if clean, else a list of violations. +--- @param path string +--- @param max_nesting integer -- default 5 +--- @return table|nil +function M.audit(path, max_nesting) + local violations = audit_file(path, max_nesting or 5) + if #violations == 0 then return nil end + return violations +end + +-- Module CLI. +if arg and arg[1] then + local max_nesting = 5 + local files = {} + for i = 1, #arg do + if arg[i] == "--max" and arg[i + 1] then + max_nesting = tonumber(arg[i + 1]) or 5 + else + files[#files + 1] = arg[i] + end + end + + -- Accept either a directory or a file path. Directory args are + -- expanded via `dir /b *.lua` (Windows) or `ls *.lua` (Unix). + local function is_dir(p) + -- Try opening it as a file; if that succeeds, it's not a dir. + local f = io.open(p, "r") + if f then f:close() return false end + return true + end + local function list_lua(dir) + local out = {} + local cmd + if package.config:sub(1, 1) == "\\" then + cmd = 'dir /b "' .. dir .. '\\*.lua" 2>nul' + else + cmd = 'ls -1 "' .. dir .. '"/*.lua 2>/dev/null' + end + local p = io.popen(cmd) + if p then + for line in p:lines() do + if line:match("%.lua$") then + out[#out + 1] = dir .. "/" .. line + end + end + p:close() + end + return out + end + + local to_check = {} + for _, f in ipairs(files) do + if is_dir(f) then + for _, sub in ipairs(list_lua(f)) do to_check[#to_check + 1] = sub end + else + to_check[#to_check + 1] = f + end + end + + local total_violations = 0 + for _, f in ipairs(to_check) do + local v = M.audit(f, max_nesting) + if v then + io.write(string.format("\n%s\n", f)) + for _, x in ipairs(v) do + io.write(string.format(" line %d: depth %d (after '%s')\n", x.line, x.depth, x.token)) + end + total_violations = total_violations + #v + end + end + + if total_violations == 0 then + io.write("OK: no files exceed max nesting of " .. max_nesting .. "\n") + os.exit(0) + else + io.write(string.format("\n%d nesting violation(s) found.\n", total_violations)) + os.exit(1) + end +end + +return M \ No newline at end of file diff --git a/scripts/duffle.lua b/scripts/duffle.lua index 77adb21..d9ff1c6 100644 --- a/scripts/duffle.lua +++ b/scripts/duffle.lua @@ -1,21 +1,151 @@ --- duffle.lua --- --- Shared primitives + domain tables for the tape-atom metaprograms. --- --- 5.3-compatible Lua (no 5.4/5.5-only features): --- - no / --- - no continue keyword --- - no string.dump improvements --- - LuaJIT 5.1+extensions model is the primary target --- --- No :match / :gmatch regex use anywhere; all delimiter-splitting is --- hand-rolled or via LPeg (the regex-free PEG library). --- --- Phase 3: the hot lexer primitives are LPeg-backed where it pays off. --- The hand-rolled variants remain for callers that need a fallback. +--- duffle.lua — Shared primitives + domain tables for the tape-atom +--- metaprograms. +--- +--- This module is the source for: +--- - **Character classification** (`is_space`, `is_alpha`, `is_alnum`, `is_digit`, plus the byte-fast `_byte` variants). +--- - **String primitives** (`trim`, `dirname`, `basename_no_ext`, `find_byte`). +--- - **I/O primitives** (`read_file`, `write_file`, `ensure_dir`). +--- - **C-language scanner** (`skip_ws_and_cmt`, `skip_str_or_cmt`, +--- `read_ident`, `read_parens`, `read_braces`, `read_brackets`, +--- `read_balanced`, `scan_to_char`, `split_top_level_commas`). +--- - **Word-count loader** (`load_word_counts` for `WORD_COUNT(...)` metadata files). +--- - **Line lookup** (`LineIndex` returns an O(log N) `line_of(pos)` closure for source-mapping). +--- - **Domain tables** (`WAVE_CONTEXT_REGS`, `TAPE_ATOM_MACROS`, +--- `GTE_PIPELINE_LATENCY`, `GP0_CMD_SIZE`, `GP0_CMD_BY_SHAPE`, +--- `GP0_MACRO_CONTRIB`, `INSTRUCTION_LATENCY`). +--- - **Process-bootstrap helper** (`setup_package_path` — replaces the 8-line `arg[0]`-resolution boilerplate duplicated across 7 entry scripts) +--- +--- **Conventions**: tabs (1/level), EmmyLua annotations, no regex. +--- Lua 5.3 compatible; no ``/``, no `continue`, no +--- 5.4 string.dump improvements. LuaJIT 5.1+extensions model is the +--- primary target. +--- +--- **No `:match` / `:gmatch` regex use anywhere**; all delimiter- +--- splitting is hand-rolled or via LPeg (the regex-free PEG library). +--- The hot lexer primitives are LPeg-backed where it pays off; +--- hand-rolled variants remain for callers that need a fallback. local M = {} +-- ════════════════════════════════════════════════════════════════════════════ +-- Cross-file type aliases +-- ════════════════════════════════════════════════════════════════════════════ + +--- @alias Path string -- absolute or CWD-relative file path +--- @alias LineNum integer -- 1-indexed source line number +--- @alias ByteOff integer -- 0-indexed byte offset within a source string +--- @alias MacroName string -- lower_snake_case macro identifier (e.g. "mac_yield") +--- @alias AtomName string -- lower_snake_case atom name (e.g. "cube_g4_face") +--- @alias Severity string -- "error" | "warning" | "info" + +--- @class SourceFile +--- @field path Path -- absolute path to the source file +--- @field text string -- the full source text +--- @field dir string -- the directory containing the source +--- @field basename string -- filename without extension + +-- ════════════════════════════════════════════════════════════════════════════ +-- ASCII byte constants +-- ════════════════════════════════════════════════════════════════════════════ + +local BYTE_SPACE = 32 -- ' ' +local BYTE_TAB = 9 -- '\t' +local BYTE_NEWLINE = 10 -- '\n' +local BYTE_CR = 13 -- '\r' +local BYTE_VT = 11 -- '\v' +local BYTE_FF = 12 -- '\f' + +local BYTE_UNDERSCORE = 95 -- '_' +local BYTE_DOT = 46 -- '.' +local BYTE_SLASH = 47 -- '/' +local BYTE_BACKSLASH = 92 -- '\\' +local BYTE_STAR = 42 -- '*' +local BYTE_DQUOTE = 34 -- '"' +local BYTE_SQUOTE = 39 -- '\'' +local BYTE_COMMA = 44 -- ',' +local BYTE_SEMI = 59 -- ';' + +local BYTE_OPEN_PAREN = 40 -- '(' +local BYTE_OPEN_BRACE = 123 -- '{' +local BYTE_OPEN_BRACK = 91 -- '[' + +local BYTE_LOWER_A = 97 -- 'a' +local BYTE_LOWER_Z = 122 -- 'z' +local BYTE_UPPER_A = 65 -- 'A' +local BYTE_UPPER_Z = 90 -- 'Z' + +local BYTE_DIGIT_0 = 48 -- '0' +local BYTE_DIGIT_9 = 57 -- '9' + +-- ════════════════════════════════════════════════════════════════════════════ +-- Section -1: Bootstrap (path-setup at module load) +-- ════════════════════════════════════════════════════════════════════════════ +-- +-- When duffle.lua is first loaded (via `dofile` from an entry script +-- or via `require` from a passes script), the code below runs and sets +-- `package.path` + `package.cpath` so subsequent `require`s resolve. +-- Idempotent: re-loads just re-set the same paths. +-- +-- **Entry scripts** trigger this with one line: +-- `local duffle = dofile(arg[0]:match("(.*[/\\])") .. "/../duffle.lua")` +-- which runs this top-level + returns `M`. +-- +-- **Passes scripts** are loaded via `require("passes.X")` from the +-- entry script; by the time they run, the entry script has already +-- triggered this bootstrap, so the paths are set. +-- +-- **Why `git rev-parse`?** Hardcoding paths like +-- `C:\\projects\\Pikuma\\ps1\\...` breaks portability. Git gives us +-- the canonical repo root regardless of where it lives. + +--- Resolve the repo root via `git rev-parse --show-toplevel`. Returns a +--- path with a trailing separator, or nil if not in a git repo. +--- @return string|nil +local function find_repo_root() + local p = io.popen("git rev-parse --show-toplevel 2>nul") + local root + if p then + root = p:read("*l") + p:close() + end + if not root or root == "" then return nil end + if not root:match("[/\\]$") then root = root .. "/" end + return root +end + +--- Set `package.path` (for `require("duffle")` + `require("passes.X")`) +--- and `package.cpath` (for `lpeg.dll` on Windows). +function M.setup_package_path() + local repo_root = find_repo_root() + if not repo_root then + io.stderr:write("[duffle] git rev-parse failed -- not in a git repo?\n") + os.exit(2) + end + + -- From the repo root, derive both `scripts/` and `scripts/passes/` + -- so `require("duffle")` AND `require("passes.annotation")` resolve. + local scripts_dir = repo_root .. "scripts/" + local passes_dir = repo_root .. "scripts/passes/" + package.path = scripts_dir .. "?.lua;" + .. scripts_dir .. "?/init.lua;" + .. passes_dir .. "?.lua;" + .. passes_dir .. "?/init.lua;" + .. package.path + +-- cpath: only needed on Windows for the bundled lpeg.dll. (LPeg +-- is optional -- duffle.lua's `pcall(require, "lpeg")` falls back +-- to hand-rolled scanners if the .dll isn't loadable.) + if package.config:sub(1, 1) == "\\" then + package.cpath = repo_root .. "toolchain/luajit-2.1/lib/lua/5.1/?.dll;" + .. package.cpath + end +end + +-- NOTE: `M.setup_package_path()` is NOT auto-called here. The entry +-- scripts explicitly `dofile("duffle_paths.lua")` first, which calls +-- `M.setup_package_path()`. The function exists for the helper to use +-- (so the path-setup logic is centralized in duffle.lua). + -- ════════════════════════════════════════════════════════════════════════════ -- Section 0: LPeg patterns (compiled once at module load) -- ════════════════════════════════════════════════════════════════════════════ @@ -34,40 +164,40 @@ local lpeg_str_or_cmt_pat, lpeg_ws_and_cmt_pat local lpeg_scan_to_target_pat -- generic "anything but target or balanced group" matcher if lpeg_ok then - lpeg_lib = lpeg - local P, S, R = lpeg.P, lpeg.S, lpeg.R + lpeg_lib = lpeg + local P, S, R = lpeg.P, lpeg.S, lpeg.R - -- Character class patterns - local alpha_pat = R("AZ", "az") + P("_") - local digit_pat = R("09") - lpeg_alnum_pat = alpha_pat + digit_pat + -- Character class patterns + local alpha_pat = R("AZ", "az") + P("_") + local digit_pat = R("09") + lpeg_alnum_pat = alpha_pat + digit_pat - -- Identifier: alpha followed by zero+ alnum. Capture as a string. - lpeg_alpha_pat = alpha_pat - lpeg_ident_pat = lpeg.C(alpha_pat * lpeg_alnum_pat^0) + -- Identifier: alpha followed by zero+ alnum. Capture as a string. + lpeg_alpha_pat = alpha_pat + lpeg_ident_pat = lpeg.C(alpha_pat * lpeg_alnum_pat^0) - -- String literal: "..." with backslash escapes. - local lpeg_str_pat = P('"') * (P(1) - S('"\\') + P('\\') * P(1))^0 * P('"') - -- Char literal: '...' with backslash escapes. - local lpeg_chr_pat = P("'") * (P(1) - S("'\\") + P('\\') * P(1))^0 * P("'") - -- Line comment: // ... to end-of-line. - local lpeg_line_cmt_pat = P("//") * (P(1) - S("\n"))^0 - -- Block comment: /* ... */ (no nesting per C standard). - local lpeg_block_cmt_pat = P("/*") * (P(1) - P("*/"))^0 * P("*/") - -- String or comment (any of the four forms). - lpeg_str_or_cmt_pat = lpeg_str_pat + lpeg_chr_pat + lpeg_line_cmt_pat + lpeg_block_cmt_pat + -- String literal: "..." with backslash escapes. + local lpeg_str_pat = P('"') * (P(1) - S('"\\') + P('\\') * P(1))^0 * P('"') + -- Char literal: '...' with backslash escapes. + local lpeg_chr_pat = P("'") * (P(1) - S("'\\") + P('\\') * P(1))^0 * P("'") + -- Line comment: // ... to end-of-line. + local lpeg_line_cmt_pat = P("//") * (P(1) - S("\n"))^0 + -- Block comment: /* ... */ (no nesting per C standard). + local lpeg_block_cmt_pat = P("/*") * (P(1) - P("*/"))^0 * P("*/") + -- String or comment (any of the four forms). + lpeg_str_or_cmt_pat = lpeg_str_pat + lpeg_chr_pat + lpeg_line_cmt_pat + lpeg_block_cmt_pat - -- Whitespace + comment skipper: zero+ (whitespace run | string | comment). - local ws_pat = S(" \t\n\r\v\f") - lpeg_ws_and_cmt_pat = (ws_pat + lpeg_str_or_cmt_pat)^0 + -- Whitespace + comment skipper: zero+ (whitespace run | string | comment). + local ws_pat = S(" \t\n\r\v\f") + lpeg_ws_and_cmt_pat = (ws_pat + lpeg_str_or_cmt_pat)^0 - -- Generic "skip until target, but step over balanced groups" matcher. - -- Used by scan_to_char for non-ident / non-bracket chars. - -- We accept any single char except the target. - -- The balanced-group stepping is handled by the caller (via read_balanced). - lpeg_scan_to_target_pat = function(target) - return (P(1) - P(target))^0 - end + -- Generic "skip until target, but step over balanced groups" matcher. + -- Used by scan_to_char for non-ident / non-bracket chars. + -- We accept any single char except the target. + -- The balanced-group stepping is handled by the caller (via read_balanced). + lpeg_scan_to_target_pat = function(target) + return (P(1) - P(target))^0 + end end -- ════════════════════════════════════════════════════════════════════════════ @@ -83,19 +213,20 @@ end -- Whitespace characters per C locale. function M.is_space_byte(b) - return b == 32 or b == 9 or b == 10 or b == 13 or b == 11 or b == 12 + return b == BYTE_SPACE or b == BYTE_TAB or b == BYTE_NEWLINE + or b == BYTE_CR or b == BYTE_VT or b == BYTE_FF end -- Letters (a-z, A-Z) and underscore. function M.is_alpha_byte(b) - if not b then return false end - if b >= 97 and b <= 122 then return true end -- 'a'..'z' - if b >= 65 and b <= 90 then return true end -- 'A'..'Z' - return b == 95 -- '_' + if not b then return false end + if b >= BYTE_LOWER_A and b <= BYTE_LOWER_Z then return true end -- 'a'..'z' + if b >= BYTE_UPPER_A and b <= BYTE_UPPER_Z then return true end -- 'A'..'Z' + return b == BYTE_UNDERSCORE end -- Single digit. -function M.is_digit_byte(b) return b and b >= 48 and b <= 57 end +function M.is_digit_byte(b) return b and b >= BYTE_DIGIT_0 and b <= BYTE_DIGIT_9 end -- Letter OR digit OR underscore. function M.is_alnum_byte(b) return M.is_alpha_byte(b) or M.is_digit_byte(b) end @@ -103,19 +234,19 @@ function M.is_alnum_byte(b) return M.is_alpha_byte(b) or M.is_digit_byte(b) end -- String-based wrappers (kept for callers that already have a single-char -- string; the byte versions are what the hot loops should call). function M.is_space(c) - if type(c) == "number" then return M.is_space_byte(c) end - return c == " " or c == "\t" or c == "\n" or c == "\r" or c == "\v" or c == "\f" + if type(c) == "number" then return M.is_space_byte(c) end + return c == " " or c == "\t" or c == "\n" or c == "\r" or c == "\v" or c == "\f" end function M.is_alpha(c) - if type(c) == "number" then return M.is_alpha_byte(c) end - if not c or #c == 0 then return false end - if c >= "a" and c <= "z" then return true end - if c >= "A" and c <= "Z" then return true end - return c == "_" + if type(c) == "number" then return M.is_alpha_byte(c) end + if not c or #c == 0 then return false end + if c >= "a" and c <= "z" then return true end + if c >= "A" and c <= "Z" then return true end + return c == "_" end function M.is_digit(c) - if type(c) == "number" then return M.is_digit_byte(c) end - return c and c >= "0" and c <= "9" + if type(c) == "number" then return M.is_digit_byte(c) end + return c and c >= "0" and c <= "9" end function M.is_alnum(c) return M.is_alpha(c) or M.is_digit(c) end @@ -125,44 +256,50 @@ function M.is_alnum(c) return M.is_alpha(c) or M.is_digit(c) end -- Trim leading and trailing whitespace from a string. function M.trim(s) - local a = 1 - while a <= #s and M.is_space_byte(s:byte(a)) do a = a + 1 end - local b = #s - while b >= a and M.is_space_byte(s:byte(b)) do b = b - 1 end - return s:sub(a, b) + local a = 1 + while a <= #s and M.is_space_byte(s:byte(a)) do a = a + 1 end + local b = #s + while b >= a and M.is_space_byte(s:byte(b)) do b = b - 1 end + return s:sub(a, b) end -- Linear-search for a single-byte target in a string. -- (Phase 3 retained this for places where LPeg is overkill.) +-- @param haystack string +-- @param target integer -- byte value +-- @param start integer -- optional 1-indexed start (default 1) +-- @return integer|nil function M.find_byte(haystack, target, start) - for i = start or 1, #haystack do - if haystack:byte(i) == target then return i end - end - return nil + for pos = start or 1, #haystack do + if haystack:byte(pos) == target then return pos end + end + return nil end -- Returns the directory portion of a path. function M.dirname(path) - local last_sep = 0 - for i = 1, #path do - if path:byte(i) == 47 or path:byte(i) == 92 then last_sep = i end -- '/' or '\\' - end - if last_sep == 0 then return "." end - return path:sub(1, last_sep - 1) + local last_sep = 0 + for pos = 1, #path do + local b = path:byte(pos) + if b == BYTE_SLASH or b == BYTE_BACKSLASH then last_sep = pos end + end + if last_sep == 0 then return "." end + return path:sub(1, last_sep - 1) end -- Returns the basename of a path, with the file extension stripped. function M.basename_no_ext(path) - local last_sep = 0 - for i = 1, #path do - if path:byte(i) == 47 or path:byte(i) == 92 then last_sep = i end - end - local a = last_sep + 1 - local last_dot = #path + 1 - for i = #path, a, -1 do - if path:byte(i) == 46 then last_dot = i; break end -- '.' - end - return path:sub(a, last_dot - 1) + local last_sep = 0 + for pos = 1, #path do + local b = path:byte(pos) + if b == BYTE_SLASH or b == BYTE_BACKSLASH then last_sep = pos end + end + local a = last_sep + 1 + local last_dot = #path + 1 + for pos = #path, a, -1 do + if path:byte(pos) == BYTE_DOT then last_dot = pos; break end + end + return path:sub(a, last_dot - 1) end -- ════════════════════════════════════════════════════════════════════════════ @@ -170,18 +307,18 @@ end -- ════════════════════════════════════════════════════════════════════════════ function M.read_file(path) - local f = io.open(path, "r") - if not f then error("Cannot open " .. path) end - local content = f:read("*a") - f:close() - return content + local f = io.open(path, "r") + if not f then error("Cannot open " .. path) end + local content = f:read("*a") + f:close() + return content end function M.write_file(path, content) - local f = io.open(path, "w") - if not f then error("Cannot write " .. path) end - f:write(content) - f:close() + local f = io.open(path, "w") + if not f then error("Cannot write " .. path) end + f:write(content) + f:close() end -- Cache of directories already verified to exist in this process. Each @@ -192,11 +329,11 @@ end local _ensured_dirs = {} function M.ensure_dir(path) - if _ensured_dirs[path] then return end - _ensured_dirs[path] = true - local is_win = package.config:sub(1, 1) == "\\" - os.execute(is_win and ('if not exist "' .. path .. '" mkdir "' .. path .. '"') - or ('mkdir -p "' .. path .. '" 2>/dev/null')) + if _ensured_dirs[path] then return end + _ensured_dirs[path] = true + local is_win = package.config:sub(1, 1) == "\\" + os.execute(is_win and ('if not exist "' .. path .. '" mkdir "' .. path .. '"') + or ('mkdir -p "' .. path .. '" 2>/dev/null')) end -- Test helper: clear the cache (used by tests + between process runs). @@ -211,77 +348,77 @@ function M._reset_ensured_dirs() _ensured_dirs = {} end -- Returns position just past the construct, or `i` unchanged if no -- string/comment starts at position i. function M.skip_str_or_cmt(s, i) - if lpeg_ok then - local new_pos = lpeg.match(lpeg_str_or_cmt_pat, s, i) - if new_pos then return new_pos end - return i - end - -- Hand-rolled fallback (kept for builds where LPeg isn't available). - local c = s:byte(i) - if c == 34 or c == 39 then -- '"' or '\'' - i = i + 1 - while i <= #s do - local b = s:byte(i) - if b == 92 then i = i + 2 -- '\\' - elseif b == c then return i + 1 - else i = i + 1 end - end - return #s + 1 - elseif c == 47 then -- '/' - local nx = s:byte(i + 1) - if nx == 47 then -- '//' - while i <= #s and s:byte(i) ~= 10 do i = i + 1 end - return i - elseif nx == 42 then -- '/*' - i = i + 2 - while i <= #s - 1 do - if s:byte(i) == 42 and s:byte(i + 1) == 47 then -- '*/' - return i + 2 - end - i = i + 1 - end - return #s + 1 - end - end - return i + if lpeg_ok then + local new_pos = lpeg.match(lpeg_str_or_cmt_pat, s, i) + if new_pos then return new_pos end + return i + end +-- Hand-rolled fallback (kept for builds where LPeg isn't available). + local c = s:byte(i) + if c == BYTE_DQUOTE or c == BYTE_SQUOTE then -- '"' or '\'' + i = i + 1 + while i <= #s do + local b = s:byte(i) + if b == BYTE_BACKSLASH then i = i + 2 -- '\\' + elseif b == c then return i + 1 + else i = i + 1 end + end + return #s + 1 + elseif c == BYTE_SLASH then -- '/' + local nx = s:byte(i + 1) + if nx == BYTE_SLASH then -- '//' + while i <= #s and s:byte(i) ~= BYTE_NEWLINE do i = i + 1 end + return i + elseif nx == BYTE_STAR then -- '/*' + i = i + 2 + while i <= #s - 1 do + if s:byte(i) == BYTE_STAR and s:byte(i + 1) == BYTE_SLASH then -- '*/' + return i + 2 + end + i = i + 1 + end + return #s + 1 + end + end + return i end -- Skip whitespace AND C-style comments starting at position i. -- LPeg-backed when available; ~5-10x faster than the hand-rolled version. function M.skip_ws_and_cmt(s, i) - if lpeg_ok then - local new_pos = lpeg.match(lpeg_ws_and_cmt_pat, s, i) - if new_pos then return new_pos end - return i - end - -- Hand-rolled fallback. - local len = #s - while i <= len do - if M.is_space_byte(s:byte(i)) then - i = i + 1 - else - local nx = M.skip_str_or_cmt(s, i) - if nx > i then i = nx else break end - end - end - return i + if lpeg_ok then + local new_pos = lpeg.match(lpeg_ws_and_cmt_pat, s, i) + if new_pos then return new_pos end + return i + end + -- Hand-rolled fallback. + local len = #s + while i <= len do + if M.is_space_byte(s:byte(i)) then + i = i + 1 + else + local nx = M.skip_str_or_cmt(s, i) + if nx > i then i = nx else break end + end + end + return i end -- Read a C-style identifier (alpha followed by zero+ alnum) starting at -- position i. Returns the identifier string + the position just past it, -- or nil + i if no identifier starts here. function M.read_ident(s, i) - if lpeg_ok then - local result = lpeg.match(lpeg_ident_pat, s, i) - if result then return result, i + #result end - return nil, i - end - -- Hand-rolled fallback. - if not M.is_alpha_byte(s:byte(i)) then return nil, i end - local a = i - i = i + 1 - while i <= #s and M.is_alnum_byte(s:byte(i)) do i = i + 1 end - return s:sub(a, i - 1), i + if lpeg_ok then + local result = lpeg.match(lpeg_ident_pat, s, i) + if result then return result, i + #result end + return nil, i + end + -- Hand-rolled fallback. + if not M.is_alpha_byte(s:byte(i)) then return nil, i end + local a = i + i = i + 1 + while i <= #s and M.is_alnum_byte(s:byte(i)) do i = i + 1 end + return s:sub(a, i - 1), i end -- Read a balanced-delimited group (parens, braces, or brackets) starting @@ -291,27 +428,27 @@ end -- -- (Hand-rolled; the depth counting makes pure LPeg awkward here.) function M.read_balanced(s, open_char, close_char, i) - local open_byte = open_char:byte() - if s:byte(i) ~= open_byte then return nil, i end - i = i + 1 - local len = #s - local depth = 1 - local a = i - while i <= len and depth > 0 do - local c = s:byte(i) - if c == open_byte then - depth = depth + 1 - i = i + 1 - elseif c == close_char:byte() then - depth = depth - 1 - if depth == 0 then break end - i = i + 1 - else - local nx = M.skip_str_or_cmt(s, i) - if nx > i then i = nx else i = i + 1 end - end - end - return s:sub(a, i - 1), i + 1 + local open_byte = open_char:byte() + if s:byte(i) ~= open_byte then return nil, i end + local pos = i + 1 + local len = #s + local depth = 1 + local a = pos + while pos <= len and depth > 0 do + local c = s:byte(pos) + if c == open_byte then + depth = depth + 1 + pos = pos + 1 + elseif c == close_char:byte() then + depth = depth - 1 + if depth == 0 then break end + pos = pos + 1 + else + local nx = M.skip_str_or_cmt(s, pos) + pos = (nx > pos) and nx or (pos + 1) + end + end + return s:sub(a, pos - 1), pos + 1 end -- Convenience specializations of read_balanced. @@ -323,20 +460,20 @@ M.read_brackets = function(s, i) return M.read_balanced(s, "[", "]", i) end -- `target`, transparently stepping over balanced parens/braces/brackets. -- Returns the position of `target`, or nil if not found. function M.scan_to_char(s, target, start) - local target_byte = target:byte() - local i = start - while i <= #s do - local c = s:byte(i) - if c == target_byte then return i end - if c == 40 then local _, a = M.read_balanced(s, "(", ")", i); i = a -- '(' - elseif c == 123 then local _, a = M.read_balanced(s, "{", "}", i); i = a -- '{' - elseif c == 91 then local _, a = M.read_balanced(s, "[", "]", i); i = a -- '[' - else - local nx = M.skip_str_or_cmt(s, i) - if nx > i then i = nx else i = i + 1 end - end - end - return nil + local target_byte = target:byte() + local pos = start + while pos <= #s do + local c = s:byte(pos) + if c == target_byte then return pos end + if c == BYTE_OPEN_PAREN then local _, a = M.read_balanced(s, "(", ")", pos); pos = a + elseif c == BYTE_OPEN_BRACE then local _, a = M.read_balanced(s, "{", "}", pos); pos = a + elseif c == BYTE_OPEN_BRACK then local _, a = M.read_balanced(s, "[", "]", pos); pos = a + else + local nx = M.skip_str_or_cmt(s, pos) + pos = (nx > pos) and nx or (pos + 1) + end + end + return nil end -- Split a brace-body into top-level comma-separated tokens. Honors nested @@ -351,96 +488,96 @@ end -- chunks (which now appear between real statements) are filtered out so -- they contribute 0 words instead of 1. function M.split_top_level_commas(body) - local tokens = {} - local i = 1 - local len = #body - local token_start = 1 + local tokens = {} + local pos = 1 + local body_len = #body + local token_start = 1 - -- True iff `chunk` contains any non-whitespace, non-comment, non-string - -- content (i.e., real token material). Walks through ws + comments - -- individually so a chunk like " /* trailing */ shift_lleft(...)" - -- is correctly classified as having real content (the macro call). - local function has_real_content(chunk) - local k = 1 - local klen = #chunk - while k <= klen do - if M.is_space(chunk:sub(k, k)) then - k = k + 1 - else - local nx = M.skip_str_or_cmt(chunk, k) - if nx > k then - k = nx -- skipped a comment or string - else - return true -- found real content - end - end - end - return false - end + -- True iff `chunk` contains any non-whitespace, non-comment, non-string + -- content (i.e., real token material). Walks through ws + comments + -- individually so a chunk like " /* trailing */ shift_lleft(...)" + -- is correctly classified as having real content (the macro call). + local function has_real_content(chunk) + local scan = 1 + local len = #chunk + while scan <= len do + if M.is_space(chunk:sub(scan, scan)) then + scan = scan + 1 + else + local nx = M.skip_str_or_cmt(chunk, scan) + if nx > scan then + scan = nx -- skipped a comment or string + else + return true -- found real content + end + end + end + return false + end - local function emit(end_pos) - if end_pos >= token_start then - local chunk = body:sub(token_start, end_pos) - if M.trim(chunk) ~= "" then - if has_real_content(chunk) then - tokens[#tokens + 1] = chunk - elseif #tokens > 0 then - -- Pure comment/string chunk at top level (no - -- preceding instruction content within this chunk). - -- APPEND it to the LAST token so emit-context - -- callers (components.lua build_component_lines) - -- can convert `// trailing comment` to `/* */` - -- and emit it with the macro body. For word - -- counting, count_token_words only inspects the - -- leading ident, so a trailing comment doesn't - -- affect the count. - -- - -- This is the second-half fix to commit 98e27c2: - -- the first fix correctly broke top-level comments - -- off from the NEXT statement (fixing macro-call - -- word counts); this fix preserves them on the - -- PREVIOUS statement (restoring the comments in - -- the emitted .macs.h output). - tokens[#tokens] = tokens[#tokens] .. chunk - end - end - token_start = end_pos + 1 - end - end + local function emit(end_pos) + if end_pos >= token_start then + local chunk = body:sub(token_start, end_pos) + if M.trim(chunk) ~= "" then + if has_real_content(chunk) then + tokens[#tokens + 1] = chunk + elseif #tokens > 0 then + -- Pure comment/string chunk at top level (no + -- preceding instruction content within this chunk). + -- APPEND it to the LAST token so emit-context + -- callers (components.lua build_component_lines) + -- can convert `// trailing comment` to `/* */` + -- and emit it with the macro body. For word + -- counting, count_token_words only inspects the + -- leading ident, so a trailing comment doesn't + -- affect the count. + -- + -- This is the second-half fix to commit 98e27c2: + -- the first fix correctly broke top-level comments + -- off from the NEXT statement (fixing macro-call + -- word counts); this fix preserves them on the + -- PREVIOUS statement (restoring the comments in + -- the emitted .macs.h output). + tokens[#tokens] = tokens[#tokens] .. chunk + end + end + token_start = end_pos + 1 + end + end - while i <= len do - local c = body:byte(i) - if c == 40 then -- '(' - local _, a = M.read_parens(body, i); i = a - elseif c == 123 then -- '{' - local _, a = M.read_braces(body, i); i = a - elseif c == 91 then -- '[' - local _, a = M.read_brackets(body, i); i = a - elseif c == 44 then -- ',' - emit(i - 1) - i = i + 1 - token_start = i - elseif c == 59 then -- ';' - emit(i - 1) - i = i + 1 - token_start = i - elseif c == 10 then -- '\n' - emit(i - 1) - i = i + 1 - token_start = i - else - local nx = M.skip_str_or_cmt(body, i) - if nx > i then - -- Skipped a comment or string at top level: emit token break. - i = nx - emit(i - 1) - else - i = i + 1 - end - end - end - emit(len) - return tokens + while pos <= body_len do + local c = body:byte(pos) + if c == BYTE_OPEN_PAREN then -- '(' + local _, a = M.read_parens(body, pos); pos = a + elseif c == BYTE_OPEN_BRACE then -- '{' + local _, a = M.read_braces(body, pos); pos = a + elseif c == BYTE_OPEN_BRACK then -- '[' + local _, a = M.read_brackets(body, pos); pos = a + elseif c == BYTE_COMMA then -- ',' + emit(pos - 1) + pos = pos + 1 + token_start = pos + elseif c == BYTE_SEMI then -- ';' + emit(pos - 1) + pos = pos + 1 + token_start = pos + elseif c == BYTE_NEWLINE then -- '\n' + emit(pos - 1) + pos = pos + 1 + token_start = pos + else + local nx = M.skip_str_or_cmt(body, pos) + if nx > pos then + -- Skipped a comment or string at top level: emit token break. + pos = nx + emit(pos - 1) + else + pos = pos + 1 + end + end + end + emit(body_len) + return tokens end -- ════════════════════════════════════════════════════════════════════════════ @@ -448,27 +585,27 @@ end -- ════════════════════════════════════════════════════════════════════════════ function M.load_word_counts(metadata_path) - local counts = {} - local content = M.read_file(metadata_path) - local len = #content - local i = 1 - local prefix = "WORD_COUNT(" - while i <= len do - local nl = M.find_byte(content, 10, i) -- '\n' - local line_end = nl or (len + 1) - local line = content:sub(i, line_end - 1) - local trimmed = M.trim(line) - if trimmed:sub(1, #prefix) == prefix and trimmed:sub(-1) == ")" then - local inner = trimmed:sub(#prefix + 1, #trimmed - 1) - local comma = M.find_byte(inner, 44, 1) -- ',' - if comma then - counts[M.trim(inner:sub(1, comma - 1))] = - tonumber(M.trim(inner:sub(comma + 1))) - end - end - i = line_end + 1 - end - return counts + local counts = {} + local content = M.read_file(metadata_path) + local len = #content + local pos = 1 + local prefix = "WORD_COUNT(" + while pos <= len do + local nl = M.find_byte(content, BYTE_NEWLINE, pos) + local line_end = nl or (len + 1) + local line = content:sub(pos, line_end - 1) + local trimmed = M.trim(line) + if trimmed:sub(1, #prefix) == prefix and trimmed:sub(-1) == ")" then + local inner = trimmed:sub(#prefix + 1, #trimmed - 1) + local comma = M.find_byte(inner, BYTE_COMMA, 1) + if comma then + counts[M.trim(inner:sub(1, comma - 1))] = + tonumber(M.trim(inner:sub(comma + 1))) + end + end + pos = line_end + 1 + end + return counts end -- ════════════════════════════════════════════════════════════════════════════ @@ -476,27 +613,28 @@ end -- ════════════════════════════════════════════════════════════════════════════ function M.LineIndex(source) - local positions = {} - local n = 0 - for i = 1, #source do - if source:byte(i) == 10 then -- '\n' - n = n + 1 - positions[n] = i - end - end - local function line_of(pos) - local lo, hi = 1, n - while lo <= hi do - local mid = math.floor((lo + hi) / 2) - if positions[mid] <= pos then - lo = mid + 1 - else - hi = mid - 1 - end - end - return hi + 1 - end - return line_of + local positions = {} + local n = 0 + for pos = 1, #source do + if source:byte(pos) == BYTE_NEWLINE then + n = n + 1 + positions[n] = pos + end + end + -- (internal) Binary-search for the line number containing `query_pos`. + local function line_of(query_pos) + local lo, hi = 1, n + while lo <= hi do + local mid = math.floor((lo + hi) / 2) + if positions[mid] <= query_pos then + lo = mid + 1 + else + hi = mid - 1 + end + end + return hi + 1 + end + return line_of end -- ════════════════════════════════════════════════════════════════════════════ @@ -504,10 +642,10 @@ end -- ════════════════════════════════════════════════════════════════════════════ M.WAVE_CONTEXT_REGS = { - ["R_PrimCursor"] = { alias = "R_T7", size = 4, role = "output cursor (prim arena)" }, - ["R_FaceCursor"] = { alias = "R_T4", size = 4, role = "input cursor (face array)" }, - ["R_VertBase"] = { alias = "R_T5", size = 4, role = "base pointer (vertex array)" }, - ["R_OtBase"] = { alias = "R_T6", size = 4, role = "base pointer (ordering table)" }, + ["R_PrimCursor"] = { alias = "R_T7", size = 4, role = "output cursor (prim arena)" }, + ["R_FaceCursor"] = { alias = "R_T4", size = 4, role = "input cursor (face array)" }, + ["R_VertBase"] = { alias = "R_T5", size = 4, role = "base pointer (vertex array)" }, + ["R_OtBase"] = { alias = "R_T6", size = 4, role = "base pointer (ordering table)" }, } -- The annotation DSL has been reduced to a single annotation macro: @@ -517,7 +655,7 @@ M.WAVE_CONTEXT_REGS = { -- of atom_info; for now, the parser only recognizes atom_info + its -- three sub-calls (atom_bind, atom_reads, atom_writes). M.TAPE_ATOM_MACROS = { - ["atom_info"] = { kind = "info", binds = false }, + ["atom_info"] = { kind = "info", binds = false }, } -- GTE pipeline-fill latency table (static-analysis Phase 1). @@ -541,61 +679,61 @@ M.TAPE_ATOM_MACROS = { -- => 2 nops; nclip 8cy => 2 nops; avsz3/avsz4 14cy => 2 nops; op -- single-cycle atomic => 0 nops; mvmva 8cy matrix-vector => 2 nops). M.GTE_PIPELINE_LATENCY = { - -- Minimum number of consecutive `nop` words that must appear - -- IMMEDIATELY BEFORE a `gte_cmdw_` invocation -- to retire - -- any preceding `lwc2` / `swc2` / pre-existing C2 state writes - -- before the GTE pipeline starts reading from V0/V1/V2 or - -- MAC0..3 / OTZ / IR0..3 at the command's issue cycle. - -- - -- Values are from the doxygen comments in code/duffle/gte.h and - -- cross-checked against PSX-SPX `geometrytransformationenginegte.md`: - -- - -- cmd cycles min pre-nops rationale - -- rtps 14 2 8c per perspective divide + 6c for IR1..4 + mac write - -- rptt 22 2 3x rtps worth of pipeline depth - -- nclip 7 2 MAC0 write + 5c for sign - -- avsz3 14 2 14c to compute average + write OTZ - -- avsz4 16 2 avsz3 + 2c extra for avg over 4 - -- mvmva 8 2 IR1..4 write + matrix work - -- op 5 0 output to MAC0 only (atomic 5c calc) - -- - -- The `gte_rtpt()` / `gte_nclip()` / `gte_avsz3()` wrapper macros in - -- gte.h emit the pre-cmd nops internally (asm_words(nop, nop, ...)), - -- but THOSE WRAPPERS ARE NOT USED INSIDE ATOM BODIES in this - -- codebase. Every MipsAtom_(name) body uses raw `nop2, - -- gte_cmdw_, ...` form instead -- that `nop2,` is the pre-fill - -- this check validates. So values here must reflect the source-level - -- convention, NOT the wrapper-internal pre-fill (which is invisible - -- at the source level). - -- - -- Existing clean-atom bodies (cube_g4_face, floor_f3_face, - -- diag_gte) all emit `nop2,` before every `gte_cmdw_` (which - -- matches values >= 2). The check passes them all. - -- - -- Aliases are listed separately because source code may use either - -- the alias or the canonical name. The check looks up the EXACT - -- macro text, so both forms must be in the table. + -- Minimum number of consecutive `nop` words that must appear + -- IMMEDIATELY BEFORE a `gte_cmdw_` invocation -- to retire + -- any preceding `lwc2` / `swc2` / pre-existing C2 state writes + -- before the GTE pipeline starts reading from V0/V1/V2 or + -- MAC0..3 / OTZ / IR0..3 at the command's issue cycle. + -- + -- Values are from the doxygen comments in code/duffle/gte.h and + -- cross-checked against PSX-SPX `geometrytransformationenginegte.md`: + -- + -- cmd cycles min pre-nops rationale + -- rtps 14 2 8c per perspective divide + 6c for IR1..4 + mac write + -- rptt 22 2 3x rtps worth of pipeline depth + -- nclip 7 2 MAC0 write + 5c for sign + -- avsz3 14 2 14c to compute average + write OTZ + -- avsz4 16 2 avsz3 + 2c extra for avg over 4 + -- mvmva 8 2 IR1..4 write + matrix work + -- op 5 0 output to MAC0 only (atomic 5c calc) + -- + -- The `gte_rtpt()` / `gte_nclip()` / `gte_avsz3()` wrapper macros in + -- gte.h emit the pre-cmd nops internally (asm_words(nop, nop, ...)), + -- but THOSE WRAPPERS ARE NOT USED INSIDE ATOM BODIES in this + -- codebase. Every MipsAtom_(name) body uses raw `nop2, + -- gte_cmdw_, ...` form instead -- that `nop2,` is the pre-fill + -- this check validates. So values here must reflect the source-level + -- convention, NOT the wrapper-internal pre-fill (which is invisible + -- at the source level). + -- + -- Existing clean-atom bodies (cube_g4_face, floor_f3_face, + -- diag_gte) all emit `nop2,` before every `gte_cmdw_` (which + -- matches values >= 2). The check passes them all. + -- + -- Aliases are listed separately because source code may use either + -- the alias or the canonical name. The check looks up the EXACT + -- macro text, so both forms must be in the table. - -- Canonical macros (from code/duffle/gte.h) - ["gte_cmdw_rtps"] = 2, - ["gte_cmdw_rtpt"] = 2, - ["gte_cmdw_nclip"] = 2, - ["gte_cmdw_op"] = 0, - ["gte_cmdw_mvmva"] = 2, - ["gte_cmdw_avsz3"] = 2, - ["gte_cmdw_avsz4"] = 2, + -- Canonical macros (from code/duffle/gte.h) + ["gte_cmdw_rtps"] = 2, + ["gte_cmdw_rtpt"] = 2, + ["gte_cmdw_nclip"] = 2, + ["gte_cmdw_op"] = 0, + ["gte_cmdw_mvmva"] = 2, + ["gte_cmdw_avsz3"] = 2, + ["gte_cmdw_avsz4"] = 2, - -- Aliases (must have the same value as their canonical target) - ["gte_cmdw_rotate_translate_perspective_single"] = 2, - ["gte_cmdw_rotate_translate_perspective_triple"] = 2, - ["gte_cmdw_avg_sort_z4"] = 2, + -- Aliases (must have the same value as their canonical target) + ["gte_cmdw_rotate_translate_perspective_single"] = 2, + ["gte_cmdw_rotate_translate_perspective_triple"] = 2, + ["gte_cmdw_avg_sort_z4"] = 2, - -- Outer product aliases (same canonical op, 0 pre-fill nops). - -- gte_cmdw_op = canonical GTE-internal short form - -- gte_cmdw_outer_product = NOCASH / SDK-readable form - -- gte_cmdw_wedge = geometric-algebra (exterior-product) form - ["gte_cmdw_outer_product"] = 0, - ["gte_cmdw_wedge"] = 0, + -- Outer product aliases (same canonical op, 0 pre-fill nops). + -- gte_cmdw_op = canonical GTE-internal short form + -- gte_cmdw_outer_product = NOCASH / SDK-readable form + -- gte_cmdw_wedge = geometric-algebra (exterior-product) form + ["gte_cmdw_outer_product"] = 0, + ["gte_cmdw_wedge"] = 0, } -- GP0 packet sizes (total words including the 1-word tag) per GP0 cmd byte. @@ -610,24 +748,24 @@ M.GTE_PIPELINE_LATENCY = { -- set_poly_g4(p) -> set_len(p, 8) -> 9 total GP0 0x38 -- set_poly_gt4(p) -> set_len(p, 12) -> 13 total GP0 0x3C M.GP0_CMD_SIZE = { - [0x20] = 5, -- Poly_F3 - [0x24] = 8, -- Poly_FT3 - [0x28] = 6, -- Poly_F4 - [0x2C] = 10, -- Poly_FT4 - [0x30] = 7, -- Poly_G3 - [0x34] = 10, -- Poly_GT3 - [0x38] = 9, -- Poly_G4 - [0x3C] = 13, -- Poly_GT4 + [0x20] = 5, -- Poly_F3 + [0x24] = 8, -- Poly_FT3 + [0x28] = 6, -- Poly_F4 + [0x2C] = 10, -- Poly_FT4 + [0x30] = 7, -- Poly_G3 + [0x34] = 10, -- Poly_GT3 + [0x38] = 9, -- Poly_G4 + [0x3C] = 13, -- Poly_GT4 } -- Shape suffix (after `ac_format_` / `mac_format_` prefix) -> GP0 cmd byte. -- Lets the static-analysis check derive the cmd byte from a macro name -- like `mac_format_g4_color` -> `g4` -> 0x38 -> 9 expected words. M.GP0_CMD_BY_SHAPE = { - ["f3"] = 0x20, ["ft3"] = 0x24, - ["f4"] = 0x28, ["ft4"] = 0x2C, - ["g3"] = 0x30, ["gt3"] = 0x34, - ["g4"] = 0x38, ["gt4"] = 0x3C, + ["f3"] = 0x20, ["ft3"] = 0x24, + ["f4"] = 0x28, ["ft4"] = 0x2C, + ["g3"] = 0x30, ["gt3"] = 0x34, + ["g4"] = 0x38, ["gt4"] = 0x3C, } -- Per-macro prim-buffer contribution (NOT .text instruction count -- @@ -636,15 +774,15 @@ M.GP0_CMD_BY_SHAPE = { -- `mac_gte_store_X_post_*` + `mac_insert_ot_tag_X` calls in an atom body -- must equal GP0_CMD_SIZE[GP0_CMD_BY_SHAPE[shape]]. M.GP0_MACRO_CONTRIB = { - ["mac_format_f3_color"] = 1, - ["mac_format_g3_color"] = 3, - ["mac_format_g4_color"] = 4, - ["mac_gte_store_f3_post_rtpt"] = 3, - ["mac_gte_store_g3_post_rtpt"] = 3, - ["mac_gte_store_g4_p012_post_rtpt_pre_rtps"] = 3, - ["mac_gte_store_g4_p3_post_rtps"] = 1, - ["mac_insert_ot_tag_f3"] = 1, - ["mac_insert_ot_tag_g4"] = 1, + ["mac_format_f3_color"] = 1, + ["mac_format_g3_color"] = 3, + ["mac_format_g4_color"] = 4, + ["mac_gte_store_f3_post_rtpt"] = 3, + ["mac_gte_store_g3_post_rtpt"] = 3, + ["mac_gte_store_g4_p012_post_rtpt_pre_rtps"] = 3, + ["mac_gte_store_g4_p3_post_rtps"] = 1, + ["mac_insert_ot_tag_f3"] = 1, + ["mac_insert_ot_tag_g4"] = 1, } -- Per-macro cycle cost (best-case, no stalls). Used by the static-analysis @@ -675,121 +813,121 @@ M.GP0_MACRO_CONTRIB = { -- mvmva = 6 + 2 nops = 8 total -- op = 5 (no pre-cmd nops required; single-cycle atomic) M.INSTRUCTION_LATENCY = { - -- CPU ALU (single-cycle R3000A ops) - ["nop"] = 1, - ["nop2"] = 2, - ["add_ui"] = 1, ["add_ui_self"] = 1, - ["add_s"] = 1, ["add_si"] = 1, - ["add_u"] = 1, ["add_u_self"] = 1, - ["sub_u"] = 1, ["sub_s"] = 1, - ["and_i"] = 1, ["and_u"] = 1, - ["or_i"] = 1, ["or_i_self"] = 1, - ["or_u"] = 1, ["or_u_self"] = 1, - ["xor_i"] = 1, ["xor_u"] = 1, - ["nor_u"] = 1, - ["shift_lleft"] = 1, ["shift_lleft_self"] = 1, - ["shift_lright"] = 1, - ["shift_aright"] = 1, - ["mask_upper"] = 1, - ["mov_from_high"] = 2, -- mfhi: 2 cycles - ["mov_from_low"] = 2, -- mflo: 2 cycles - ["mov_to_high"] = 1, -- mthi: 1 cycle - ["mov_to_low"] = 1, -- mtlo: 1 cycle - -- Set-on-condition (SLT family) - ["set_lt_u"] = 1, ["set_lt_ui"] = 1, - ["set_lt_s"] = 1, ["set_lt_si"] = 1, - -- Multiply / divide (no hardware multiplier; software via inline asm) - ["mult_u"] = 12, ["mult_s"] = 12, - ["div_u"] = 35, ["div_s"] = 35, - -- Loads (1 cycle + load-delay slot; the delay is typically absorbed by - -- the next instruction in a well-pipelined sequence, so we count 1) - ["load_word"] = 1, - ["load_half_u"] = 1, ["load_half"] = 1, - ["load_byte_u"] = 1, ["load_byte"] = 1, - ["load_upper_i"] = 1, - -- 2-word loads (lui + ori) used for >16-bit immediates - ["load_imm"] = 2, - ["load_imm_1w"] = 1, - ["load_imm_1w_s0"] = 1, - ["load_imm_2w"] = 2, - ["load_imm_2w_addi_forced"] = 2, - ["load_imm_2w_ori_forced"] = 2, - -- Stores (1 cycle each) - ["store_word"] = 1, - ["store_half"] = 1, - ["store_byte"] = 1, - -- Branches (branch + BD slot nop = 2 cycles; the BD slot's nop is - -- counted as part of the branch's cost) - ["branch_equal"] = 2, ["branch_ne"] = 2, - ["branch_le_zero"] = 2, ["branch_lt_zero"] = 2, - ["branch_ge_zero"] = 2, ["branch_gt_zero"] = 2, - -- Jumps (jump + BD slot nop = 2 cycles) - ["jump"] = 2, ["jump_reg"] = 2, - ["jump_link"] = 2, ["call_reg"] = 2, - ["call_addr"] = 2, - -- COP2 transfers (mtc2/mfc2/ctc2/cfc2 = 1 cycle + COP2 latency; the - -- COP2 latency is usually absorbed by subsequent nops or by the next - -- GTE command's pre-fill nops, so we count 1) - ["gte_mv_to_data_r"] = 1, - ["gte_mv_from_data_r"] = 1, - ["gte_mv_to_ctrl_r"] = 1, - ["gte_mv_from_ctrl_r"] = 1, - ["gte_lw"] = 1, ["gte_lwc2"] = 1, - ["gte_sw"] = 1, ["gte_swc2"] = 1, - -- COP2 commands (intrinsic cycles, EXCLUDING the 2 pre-cmd nops that - -- the source typically emits as `nop2, gte_cmdw_X`; those nops are - -- counted separately via the `nop2` entry above) - ["gte_cmdw_rtpt"] = 21, - ["gte_cmdw_rtps"] = 12, - ["gte_cmdw_nclip"] = 6, - ["gte_cmdw_avsz3"] = 12, - ["gte_cmdw_avsz4"] = 14, - ["gte_cmdw_mvmva"] = 6, - ["gte_cmdw_op"] = 5, - ["gte_cmdw_outer_product"] = 5, - ["gte_cmdw_wedge"] = 5, - -- Long-form aliases (same cost as canonical) - ["gte_cmdw_rotate_translate_perspective_single"] = 12, -- alias for rtps - ["gte_cmdw_rotate_translate_perspective_triple"] = 21, -- alias for rtpt - ["gte_cmdw_avg_sort_z4"] = 14, -- alias for avsz4 - -- Non-cmdw aliases from gte.h (these are `#define gte_X gte_cmdw_Y`): - ["gte_avg_sort_z3"] = 12, -- alias for avsz3 - ["gte_avg_sort_z4"] = 14, -- alias for avsz4 - ["gte_rtps"] = 12, -- alias for rtps - ["gte_rtpt"] = 21, -- alias for rtpt - ["gte_nclip"] = 6, -- alias for nclip - ["gte_avsz3"] = 12, - ["gte_avsz4"] = 14, - -- Legacy single-cycle store helpers (gte_stotz, gte_stsxy3 are 1 cycle) - ["gte_stotz"] = 1, - ["gte_stsxy3"] = 1, - -- High-level GTE helpers (gte_load_v0/v1/v2 do multiple lwc2s) - ["gte_load_v0"] = 2, -- 1 lwc2 for VXY0 + 1 for VZ0 - ["gte_load_v1"] = 2, - ["gte_load_v2"] = 2, - ["gte_load_v0v1v2"] = 6, - -- mac_* helpers (cycle cost = sum of the expanded instructions) - -- mac_yield transfers control; cycle budget is 0 (the next atom - -- absorbs the cost). - ["mac_yield"] = 0, - ["mac_pack_color_word"] = 3, -- lui + ori + sw - ["mac_format_f3_color"] = 3, -- = mac_pack_color_word - ["mac_format_g4_color"] = 12, -- 4 x mac_pack_color_word - ["mac_load_tri_indices"] = 3, -- 3 x lhu - ["mac_gte_load_tri_verts"] = 18, -- 3 x {sll, addu, lw, lw, mtc2, mtc2} - ["mac_gte_store_f3_post_rtpt"] = 3, - ["mac_gte_store_g3_post_rtpt"] = 3, - ["mac_gte_store_g4_p012_post_rtpt_pre_rtps"] = 3, - ["mac_gte_store_g4_p3_post_rtps"] = 1, - ["mac_insert_ot_tag_f3"] = 11, -- 11 .word slots in the macro body - ["mac_insert_ot_tag_g4"] = 11, - -- Annotation markers (emit no code; pure metaprogram hints) - ["atom_label"] = 0, - ["atom_offset"] = 0, - ["atom_info"] = 0, - ["atom_bind"] = 0, - ["atom_reads"] = 0, - ["atom_writes"] = 0, + -- CPU ALU (single-cycle R3000A ops) + ["nop"] = 1, + ["nop2"] = 2, + ["add_ui"] = 1, ["add_ui_self"] = 1, + ["add_s"] = 1, ["add_si"] = 1, + ["add_u"] = 1, ["add_u_self"] = 1, + ["sub_u"] = 1, ["sub_s"] = 1, + ["and_i"] = 1, ["and_u"] = 1, + ["or_i"] = 1, ["or_i_self"] = 1, + ["or_u"] = 1, ["or_u_self"] = 1, + ["xor_i"] = 1, ["xor_u"] = 1, + ["nor_u"] = 1, + ["shift_lleft"] = 1, ["shift_lleft_self"] = 1, + ["shift_lright"] = 1, + ["shift_aright"] = 1, + ["mask_upper"] = 1, + ["mov_from_high"] = 2, -- mfhi: 2 cycles + ["mov_from_low"] = 2, -- mflo: 2 cycles + ["mov_to_high"] = 1, -- mthi: 1 cycle + ["mov_to_low"] = 1, -- mtlo: 1 cycle + -- Set-on-condition (SLT family) + ["set_lt_u"] = 1, ["set_lt_ui"] = 1, + ["set_lt_s"] = 1, ["set_lt_si"] = 1, + -- Multiply / divide (no hardware multiplier; software via inline asm) + ["mult_u"] = 12, ["mult_s"] = 12, + ["div_u"] = 35, ["div_s"] = 35, + -- Loads (1 cycle + load-delay slot; the delay is typically absorbed by + -- the next instruction in a well-pipelined sequence, so we count 1) + ["load_word"] = 1, + ["load_half_u"] = 1, ["load_half"] = 1, + ["load_byte_u"] = 1, ["load_byte"] = 1, + ["load_upper_i"] = 1, + -- 2-word loads (lui + ori) used for >16-bit immediates + ["load_imm"] = 2, + ["load_imm_1w"] = 1, + ["load_imm_1w_s0"] = 1, + ["load_imm_2w"] = 2, + ["load_imm_2w_addi_forced"] = 2, + ["load_imm_2w_ori_forced"] = 2, + -- Stores (1 cycle each) + ["store_word"] = 1, + ["store_half"] = 1, + ["store_byte"] = 1, + -- Branches (branch + BD slot nop = 2 cycles; the BD slot's nop is + -- counted as part of the branch's cost) + ["branch_equal"] = 2, ["branch_ne"] = 2, + ["branch_le_zero"] = 2, ["branch_lt_zero"] = 2, + ["branch_ge_zero"] = 2, ["branch_gt_zero"] = 2, + -- Jumps (jump + BD slot nop = 2 cycles) + ["jump"] = 2, ["jump_reg"] = 2, + ["jump_link"] = 2, ["call_reg"] = 2, + ["call_addr"] = 2, + -- COP2 transfers (mtc2/mfc2/ctc2/cfc2 = 1 cycle + COP2 latency; the + -- COP2 latency is usually absorbed by subsequent nops or by the next + -- GTE command's pre-fill nops, so we count 1) + ["gte_mv_to_data_r"] = 1, + ["gte_mv_from_data_r"] = 1, + ["gte_mv_to_ctrl_r"] = 1, + ["gte_mv_from_ctrl_r"] = 1, + ["gte_lw"] = 1, ["gte_lwc2"] = 1, + ["gte_sw"] = 1, ["gte_swc2"] = 1, + -- COP2 commands (intrinsic cycles, EXCLUDING the 2 pre-cmd nops that + -- the source typically emits as `nop2, gte_cmdw_X`; those nops are + -- counted separately via the `nop2` entry above) + ["gte_cmdw_rtpt"] = 21, + ["gte_cmdw_rtps"] = 12, + ["gte_cmdw_nclip"] = 6, + ["gte_cmdw_avsz3"] = 12, + ["gte_cmdw_avsz4"] = 14, + ["gte_cmdw_mvmva"] = 6, + ["gte_cmdw_op"] = 5, + ["gte_cmdw_outer_product"] = 5, + ["gte_cmdw_wedge"] = 5, + -- Long-form aliases (same cost as canonical) + ["gte_cmdw_rotate_translate_perspective_single"] = 12, -- alias for rtps + ["gte_cmdw_rotate_translate_perspective_triple"] = 21, -- alias for rtpt + ["gte_cmdw_avg_sort_z4"] = 14, -- alias for avsz4 + -- Non-cmdw aliases from gte.h (these are `#define gte_X gte_cmdw_Y`): + ["gte_avg_sort_z3"] = 12, -- alias for avsz3 + ["gte_avg_sort_z4"] = 14, -- alias for avsz4 + ["gte_rtps"] = 12, -- alias for rtps + ["gte_rtpt"] = 21, -- alias for rtpt + ["gte_nclip"] = 6, -- alias for nclip + ["gte_avsz3"] = 12, + ["gte_avsz4"] = 14, + -- Legacy single-cycle store helpers (gte_stotz, gte_stsxy3 are 1 cycle) + ["gte_stotz"] = 1, + ["gte_stsxy3"] = 1, + -- High-level GTE helpers (gte_load_v0/v1/v2 do multiple lwc2s) + ["gte_load_v0"] = 2, -- 1 lwc2 for VXY0 + 1 for VZ0 + ["gte_load_v1"] = 2, + ["gte_load_v2"] = 2, + ["gte_load_v0v1v2"] = 6, + -- mac_* helpers (cycle cost = sum of the expanded instructions) + -- mac_yield transfers control; cycle budget is 0 (the next atom + -- absorbs the cost). + ["mac_yield"] = 0, + ["mac_pack_color_word"] = 3, -- lui + ori + sw + ["mac_format_f3_color"] = 3, -- = mac_pack_color_word + ["mac_format_g4_color"] = 12, -- 4 x mac_pack_color_word + ["mac_load_tri_indices"] = 3, -- 3 x lhu + ["mac_gte_load_tri_verts"] = 18, -- 3 x {sll, addu, lw, lw, mtc2, mtc2} + ["mac_gte_store_f3_post_rtpt"] = 3, + ["mac_gte_store_g3_post_rtpt"] = 3, + ["mac_gte_store_g4_p012_post_rtpt_pre_rtps"] = 3, + ["mac_gte_store_g4_p3_post_rtps"] = 1, + ["mac_insert_ot_tag_f3"] = 11, -- 11 .word slots in the macro body + ["mac_insert_ot_tag_g4"] = 11, + -- Annotation markers (emit no code; pure metaprogram hints) + ["atom_label"] = 0, + ["atom_offset"] = 0, + ["atom_info"] = 0, + ["atom_bind"] = 0, + ["atom_reads"] = 0, + ["atom_writes"] = 0, } -- Default cycle cost for unknown macros. The static-analysis pass adds 1 diff --git a/scripts/duffle_paths.lua b/scripts/duffle_paths.lua new file mode 100644 index 0000000..957b52a --- /dev/null +++ b/scripts/duffle_paths.lua @@ -0,0 +1,75 @@ +--- duffle_paths.lua — Single-line bootstrap helper for the tape-atom +--- Lua scripts. +--- +--- Each entry script (ps1_meta.lua, word_count_eval.lua, and the 5 +--- passes/*.lua files) starts with: +--- +--- ```lua +--- local duffle = dofile((arg[0]:match("(.*[/\\])") or "./") .. "duffle_paths.lua") +--- ``` +--- +--- That single line: (a) locates this helper via `arg[0]`, (b) loads +--- it (which sets `package.path` + `package.cpath` via `git rev-parse`), +--- (c) returns the `M` table (a wrapper around the setup function). +--- After this line, `require("duffle")` and `require("passes.X")` both +--- resolve normally. +--- +--- **Why a helper instead of inline?** +--- - The 8-line path-setup boilerplate was duplicated across 7 entry +--- scripts (one per file). Single source of truth here. +--- - Mirrors the build script's pattern in `build_psyq.ps1`: +--- `$path_root = split-path -Path $PSScriptRoot -Parent;` then +--- derive everything from there. +--- +--- **Why `git rev-parse --show-toplevel`?** +--- Hardcoding `C:\\projects\\Pikuma\\ps1\\...` breaks portability. Git +--- gives us the canonical repo root regardless of where the repo lives +--- on disk. + +local M = {} + +--- Resolve the repo root via git. Returns a normalized path with a +--- trailing forward-slash, or nil if not in a git repo. +--- @return string|nil +local function find_repo_root() + local p = io.popen("git rev-parse --show-toplevel 2>nul") + local root + if p then + root = p:read("*l") + p:close() + end + if not root or root == "" then return nil end + -- Normalize to forward slashes (Windows accepts both, but mixed + -- `\` + `/` confuses LuaJIT's file APIs). + root = root:gsub("\\", "/") + if not root:match("/$") then root = root .. "/" end + return root +end + +--- Set `package.path` (for `require("duffle")` + `require("passes.X")`) +--- and `package.cpath` (for `lpeg.dll` on Windows). +function M.setup() + local repo_root = find_repo_root() + if not repo_root then + io.stderr:write("[duffle_paths] git rev-parse failed -- not in a git repo?\n") + os.exit(2) + end + + local scripts_dir = repo_root .. "scripts/" + local passes_dir = repo_root .. "scripts/passes/" + package.path = scripts_dir .. "?.lua;" + .. scripts_dir .. "?/init.lua;" + .. passes_dir .. "?.lua;" + .. passes_dir .. "?/init.lua;" + .. package.path + + if package.config:sub(1, 1) == "\\" then + package.cpath = repo_root .. "toolchain/luajit-2.1/lib/lua/5.1/?.dll;" + .. package.cpath + end +end + +-- Run the setup as a side effect. +M.setup() + +return M \ No newline at end of file diff --git a/scripts/passes/annotation.lua b/scripts/passes/annotation.lua index aaed5a1..089b643 100644 --- a/scripts/passes/annotation.lua +++ b/scripts/passes/annotation.lua @@ -1,32 +1,33 @@ --- passes/annotation.lua --- --- Validate atom annotation DSL usage in source files. Reads: --- - atom_annot / atom_init / atom_setup / atom_commit / atom_bind --- / atom_terminate / atom_label / atom_offset / TAPE_WORDS --- - atom_resource / atom_region / atom_group / atom_cadence / atom_async --- - Binds_* struct declarations --- Writes: --- - /.errors.h (with #error directives on findings) --- - /.annotations.txt (human-readable summary) --- Ported from scripts/tape_atom_annotation_pass.lua:78-545 + 1081-1407 --- (validation only — NOT rendering, which goes to passes/report.lua). --- --- Coding standard: tabs (1/level), EmmyLua annotations, no regex. +--- passes/annotation.lua — Atom-annotation DSL validator. +--- +--- Validates `MipsAtom_(name) atom_info(atom_bind(Binds_X), +--- atom_reads(...), atom_writes(...)) { ... }` declarations in source files. +--- Also reads: +--- - `Binds_*` struct declarations (`typedef Struct_(Binds_X) { ... };`) +--- - `TAPE_WORDS(mac_X, N)` pragma directives (`#pragma` + `_Pragma`) +--- +--- Writes: +--- - `/.errors.h` — one per module, with +--- `#error` directives on findings (the C compile will surface the +--- error) +--- - The annotations.txt report is rendered by `passes/report.lua` +--- from the per-module results stashed in `ctx.flags._annot_results` +--- +--- **Ported from** `scripts/tape_atom_annotation_pass.lua:78-545 + +--- 1081-1407` (validation only — NOT rendering, which goes to +--- `passes/report.lua`). +--- +--- **Conventions**: tabs (1/level), EmmyLua annotations, no regex, +--- Lua 5.3 compatible. See +--- `C:\projects\Pikuma\ps1-ai\conductor\code_styleguides\lua.md`. -- ════════════════════════════════════════════════════════════════════════════ -- Module-scope requires + package.path setup -- ════════════════════════════════════════════════════════════════════════════ -local script_path = arg and arg[0] or "?" -local last_sep = 0 -for i = 1, #script_path do - local c = script_path:sub(i, i) - if c == "/" or c == "\\" then last_sep = i end -end -local script_dir = last_sep == 0 and "./" or script_path:sub(1, last_sep) -package.path = script_dir .. "../?.lua;" .. script_dir .. "../?/init.lua;" .. script_dir .. "?.lua;" .. package.path - -local duffle = require("duffle") +-- Bootstrap: same as entry scripts. See `ps1_meta.lua` for the rationale. +dofile((arg[0]:match("(.*[/\\])") or "./") .. "duffle_paths.lua") +local duffle = require("duffle") local is_space = duffle.is_space local is_alpha = duffle.is_alpha local is_alnum = duffle.is_alnum @@ -52,50 +53,166 @@ local TAPE_ATOM_MACROS = duffle.TAPE_ATOM_MACROS local function is_wave_context_reg(n) return WAVE_CONTEXT_REGS[n] ~= nil end -- ════════════════════════════════════════════════════════════════════════════ +-- Constants +-- ════════════════════════════════════════════════════════════════════════════ + +-- Atom declaration + annotation identifiers. +local ATOM_DECL = "MipsAtom_" +local ATOM_INFO = "atom_info" +local STRUCT_TYPE = "Struct_" +local PRAGMA_IDENT = "pragma" +local PRAGMA_OPERATOR = "_Pragma" + +-- Struct-name prefix + byte size of U4 fields. +local BINDS_PREFIX = "Binds_" +local BINDS_PREFIX_LEN = 6 -- = #BINDS_PREFIX +local U4_TYPE = "U4" +local U4_BYTES = 4 -- sizeof(U4) +local BINDS_FIELD_PREFIX = "R_" -- wave-context register name prefix + +-- TAPE_WORDS pragma keys (the third token after #pragma). +local WORDS_KEY = "words" +local WORDS_KEY_PREFIX = "words=" -- the per-macro `words=N` form +local WORDS_KEY_PREFIX_LEN = 6 -- = #WORDS_KEY_PREFIX +local TAPE_ATOM_WORDS_KEY = "tape_atom words" -- the _Pragma form + +-- ASCII byte values used in tokenization. +local BYTE_NEWLINE = 10 +local BYTE_SPACE = 32 +local BYTE_DQUOTE = 34 +local BYTE_EQUALS = 61 +local BYTE_OPEN_PAREN = 40 +local BYTE_OPEN_BRACE = 123 +local BYTE_OPEN_BRACK = 91 +local BYTE_CLOSE_PAREN = 41 +local BYTE_CLOSE_BRACE = 125 +local BYTE_CLOSE_BRACK = 93 +local BYTE_COMMA = 44 + +-- ════════════════════════════════════════════════════════════════════════════ +-- Type declarations +-- ════════════════════════════════════════════════════════════════════════════ + +--- @class SourceFile +--- @field path string -- absolute path to the source file +--- @field text string -- the full source text +--- @field dir string -- the directory containing the source +--- @field basename string -- filename without extension + +--- @class PassCtx +--- @field sources SourceFile[] +--- @field metadata_path string +--- @field shared table +--- @field shared.word_counts table +--- @field out_root string +--- @field project_root string +--- @field upstream table +--- @field flags table +--- @field flags._annot_results table[] -- stashed by annotation pass; consumed by report.lua +--- @field dry_run boolean +--- @field verbose boolean + +--- @class PassResult +--- @field outputs table[] +--- @field errors table[] +--- @field warnings table[] + +--- @class Atom +--- @field line integer -- source line of the MipsAtom_ declaration +--- @field name string -- atom name (e.g. "cube_g4_face") + +--- @class AtomAnnotation +--- @field line integer -- source line of the atom_info call +--- @field macro string -- the macro name (always "atom_info" in the new shape) +--- @field name string -- the atom name +--- @field kind string -- always "info" +--- @field binds string|nil -- Binds_X name if any +--- @field reads string[] -- R_* names (read targets) +--- @field writes string[] -- R_* names (write targets) +--- @field error string|nil -- error message if annotation was malformed +--- @field errors string[] -- nested errors from per-arg validation + +--- @class BindsField +--- @field name string -- field name +--- @field offset integer -- byte offset within the Binds_X struct + +--- @class BindsStruct +--- @field name string -- struct name (e.g. "Binds_Floor") +--- @field line integer -- source line of the typedef +--- @field bytes integer -- total byte size +--- @field fields BindsField[] -- the field list + +--- @class MacroEntry +--- @field name string -- macro name (e.g. "mac_format_f3_color") +--- @field line integer -- source line of the TAPE_WORDS pragma +--- @field words integer -- declared word count + +--- @class Finding +--- @field line integer -- source line (or 0 for pass-level) +--- @field msg string -- finding message + +--- @class AnnotatedResult +--- @field atoms Atom[] +--- @field annots AtomAnnotation[] +--- @field macros MacroEntry[] +--- @field binds BindsStruct[] +--- @field errors Finding[] +--- @field warnings Finding[] +--- @field info Finding[] +--- @field pragmas table -- reserved (currently always nil; legacy compat) + +--- ════════════════════════════════════════════════════════════════════════════ -- Hand-rolled split helpers (no regex patterns used) -- ════════════════════════════════════════════════════════════════════════════ --- Split a string at top-level commas. Used inside TAPE_ATOM_* macro --- bodies where nested parens/braces/brackets are possible. +--- @param s string +--- @return string[] local function split_csv_top(s) - local tokens = {} - local i, start = 1, 1 - local depth = 0 - while i <= #s do - local c = s:sub(i, i) - if c == "(" or c == "{" or c == "[" then + local tokens = {} + local pos = 1 + local chunk_a = 1 + local depth = 0 + local str_len = #s + while pos <= str_len do + local ch = s:byte(pos) + if ch == BYTE_OPEN_PAREN or ch == BYTE_OPEN_BRACE or ch == BYTE_OPEN_BRACK then depth = depth + 1 - i = i + 1 - elseif c == ")" or c == "}" or c == "]" then + pos = pos + 1 + elseif ch == BYTE_CLOSE_PAREN or ch == BYTE_CLOSE_BRACE or ch == BYTE_CLOSE_BRACK then depth = depth - 1 - i = i + 1 - elseif c == "," and depth == 0 then - tokens[#tokens + 1] = s:sub(start, i - 1) - i = i + 1 - start = i + pos = pos + 1 + elseif ch == BYTE_COMMA and depth == 0 then + tokens[#tokens + 1] = s:sub(chunk_a, pos - 1) + pos = pos + 1 + chunk_a = pos else - i = i + 1 + pos = pos + 1 end end - local last = s:sub(start) + local last = s:sub(chunk_a) if trim(last) ~= "" then tokens[#tokens + 1] = last end return tokens end --- Split a string into whitespace-separated tokens. --- Hand-rolled (no regex patterns). +--- @param s string +--- @return string[] local function split_ws(s) local tokens = {} - local i, n = 1, 1 - local len = #s - while i <= len do + local pos = 1 + local n = 1 + local len = #s + while pos <= len do -- Skip whitespace. - while i <= len and is_space(s:sub(i, i)) do i = i + 1 end - if i > len then break end - local start = i + while pos <= len and is_space(s:sub(pos, pos)) do pos = pos + 1 end + if pos > len then break end + local chunk_a = pos -- Take non-whitespace run. - while i <= len and not is_space(s:sub(i, i)) do i = i + 1 end - tokens[n] = s:sub(start, i - 1) + while pos <= len and not is_space(s:sub(pos, pos)) do pos = pos + 1 end + tokens[n] = s:sub(chunk_a, pos - 1) n = n + 1 end return tokens @@ -196,75 +313,120 @@ end -- Parse TAPE_WORDS(mac_X, N) pragma directives -- ════════════════════════════════════════════════════════════════════════════ ---- Find every TAPE_WORDS(mac_X, N) pragma in source. +--- Skip preprocessor directives (lines starting with `#`). +--- Returns the position past the newline at the end of the line. +--- @param source string +--- @param pos integer +--- @return integer +local function skip_preprocessor_line(source, pos) + local str_len = #source + local scan = pos + while scan <= str_len and source:byte(scan) ~= BYTE_NEWLINE do + scan = scan + 1 + end + return scan + 1 +end + +--- Parse `_Pragma("mac_X tape_atom words=N")` (operator form). +--- @param source string +--- @param ident_pos integer -- position of the `_Pragma` ident +--- @param after_ident integer -- position just past the ident +--- @return MacroEntry|nil, integer -- (entry or nil, new source position) +local function parse_pragma_operator(source, ident_pos, after_ident) + local open_paren = skip_ws_and_cmt(source, after_ident) + if source:byte(open_paren) ~= BYTE_OPEN_PAREN then + return nil, open_paren + 1 + end + local str, str_end = read_parens(source, open_paren) + str = trim(str) + if str:sub(1, 1) ~= '"' or str:sub(-1) ~= '"' then + return nil, str_end + end + local inner = str:sub(2, -2) + local space = find_byte(inner, BYTE_SPACE, 1) + if not space then return nil, str_end end + local name = inner:sub(1, space - 1) + local rest = inner:sub(space + 1) + local eq = find_byte(rest, BYTE_EQUALS, 1) + if not eq then return nil, str_end end + local key = trim(rest:sub(1, eq - 1)) + local val = trim(rest:sub(eq + 1)) + if key ~= TAPE_ATOM_WORDS_KEY and key ~= WORDS_KEY then return nil, str_end end + return { + line = source:sub(1, ident_pos) and 0 or 0, -- see line_of below + name = name, + words = tonumber(val) or 0, + }, str_end +end + +--- Parse `#pragma mac_X tape_atom words=N` (directive form). +--- @param source string +--- @param ident_pos integer -- position of the `pragma` ident +--- @param after_ident integer -- position just past the ident +--- @return MacroEntry|nil, integer -- (entry or nil, new source position) +local function parse_pragma_directive(source, ident_pos, after_ident) + local str_len = #source + local rest_start = skip_ws_and_cmt(source, after_ident) + local eol = rest_start + while eol <= str_len and source:byte(eol) ~= BYTE_NEWLINE do + eol = eol + 1 + end + local line_text = trim(source:sub(rest_start, eol - 1)) + local tokens = split_ws(line_text) + local entry + if #tokens >= 3 and tokens[2] == "tape_atom" and tokens[3]:sub(1, WORDS_KEY_PREFIX_LEN) == WORDS_KEY_PREFIX then + entry = { + name = tokens[1], + words = tonumber(tokens[3]:sub(WORDS_KEY_PREFIX_LEN + 1)) or 0, + } + elseif #tokens >= 2 and tokens[2]:sub(1, WORDS_KEY_PREFIX_LEN) == WORDS_KEY_PREFIX then + entry = { + name = tokens[1], + words = tonumber(tokens[2]:sub(WORDS_KEY_PREFIX_LEN + 1)) or 0, + } + end + if entry then + local line_of = duffle.LineIndex(source) + entry.line = line_of(ident_pos) + end + return entry, eol +end + +--- Find every `TAPE_WORDS(mac_X, N)` pragma in source. --- Accepts both forms: ---- _Pragma("mac_X tape_atom words=N") (operator form) ---- #pragma mac_X tape_atom words=N (directive form) +--- `_Pragma("mac_X tape_atom words=N")` (operator form) +--- `#pragma mac_X tape_atom words=N` (directive form) +--- @param source string +--- @return MacroEntry[] local function find_macro_word_annotations(source) - local line_of = duffle.LineIndex(source) - local out = {} - local len = #source - local i = 1 - while i <= len do - i = skip_ws_and_cmt(source, i); if i > len then break end + local out = {} + local pos = 1 + local str_len = #source + while pos <= str_len do + pos = skip_ws_and_cmt(source, pos) + if pos > str_len then break end + -- Skip preprocessor directives (lines starting with #). - if source:sub(i, i) == "#" then - local j = i - while j <= len and source:sub(j, j) ~= "\n" do j = j + 1 end - i = j + 1 + if source:byte(pos) == 35 then -- '#' + pos = skip_preprocessor_line(source, pos) else - local ident, after = read_ident(source, i) + local ident, after_ident = read_ident(source, pos) if not ident then - i = i + 1 - elseif ident == "_Pragma" then - local open = skip_ws_and_cmt(source, after) - if source:sub(open, open) == "(" then - local str, str_end = read_parens(source, open) - str = trim(str) - if str:sub(1, 1) == '"' and str:sub(-1) == '"' then - local inner = str:sub(2, -2) - local space = find_byte(inner, " ", 1) - if space then - local name = inner:sub(1, space - 1) - local rest = inner:sub(space + 1) - local eq = find_byte(rest, "=", 1) - if eq then - local key = trim(rest:sub(1, eq - 1)) - local val = trim(rest:sub(eq + 1)) - if key == "tape_atom words" or key == "words" then - local n = tonumber(val) or 0 - out[#out + 1] = { - line = line_of(i), - name = name, - words = n, - } - end - end - end - end - i = str_end - else - i = open + 1 + pos = pos + 1 + elseif ident == PRAGMA_OPERATOR then + local entry, new_pos = parse_pragma_operator(source, pos, after_ident) + if entry then + local line_of = duffle.LineIndex(source) + entry.line = line_of(pos) + out[#out + 1] = entry end - elseif ident == "pragma" then - -- Directive form: `#pragma mac_X tape_atom words=N` - local rest_start = skip_ws_and_cmt(source, after) - local j = rest_start - while j <= len and source:sub(j, j) ~= "\n" do j = j + 1 end - local line_text = trim(source:sub(rest_start, j - 1)) - local tokens = split_ws(line_text) - if #tokens >= 3 and tokens[2] == "tape_atom" and tokens[3]:sub(1, 6) == "words=" then - local name = tokens[1] - local n = tonumber(tokens[3]:sub(7)) or 0 - out[#out + 1] = { line = line_of(i), name = name, words = n } - elseif #tokens >= 2 and tokens[2]:sub(1, 6) == "words=" then - local name = tokens[1] - local n = tonumber(tokens[2]:sub(7)) or 0 - out[#out + 1] = { line = line_of(i), name = name, words = n } - end - i = j + pos = new_pos + elseif ident == PRAGMA_IDENT then + local entry, new_pos = parse_pragma_directive(source, pos, after_ident) + if entry then out[#out + 1] = entry end + pos = new_pos else - i = after + pos = after_ident end end end @@ -275,76 +437,105 @@ end -- Parse `typedef Struct_(Binds_X) { ... };` declarations -- ════════════════════════════════════════════════════════════════════════════ ---- Find every Binds_* struct declaration. ---- Returns a list of {line, name, fields = {{name, byte_offset}, ...}}. +--- Walk a `Binds_X` body string and extract U4 fields (name + byte offset). +--- Only U4 fields are tracked (Binds_* are always word arrays in this +--- codebase -- pointers stored as U4, indices as U4, etc.). +--- @param body string -- the brace-delimited body (without the braces) +--- @return BindsField[] -- field list +--- @return integer -- total byte size +local function parse_binds_body(body) + local fields = {} + local byte_off = 0 + local pos = 1 + local body_len = #body + while pos <= body_len do + pos = skip_ws_and_cmt(body, pos) + if pos > body_len then break end + local type_ident, type_after = read_ident(body, pos) + if not type_ident then + pos = pos + 1 + elseif type_ident == U4_TYPE then + local field_after = skip_ws_and_cmt(body, type_after) + local fid, fafter = read_ident(body, field_after) + if fid then + fields[#fields + 1] = { name = fid, offset = byte_off } + byte_off = byte_off + U4_BYTES + end + pos = fafter or (type_after + 1) + else + pos = type_after + 1 + end + end + return fields, byte_off +end + +--- Try to parse a `typedef Struct_(Binds_X) { ... };` declaration. +--- Returns the parsed BindsStruct (if the form matched) and the new +--- source position. If the form didn't match, returns nil + a position +--- to continue scanning from. +--- @param source string +--- @param ident_pos integer -- position of the `typedef` ident start +--- @param after_typedef integer -- position just past `typedef` +--- @param line_of fun(pos: integer): integer +--- @return BindsStruct|nil, integer +local function parse_typedef_binds(source, ident_pos, after_typedef, line_of) + local after_type = skip_ws_and_cmt(source, after_typedef) + local type_ident, after_type_ident = read_ident(source, after_type) + if type_ident ~= STRUCT_TYPE then + return nil, after_type_ident or (after_type + 1) + end + + local open_paren = skip_ws_and_cmt(source, after_type_ident) + if source:byte(open_paren) ~= BYTE_OPEN_PAREN then + return nil, open_paren + 1 + end + + local inner, after_paren = read_parens(source, open_paren) + local name = trim(inner) + + local brace = scan_to_char(source, "{", after_paren) + if not brace then return nil, open_paren + 1 end + + local body, after_brace = read_braces(source, brace) + local fields, bytes = parse_binds_body(body) + + -- Only emit Binds_* structs (other Struct_ typedefs are ignored). + if name:sub(1, BINDS_PREFIX_LEN) ~= BINDS_PREFIX then + return nil, after_brace + end + + return { + line = line_of(ident_pos), + name = name, + fields = fields, + bytes = bytes, + }, after_brace +end + +--- Find every `Binds_*` struct declaration. +--- @param source string +--- @return BindsStruct[] local function find_binds_structs(source) local line_of = duffle.LineIndex(source) - local out = {} - local len = #source - local i = 1 - while i <= len do - i = skip_ws_and_cmt(source, i); if i > len then break end - if source:sub(i, i) == "#" then - local j = i - while j <= len and source:sub(j, j) ~= "\n" do j = j + 1 end - i = j + 1 + local out = {} + local pos = 1 + local str_len = #source + while pos <= str_len do + pos = skip_ws_and_cmt(source, pos) + if pos > str_len then break end + + if source:byte(pos) == 35 then -- '#' + pos = skip_preprocessor_line(source, pos) else - local ident, after = read_ident(source, i) + local ident, after_ident = read_ident(source, pos) if not ident then - i = i + 1 + pos = pos + 1 elseif ident == "typedef" then - local j = skip_ws_and_cmt(source, after) - local id2, after2 = read_ident(source, j) - if id2 ~= "Struct_" then - i = after2 or (j + 1) - elseif id2 == "Struct_" then - local open = skip_ws_and_cmt(source, after2) - if source:sub(open, open) == "(" then - local inner, after_paren = read_parens(source, open) - local name = trim(inner) - local brace = scan_to_char(source, "{", after_paren) - if brace then - local body, after_brace = read_braces(source, brace) - local fields = {} - local byte_off = 0 - local k = 1 - while k <= #body do - k = skip_ws_and_cmt(body, k); if k > #body then break end - local tid, tafter = read_ident(body, k) - if not tid then - k = k + 1 - elseif tid == "U4" then - local fid, fafter = read_ident(body, skip_ws_and_cmt(body, tafter)) - if fid then - fields[#fields + 1] = { name = fid, offset = byte_off } - byte_off = byte_off + 4 - end - k = fafter or tafter + 1 - else - k = tafter + 1 - end - end - -- Only emit Binds_* structs. - if name:sub(1, 6) == "Binds_" then - out[#out + 1] = { - line = line_of(i), - name = name, - fields = fields, - bytes = byte_off, - } - end - i = after_brace - else - i = open + 1 - end - else - i = open + 1 - end - else - i = after2 or (j + 1) - end + local binds_struct, new_pos = parse_typedef_binds(source, pos, after_ident, line_of) + if binds_struct then out[#out + 1] = binds_struct end + pos = new_pos else - i = after + pos = after_ident end end end @@ -355,40 +546,58 @@ end -- Find every MipsAtom_(name) { ... } declaration in source -- ════════════════════════════════════════════════════════════════════════════ +--- Read the next identifier token from `s` starting at `pos`, where the +--- identifier is a contiguous run of `[a-zA-Z0-9_]` characters (no +--- underscore-starting alpha-only constraint). Returns the ident + the +--- position just past it, or nil + pos if no identifier starts there. +--- @param s string +--- @param pos integer +--- @return string|nil, integer +local function read_alnum_ident(s, pos) + local str_len = #s + while pos <= str_len and is_space(s:sub(pos, pos)) do pos = pos + 1 end + local start = pos + while pos <= str_len and is_alnum(s:sub(pos, pos)) do pos = pos + 1 end + if pos == start then return nil, pos end + return s:sub(start, pos - 1), pos +end + +--- Find every `MipsAtom_(name)` declaration in source. (Just the name + +--- source line; the body is parsed separately by `parse_mips_atom`.) +--- @param source string +--- @return Atom[] local function find_atom_names(source) local line_of = duffle.LineIndex(source) - local out = {} - local len = #source - local i = 1 - while i <= len do - i = skip_ws_and_cmt(source, i); if i > len then break end - local ident, after = read_ident(source, i) + local out = {} + local pos = 1 + local str_len = #source + while pos <= str_len do + pos = skip_ws_and_cmt(source, pos) + if pos > str_len then break end + + local ident, after_ident = read_ident(source, pos) if not ident then - i = i + 1 - elseif ident == "MipsAtom_" then - local open = skip_ws_and_cmt(source, after) - if source:sub(open, open) == "(" then - local inner, after_paren = read_parens(source, open) - local a = 1 - while a <= #inner and is_space(inner:sub(a, a)) do a = a + 1 end - local b = a - while b <= #inner and is_alnum(inner:sub(b, b)) do b = b + 1 end - local name = inner:sub(a, b - 1) - if name ~= "" then - out[#out + 1] = { line = line_of(i), name = name } + pos = pos + 1 + elseif ident ~= ATOM_DECL then + pos = after_ident + else + local open_paren = skip_ws_and_cmt(source, after_ident) + if source:byte(open_paren) ~= BYTE_OPEN_PAREN then + pos = open_paren + 1 + else + local inner, after_paren = read_parens(source, open_paren) + local name, _ = read_alnum_ident(inner, 1) + if name and name ~= "" then + out[#out + 1] = { line = line_of(pos), name = name } end local brace = scan_to_char(source, "{", after_paren) if brace then local _, after_brace = read_braces(source, brace) - i = after_brace + pos = after_brace else - i = open + 1 + pos = open_paren + 1 end - else - i = open + 1 end - else - i = after end end return out @@ -451,72 +660,76 @@ local function new_annot_entry(line, ident, name, kind) } end ---- Find every MipsAtom_(name) declaration in source, then look for an ---- immediately-following atom_info(...) call. If present, parse its ---- sub-calls into a normalized annotation entry linked to the MipsAtom_ ---- name. If no atom_info follows, emit NO annotation entry (atoms ---- without annotations are valid in the new minimal shape). +--- Try to parse an `atom_info(...)` call right after the `MipsAtom_(name)` +--- parens. Returns the annotation entry (if present) and the new source +--- position past the atom_info call. Returns nil if no atom_info follows. +--- @param source string +--- @param atom_name string +--- @param after_mipsatom_paren integer -- position past the MipsAtom_(...) close paren +--- @param line_of fun(pos: integer): integer +--- @return AtomAnnotation|nil, integer -- (entry or nil, new source position) +local function parse_atom_info_call(source, atom_name, after_mipsatom_paren, line_of) + local lookahead = skip_ws_and_cmt(source, after_mipsatom_paren) + local look_ident, look_after = read_ident(source, lookahead) + if look_ident ~= ATOM_INFO then return nil, after_mipsatom_paren end + + local info_open = skip_ws_and_cmt(source, look_after) + if source:byte(info_open) ~= BYTE_OPEN_PAREN then + return nil, info_open + 1 + end + + local info_inner, info_after = read_parens(source, info_open) + local args = parse_atom_annot_args(info_inner) + local entry = new_annot_entry(line_of(lookahead), ATOM_INFO, atom_name, "info") + ANNOT_ARG_HANDLERS.info(entry, args) + return entry, info_after +end + +--- Find every `MipsAtom_(name) atom_info(...) { ... };` annotation in source. +--- Returns a list of annotation entries. Atoms without a following +--- `atom_info(...)` call produce NO entry (atoms without annotations are +--- valid in the new minimal shape). +--- @param source string +--- @return AtomAnnotation[] local function find_atom_annotations(source) local line_of = duffle.LineIndex(source) - local annots = {} - local len = #source - local i = 1 - while i <= len do - i = skip_ws_and_cmt(source, i); if i > len then break end + local annots = {} + local pos = 1 + local str_len = #source + while pos <= str_len do + pos = skip_ws_and_cmt(source, pos) + if pos > str_len then break end + -- Skip preprocessor directives (lines starting with #). - if source:sub(i, i) == "#" then - local j = i - while j <= len and source:sub(j, j) ~= "\n" do j = j + 1 end - i = j + 1 - goto continue - end - - local ident, after = read_ident(source, i) - if not ident then - i = i + 1 - elseif ident == "MipsAtom_" then - local open = skip_ws_and_cmt(source, after) - if source:sub(open, open) ~= "(" then - i = open + 1 - goto continue - end - local inner, after_paren = read_parens(source, open) - local a = 1 - while a <= #inner and is_space(inner:sub(a, a)) do a = a + 1 end - local b = a - while b <= #inner and is_alnum(inner:sub(b, b)) do b = b + 1 end - local name = inner:sub(a, b - 1) - - -- Look for atom_info(...) right after MipsAtom_(name). - local lookahead = skip_ws_and_cmt(source, after_paren) - local look_ident, look_after = read_ident(source, lookahead) - if look_ident == "atom_info" then - local info_open = skip_ws_and_cmt(source, look_after) - if source:sub(info_open, info_open) == "(" then - local info_inner, info_after = read_parens(source, info_open) - local args = parse_atom_annot_args(info_inner) - local entry = new_annot_entry(line_of(lookahead), "atom_info", name, "info") - ANNOT_ARG_HANDLERS.info(entry, args) - annots[#annots + 1] = entry - i = info_after + if source:byte(pos) == 35 then -- '#' + pos = skip_preprocessor_line(source, pos) + else + local ident, after_ident = read_ident(source, pos) + if not ident then + pos = pos + 1 + elseif ident == ATOM_DECL then + local open_paren = skip_ws_and_cmt(source, after_ident) + if source:byte(open_paren) ~= BYTE_OPEN_PAREN then + pos = open_paren + 1 else - i = info_open + 1 + local inner, after_paren = read_parens(source, open_paren) + local name, _ = read_alnum_ident(inner, 1) + + local entry, new_pos = parse_atom_info_call(source, name, after_paren, line_of) + if entry then annots[#annots + 1] = entry end + pos = new_pos + + -- Skip past the body { ... } if present. + local brace = scan_to_char(source, "{", pos) + if brace then + local _, after_brace = read_braces(source, brace) + pos = after_brace + end end else - -- No atom_info follows this MipsAtom_. Valid in new shape. - i = after_paren + pos = after_ident end - - -- Skip past the body { ... } if present. - local brace = scan_to_char(source, "{", i) - if brace then - local _, after_brace = read_braces(source, brace) - i = after_brace - end - else - i = after end - ::continue:: end return annots end diff --git a/scripts/passes/components.lua b/scripts/passes/components.lua index 239a8c1..b6de5f5 100644 --- a/scripts/passes/components.lua +++ b/scripts/passes/components.lua @@ -1,12 +1,19 @@ --- passes/components.lua --- --- Generate /gen/.macs.h from MipsAtomComp_ declarations --- in source files. Ported from tape_atom_annotation_pass.lua:604-1079 --- (find_component_atoms, preceding_comment_block, extract_arg_names, --- convert_line_comments_to_block, compute_component_word_count, --- emit_component_macros_h). --- --- Coding standard: tabs (1/level), EmmyLua annotations, no regex. +--- passes/components.lua — Component-macro header generator. +--- +--- Walks every source for `MipsAtomComp_(ac_X) { body }` (and the +--- function-form `MipsAtomComp_Proc_(ac_X, { body })`) declarations +--- and emits a per-directory `.macs.h` containing one +--- `#define mac_X(sig) \` macro per component + `WORD_COUNT(mac_X, N)` +--- entries for downstream offset computation. +--- +--- **Ported from** `tape_atom_annotation_pass.lua:604-1079` +--- (`find_component_atoms`, `preceding_comment_block`, +--- `extract_arg_names`, `convert_line_comments_to_block`, +--- `compute_component_word_count`, `emit_component_macros_h`). +--- +--- **Conventions**: tabs (1/level), EmmyLua annotations, no regex, +--- Lua 5.3 compatible. See +--- `C:\projects\Pikuma\ps1-ai\conductor\code_styleguides\lua.md`. --- @class Component --- @field name string @@ -21,43 +28,80 @@ -- Module-scope requires + package.path setup -- ════════════════════════════════════════════════════════════════════════════ -local script_path = arg and arg[0] or "?" -local last_sep = 0 -for i = 1, #script_path do - local c = script_path:sub(i, i) - if c == "/" or c == "\\" then last_sep = i end -end -local script_dir = last_sep == 0 and "./" or script_path:sub(1, last_sep) -package.path = script_dir .. "../?.lua;" .. script_dir .. "../?/init.lua;" .. script_dir .. "?.lua;" .. package.path - -local duffle = require("duffle") -local trim = duffle.trim -local read_ident = duffle.read_ident -local is_space = duffle.is_space -local is_alpha = duffle.is_alpha -local is_alnum = duffle.is_alnum -local skip_ws_and_cmt = duffle.skip_ws_and_cmt -local skip_str_or_cmt = duffle.skip_str_or_cmt -local split_top_level_commas = duffle.split_top_level_commas -local ensure_dir = duffle.ensure_dir -local basename_no_ext = duffle.basename_no_ext -local dirname = duffle.dirname -local read_parens = duffle.read_parens -local read_braces = duffle.read_braces -local scan_to_char = duffle.scan_to_char - -local word_count_eval = require("word_count_eval") -local count_body_words = word_count_eval.count_body_words - -local M = {} +-- Resolve `arg[0]` to an absolute-ish script directory so that +-- `require("duffle")` resolves against `scripts/` regardless of CWD. +-- Note: this boilerplate is duplicated in 6 other entry scripts; a +-- Phase-6 extraction target (`duffle.setup_package_path()`). +-- Bootstrap: see `ps1_meta.lua` for the rationale. +dofile((arg[0]:match("(.*[/\\])") or "./") .. "duffle_paths.lua") +local duffle = require("duffle") +local word_count_eval = require("word_count_eval") -- ════════════════════════════════════════════════════════════════════════════ --- Local helpers +-- Constants +-- ════════════════════════════════════════════════════════════════════════════ + +-- Atom component declaration identifiers. +local ATOM_COMP_PROC = "MipsAtomComp_Proc_" +local ATOM_COMP = "MipsAtomComp_" +local MIPS_ATOM = "MipsAtom" -- prefix on the function declaration that wraps an AtomComp_Proc_ + +-- Component-name prefixes. +local AC_PREFIX = "ac_" -- arg to MipsAtomComp_(ac_X); the X is the atom name +local AC_PREFIX_LEN = 3 +local MAC_PREFIX = "mac_" -- prefix on generated macros; the rest is the atom name +local MAC_PREFIX_LEN = 4 + +-- ASCII byte values used in tokenization. +local BYTE_NEWLINE = 10 +local BYTE_SLASH = 47 + +-- Source dir basename used as the output `.macs.h` filename. +local GEN_SUBDIR = "gen" + +-- ════════════════════════════════════════════════════════════════════════════ +-- Type declarations +-- ════════════════════════════════════════════════════════════════════════════ + +--- @class SourceFile +--- @field path string -- absolute path to the source file +--- @field text string -- the full source text +--- @field dir string -- the directory containing the source +--- @field basename string -- filename without extension + +--- @class PassCtx +--- @field sources SourceFile[] -- all source files in the build +--- @field metadata_path string -- path to word_count.metadata.h +--- @field shared table -- cross-pass shared state +--- @field shared.word_counts table -- populated by word-counts + components +--- @field out_root string -- output root (e.g. "build/gen") +--- @field project_root string -- project root (e.g. "code/") +--- @field upstream table -- per-pass upstream outputs +--- @field flags table -- CLI flags +--- @field dry_run boolean -- if true, compute but don't write +--- @field verbose boolean -- if true, log diagnostic info + +--- @class PassResult +--- @field outputs table[] -- {kind=, path=} entries describing emit files +--- @field errors table[] -- {line=, msg=} entries; build-stops +--- @field warnings table[] -- {line=, msg=} entries; build-succeeds + +--- @class Component +--- @field name string -- atom name (without `ac_` prefix) +--- @field body string -- brace-delimited body (without the braces) +--- @field args string|nil -- function-args string (function form only) +--- @field line integer -- source line of the declaration +--- @field comment string|nil -- preceding `/* */` or `//` comment block (signature doc) + +-- ════════════════════════════════════════════════════════════════════════════ +-- Local helpers (file I/O + path normalization) -- ════════════════════════════════════════════════════════════════════════════ -- Write content to disk in binary mode so LF line endings are preserved on -- Windows (text mode would convert LF -> CRLF, breaking byte-identical diffs -- against git-tracked gen/*.macs.h files which are stored as LF). +-- @param path string +-- @param content string local function write_file_lf(path, content) local f = io.open(path, "wb") if not f then error("Cannot write " .. path) end @@ -70,6 +114,8 @@ end -- (e.g. "C:\projects\Pikuma\ps1\code\duffle\lottes_tape.h"); if we want -- byte-identical output, we must normalize relative -> absolute before -- emitting that comment. +-- @param path string +-- @return string local function to_absolute_path(path) if #path >= 2 and path:sub(2, 2) == ":" then -- Already absolute; normalize slashes for consistency. @@ -87,429 +133,534 @@ local function to_absolute_path(path) return cwd .. "\\" .. tail end +local M = {} + -- ════════════════════════════════════════════════════════════════════════════ -- Ported helpers (verbatim from tape_atom_annotation_pass.lua:604-1079) -- ════════════════════════════════════════════════════════════════════════════ --- ============================================================ --- Find the args of the function declaration that immediately precedes --- a MipsAtomComp_Proc_ invocation of the given name. Returns the --- args string (e.g., "U4 off, U4 code, U1 r, U1 g, U1 b") or nil --- if no function declaration is found. --- --- Convention: function form is --- FI_ MipsAtom ac_X(args) MipsAtomComp_Proc_(ac_X, { body }) --- We find the LAST occurrence of "ac_X(" before before_pos and --- extract the args from inside the parens. --- --- No regex (per the no_regex constraint). Uses string.find with --- plain mode (4th arg = true) to find the name + open paren. --- ============================================================ +-- ════════════════════════════════════════════════════════════════════════════ +-- Function-args extraction (precedes MipsAtomComp_Proc_ invocations) +-- ════════════════════════════════════════════════════════════════════════════ +-- Find the LAST occurrence of `name + "("` in `source[1..before_pos]`. +-- Returns the position of the open paren, or nil if not found. +-- @param source string +-- @param name string +-- @param before_pos integer +-- @return integer|nil +local function find_last_name_open_paren(source, name, before_pos) + local search = source:sub(1, before_pos) + local name_open = name .. "(" + local last_idx = nil + local scan_pos = 1 + while true do + local found = search:find(name_open, scan_pos, true) -- plain (no regex) + if not found then break end + last_idx = found + scan_pos = found + #name_open + end + return last_idx +end + +--- Find the args of the function declaration that immediately precedes +--- a `MipsAtomComp_Proc_` invocation of the given name. Returns the +--- args string (e.g., `"U4 off, U4 code, U1 r, U1 g, U1 b"`) or nil +--- if no function declaration is found. +--- +--- Convention: function form is +--- `FI_ MipsAtom ac_X(args) MipsAtomComp_Proc_(ac_X, { body })` +--- We find the LAST occurrence of `"ac_X("` before `before_pos` and +--- extract the args from inside the parens. We then verify the +--- preceding context ends with `MipsAtom` (the function-decl keyword +--- with possible qualifiers between). +--- --- @param source string --- @param name string --- @param before_pos integer --- @return string|nil local function find_function_args_for(source, name, before_pos) - local search = source:sub(1, before_pos) - local name_paren = name .. "(" - local last_idx = nil - local p = 1 - while true do - local s = search:find(name_paren, p, true) -- plain (no regex) - if not s then break end - last_idx = s - p = s + #name_paren - end - if not last_idx then return nil end + local last_idx = find_last_name_open_paren(source, name, before_pos) + if not last_idx then return nil end - -- Verify the preceding context ends with "MipsAtom" (with - -- possible qualifiers between). Check the last word is - -- "MipsAtom" (or the trimmed before ends with that token). - local before = search:sub(1, last_idx - 1) - local trimmed = duffle.trim(before) - if trimmed:sub(-#"MipsAtom") ~= "MipsAtom" then - -- Preceding context is not a function declaration. - -- This shouldn't happen with the convention, but guard anyway. - return nil - end + -- Verify the preceding context ends with "MipsAtom" (with + -- possible qualifiers between). + local before = source:sub(1, last_idx - 1) + local trimmed = duffle.trim(before) + if trimmed:sub(-#MIPS_ATOM) ~= MIPS_ATOM then + -- Preceding context is not a function declaration. + return nil + end - local open_paren = last_idx + #name -- position of "(" - local inner = read_parens(source, open_paren) - if not inner then return nil end - return inner + local open_paren = last_idx + #name -- position of "(" + local inner = duffle.read_parens(source, open_paren) + if not inner then return nil end + return inner end --- ============================================================ --- Find the contiguous comment block immediately preceding `pos` in --- `source`. Returns the comment text (with the `/* */` or `//` markers --- preserved) or an empty string if no comment is adjacent. --- Used to copy signature comments from the source declaration --- (`MipsAtomComp_` / `MipsAtomComp_Proc_` / function decl) over to the generated --- `mac_X` macro, so LSP/IntelliSense displays the args doc. --- No regex (per the no_regex constraint). --- ============================================================ +-- ════════════════════════════════════════════════════════════════════════════ +-- Preceding-comment-block extraction +-- ════════════════════════════════════════════════════════════════════════════ +-- Skip whitespace (space/tab/newline/CR) backward from `pos`, returning +-- the position of the first non-whitespace char. +-- @param source string +-- @param pos integer +-- @return integer +local function skip_ws_backward(source, pos) + local back = pos - 1 + while back > 0 do + local ch = source:sub(back, back) + if ch == " " or ch == "\t" or ch == "\n" or ch == "\r" then + back = back - 1 + else + break + end + end + return back +end + +-- Find the opening `/*` for a block comment whose `*/` ends at `close_pos`. +-- Returns the position of `/`, or nil if not found. +-- @param source string +-- @param close_pos integer -- position of the closing `*` of `*/` +-- @return integer|nil +local function find_block_comment_open(source, close_pos) + local prefix = source:sub(1, close_pos - 1) + local open_at = nil + for scan = #prefix - 1, 1, -1 do + if prefix:sub(scan, scan + 1) == "/*" then + open_at = scan + break + end + end + return open_at +end + +-- Walk back from `open_at` over leading spaces + tabs to include the +-- indentation before the `/*` in the captured comment. +-- @param source string +-- @param open_at integer +-- @return integer +local function extend_left_over_indent(source, open_at) + local start = open_at + while start > 1 do + local ch = source:sub(start - 1, start - 1) + if ch == " " or ch == "\t" then + start = start - 1 + else + break + end + end + return start +end + +-- Walk back from `line_end` to the start of the source line (the most +-- recent `\n` or position 1). +-- @param source string +-- @param line_end integer +-- @return integer +local function find_line_start(source, line_end) + local start = line_end + while start > 1 and source:sub(start - 1, start - 1) ~= "\n" do + start = start - 1 + end + return start +end + +-- (internal) Capture one `/* ... */` block comment whose closing `*/` +-- ends at `close_end_pos`. Returns (block_text, new_scan_pos) where +-- `new_scan_pos` is where to continue scanning for more comments, or +-- nil if no block comment was found. +local function capture_block_comment(source, close_end_pos) + local open_at = find_block_comment_open(source, close_end_pos) + if not open_at then return nil end + local block_start = extend_left_over_indent(source, open_at) + return source:sub(block_start, close_end_pos), block_start +end + +-- (internal) Capture one `// ...` line comment ending at `line_end_pos`. +-- Returns (comment_text, new_scan_pos) or nil if the line is not a `//` comment. +local function capture_line_comment(source, line_end_pos) + local line_start = find_line_start(source, line_end_pos) + local line = source:sub(line_start, line_end_pos) + if line:sub(1, 2) == "//" then + return line, line_start - 1 + end + return nil +end + +--- Find the contiguous comment block immediately preceding `pos` in +--- `source`. Returns the comment text (with the `/* */` or `//` markers +--- preserved) or an empty string if no comment is adjacent. +--- +--- Used to copy signature comments from the source declaration +--- (`MipsAtomComp_` / `MipsAtomComp_Proc_` / function decl) over to the +--- generated `mac_X` macro, so LSP/IntelliSense displays the args doc. +--- +--- No regex (per the no_regex constraint). +--- --- @param source string --- @param pos integer --- @return string local function preceding_comment_block(source, pos) - local i = pos - local pieces = {} - while true do - -- Skip whitespace - local j = i - 1 - while j > 0 do - local c = source:sub(j, j) - if c == " " or c == "\t" or c == "\n" or c == "\r" then - j = j - 1 - else - break - end - end - if j == 0 then break end - -- Check for /* ... */ ending at j - if j >= 2 and source:sub(j-1, j) == "*/" then - local s = source:sub(1, j - 1) - local last_open = nil - for k = #s - 1, 1, -1 do - if s:sub(k, k+1) == "/*" then - last_open = k - break - end - end - if last_open then - -- Include the leading whitespace+indentation before /* - local block_start = last_open - while block_start > 1 do - local c = source:sub(block_start - 1, block_start - 1) - if c == " " or c == "\t" then - block_start = block_start - 1 - else - break - end - end - table.insert(pieces, 1, source:sub(block_start, j)) - i = block_start - else - break - end - -- Check for // comment ending at j (j is at end of line, j-1 is \n) - elseif j >= 1 and (source:sub(j, j) == "\n" or source:sub(j, j) == "\r") then - -- Walk back to the start of the line - local line_start = j - while line_start > 1 and source:sub(line_start-1, line_start-1) ~= "\n" do - line_start = line_start - 1 - end - local line = source:sub(line_start, j) - if line:sub(1, 2) == "//" then - table.insert(pieces, 1, line) - i = line_start - 1 - else - break - end - else - break - end + local scan_pos = pos + local pieces = {} + while true do + local non_ws = skip_ws_backward(source, scan_pos) + if non_ws == 0 then break end + + local is_block_close = non_ws >= 2 and source:sub(non_ws - 1, non_ws) == "*/" + local is_line_end = source:sub(non_ws, non_ws) == "\n" or source:sub(non_ws, non_ws) == "\r" + + if is_block_close then + local block_text, new_scan_pos = capture_block_comment(source, non_ws) + if not block_text then break end + table.insert(pieces, 1, block_text) + scan_pos = new_scan_pos + elseif is_line_end then + local line_text, new_scan_pos = capture_line_comment(source, non_ws) + if not line_text then break end + table.insert(pieces, 1, line_text) + scan_pos = new_scan_pos + else + break end - if #pieces == 0 then return "" end - return table.concat(pieces, "\n") + end + if #pieces == 0 then return "" end + return table.concat(pieces, "\n") end --- ============================================================ --- Extract just the parameter NAMES from a function-args string --- (stripping type annotations). E.g., --- "U4 off, U4 code, U1 r, U1 g, U1 b" -> {"off", "code", "r", "g", "b"} --- "U4 *ptr" -> {"ptr"} --- "" -> nil --- No regex — uses duffle.is_alnum + plain string ops. --- ============================================================ +-- ════════════════════════════════════════════════════════════════════════════ +-- Argument-name extraction +-- ════════════════════════════════════════════════════════════════════════════ +-- Walk `trimmed` backward from `pos` over trailing whitespace / +-- asterisks / brackets, returning the position of the first +-- non-trailer character (i.e. the end of the identifier). +-- @param trimmed string +-- @param pos integer +-- @return integer +local function trim_trailer_back(trimmed, pos) + local back = pos + while back > 0 do + local ch = trimmed:sub(back, back) + if ch == " " or ch == "\t" or ch == "*" or ch == "]" or ch == "[" then + back = back - 1 + else + break + end + end + return back +end + +-- Walk `trimmed` backward from `pos` over identifier chars (alnum + `_`), +-- returning the position just before the identifier starts. +-- @param trimmed string +-- @param pos integer +-- @return integer +local function trim_ident_back(trimmed, pos) + local back = pos + while back > 0 do + local ch = trimmed:sub(back, back) + if duffle.is_alnum(ch) or ch == "_" then + back = back - 1 + else + break + end + end + return back +end + +--- Extract just the parameter NAMES from a function-args string +--- (stripping type annotations). E.g., +--- `"U4 off, U4 code, U1 r, U1 g, U1 b"` -> `{"off", "code", "r", "g", "b"}` +--- `"U4 *ptr"` -> `{"ptr"}` +--- `""` -> nil +--- +--- No regex — uses `duffle.is_alnum` + plain string ops. +--- --- @param args_str string|nil --- @return string[]|nil local function extract_arg_names(args_str) - if not args_str or args_str == "" then return nil end - local names = {} - local tokens = duffle.split_top_level_commas(args_str) - for _, tok in ipairs(tokens) do - local trimmed = duffle.trim(tok) - if trimmed ~= "" then - -- Walk backwards from end of trimmed arg, skipping - -- trailing whitespace / asterisks / brackets. - local i = #trimmed - while i > 0 do - local c = trimmed:sub(i, i) - if c == " " or c == "\t" or c == "*" or c == "]" or c == "[" then - i = i - 1 - else - break - end - end - -- Now find the end of the last identifier (the param name). - local j = i - while j > 0 do - local c = trimmed:sub(j, j) - if duffle.is_alnum(c) or c == "_" then - j = j - 1 - else - break - end - end - local name = trimmed:sub(j + 1, i) - if name ~= "" then - names[#names + 1] = name - end - end + if not args_str or args_str == "" then return nil end + local names = {} + local tokens = duffle.split_top_level_commas(args_str) + for _, tok in ipairs(tokens) do + local trimmed = duffle.trim(tok) + if trimmed ~= "" then + local ident_end = trim_trailer_back(trimmed, #trimmed) + local ident_start = trim_ident_back(trimmed, ident_end) + 1 + local name = trimmed:sub(ident_start, ident_end) + if name ~= "" then names[#names + 1] = name end end - if #names == 0 then return nil end - return names + end + if #names == 0 then return nil end + return names end --- ============================================================ --- Find every MipsAtomComp_(ac_) { body } declaration in source. --- Supports BOTH the bare form and the function form: --- Bare: MipsAtomComp_(ac_X) { body } --- Function: MipsAtomComp_Proc_(ac_X, { body }) (with a preceding --- "FI_ MipsAtom ac_X(args)" function declaration) --- Returns: {line, name, body, args} where args is the function-args --- string (or nil for the bare form). --- ============================================================ --- WORD_COUNT entry in gen/.components.h) --- ============================================================ +-- ════════════════════════════════════════════════════════════════════════════ +-- Component scanner (bare + function forms) +-- ════════════════════════════════════════════════════════════════════════════ +-- Parse the inner content of an `AtomComp_(name, ...)` call. Returns +-- (name, body_or_nil) — `body_or_nil` is non-nil iff this is the +-- function-form `MipsAtomComp_Proc_(name, { body })` invocation. +-- @param inner string -- the content between ( and ) of the AtomComp_ call +--- @return string|nil, string|nil +local function parse_atomcomp_inner(inner) + local tokens = duffle.split_top_level_commas(inner) + if #tokens == 1 then + return duffle.trim(tokens[1]), nil + elseif #tokens == 2 then + local name = duffle.trim(tokens[1]) + local body_raw = duffle.trim(tokens[2]) + -- Strip leading { and trailing } if present. + local body + if #body_raw >= 2 and body_raw:sub(1, 1) == "{" and body_raw:sub(-1) == "}" then + body = duffle.trim(body_raw:sub(2, -2)) + else + body = body_raw + end + return name, body + end + return nil, nil +end + +-- (internal) Try to extract a bare-form `MipsAtomComp_(ac_X)` declaration. +-- Bare form: `MipsAtomComp_(ac_X) { body }` — body comes from the brace block +-- AFTER the parens. +-- @param source string +-- @param name string -- the `ac_X` ident from the parens +--- @param ident_pos integer -- position of the `MipsAtomComp_` ident start +--- @param after_paren integer -- position just past the closing `)` +--- @param line_of fun(pos: integer): integer +--- @param args string|nil -- function-args from preceding function decl +--- @param comment string -- preceding comment block +--- @return Component|nil, integer -- the component + new source position +local function make_bare_component(source, name, ident_pos, after_paren, line_of, args, comment) + local brace = duffle.scan_to_char(source, "{", after_paren) + if not brace then return nil, after_paren + 1 end + local body, after_brace = duffle.read_braces(source, brace) + return { + line = line_of(ident_pos), + name = name:sub(AC_PREFIX_LEN + 1), -- strip "ac_" prefix + body = body, + args = args, + comment = comment, + }, after_brace +end + +-- (internal) Build the function-form `MipsAtomComp_Proc_` component. Body +-- came from inside the parens; no following brace block. +local function make_proc_component(name, body, ident_pos, line_of, args, comment) + return { + line = line_of(ident_pos), + name = name:sub(AC_PREFIX_LEN + 1), + body = body, + args = args, + comment = comment, + } +end + +--- Find every `MipsAtomComp_(ac_) { body }` declaration in source. +--- Supports BOTH the bare form and the function form: +--- Bare: `MipsAtomComp_(ac_X) { body }` +--- Function: `MipsAtomComp_Proc_(ac_X, { body })` (with a preceding +--- `"FI_ MipsAtom ac_X(args)"` function declaration) +--- --- @param source string --- @return Component[] local function find_component_atoms(source) - local line_of = duffle.LineIndex(source) - local out = {} - local len = #source - local i = 1 - while i <= len do - i = skip_ws_and_cmt(source, i); if i > len then break end - local ident, after = read_ident(source, i) - if not ident then - i = i + 1 - elseif ident == "MipsAtomComp_Proc_" - or ident == "MipsAtomComp_" then - local open = skip_ws_and_cmt(source, after) - if source:sub(open, open) == "(" then - local inner, after_paren = read_parens(source, open) - -- Parse args: 1 arg = bare form, 2 args = function form - local tokens = duffle.split_top_level_commas(inner) - local name, body = nil, nil - if #tokens == 1 then - name = duffle.trim(tokens[1]) - elseif #tokens == 2 then - name = duffle.trim(tokens[1]) - local body_raw = duffle.trim(tokens[2]) - -- Strip leading { and trailing } if present - if #body_raw >= 2 - and body_raw:sub(1, 1) == "{" - and body_raw:sub(-1) == "}" then - body = duffle.trim(body_raw:sub(2, -2)) - else - body = body_raw - end - end - if name and name:sub(1, 3) == "ac_" then - -- Find the function args (preceding function decl). - -- For the bare form this returns nil (no function). - local args = find_function_args_for(source, name, open) - -- Capture the preceding comment block (signature doc). - -- Walk back from `i` (position of the identifier start) - -- so the walk-back goes through whitespace+comment and - -- stops AT the comment (not at the identifier chars). - local comment = preceding_comment_block(source, i) - if body == nil then - -- Bare form: body is the brace block AFTER the parens. - local brace = scan_to_char(source, "{", after_paren) - if brace then - local body_content, after_brace = read_braces(source, brace) - out[#out + 1] = { - line = line_of(i), - name = name:sub(4), -- strip "ac_" prefix - body = body_content, - args = args, - comment = comment, - } - i = after_brace - else - i = open + 1 - end - else - -- Function form: body is the second arg (already extracted). - out[#out + 1] = { - line = line_of(i), - name = name:sub(4), -- strip "ac_" prefix - body = body, - args = args, - comment = comment, - } - i = after_paren - end - else - i = open + 1 - end - else - i = open + 1 - end + local line_of = duffle.LineIndex(source) + local out = {} + local pos = 1 + local src_len = #source + while pos <= src_len do + pos = duffle.skip_ws_and_cmt(source, pos) + if pos > src_len then break end + + local ident, after_ident = duffle.read_ident(source, pos) + local is_comp = ident == ATOM_COMP or ident == ATOM_COMP_PROC + if not ident then + pos = pos + 1 + elseif not is_comp then + pos = after_ident + else + local open_paren = duffle.skip_ws_and_cmt(source, after_ident) + if source:sub(open_paren, open_paren) ~= "(" then + pos = open_paren + 1 + else + local inner, after_paren = duffle.read_parens(source, open_paren) + local name, body = parse_atomcomp_inner(inner) + if not name or name:sub(1, AC_PREFIX_LEN) ~= AC_PREFIX then + pos = open_paren + 1 else - i = after + local args = find_function_args_for(source, name, open_paren) + local comment = preceding_comment_block(source, pos) + if body == nil then + -- Bare form: body comes from the brace block after the parens. + local comp, new_pos = make_bare_component(source, name, pos, after_paren, line_of, args, comment) + if comp then out[#out + 1] = comp end + pos = new_pos + else + -- Function form: body was inside the parens. + out[#out + 1] = make_proc_component(name, body, pos, line_of, args, comment) + pos = after_paren + end end + end end - return out + end + return out end --- ============================================================ --- Emit a per-directory generated header with mac_X(...) macros --- derived from MipsAtomComp_ declarations + auto word-counts. --- Output: /gen/.macs.h --- ============================================================ +-- ════════════════════════════════════════════════════════════════════════════ +-- Line-comment → block-comment conversion +-- ════════════════════════════════════════════════════════════════════════════ -- Convert `//` line comments to `/* */` block comments in a token. +-- -- C macros use `\` line-continuations; a `//` comment before `\` would -- consume the continuation, breaking the macro. We convert `//` to -- `/* */` so the multi-line macro structure is preserved. +-- -- Skips `//` sequences that are inside string or character literals -- (a rough heuristic — sufficient for component bodies which don't -- have those constructs). - +-- --- @param s string --- @return string local function convert_line_comments_to_block(s) local result = s - local i = 1 - while i <= #result do - local c = result:byte(i) - if c == 47 and i + 1 <= #result and result:byte(i + 1) == 47 then - -- Found `//`. Find end of line. - local eol = i - while eol <= #result and result:byte(eol) ~= 10 do + local pos = 1 + local len = #result + while pos <= len do + local is_double_slash = result:byte(pos) == BYTE_SLASH + and pos + 1 <= len and result:byte(pos + 1) == BYTE_SLASH + if not is_double_slash then + pos = pos + 1 + else + -- Find end of line. + local eol = pos + while eol <= len and result:byte(eol) ~= BYTE_NEWLINE do eol = eol + 1 end - local before = result:sub(1, i - 1) - local comment = result:sub(i + 2, eol - 1) -- skip the `//` + local before = result:sub(1, pos - 1) + local comment = result:sub(pos + 2, eol - 1) -- skip the `//` local after - if eol <= #result and result:byte(eol) == 10 then + if eol <= len and result:byte(eol) == BYTE_NEWLINE then after = " */" .. result:sub(eol) -- keep the newline else after = " */" end result = before .. "/*" .. comment .. after - i = #before + 2 + #comment + 3 -- skip past converted comment - else - i = i + 1 + pos = #before + 2 + #comment + 3 -- skip past converted comment end end return result end --- Compute the word count of a component body, accounting for --- macro expansion. Each comma-separated entry in the body is a --- "slot" that contributes its own word count. For most entries --- (regular MIPS instructions) the count is 1. For `mac_Y(...)` --- calls, the count is the word count of mac_Y (recursive lookup --- through `components`). For encoding macros with a known multi-word --- count (e.g. `mask_upper` = 2), the count is taken from `word_counts`. --- --- The lookup is memoized via `rec()` to avoid infinite recursion --- (e.g. if two components referenced each other). This is the --- same algorithm as the original tape_atom_annotation_pass.lua --- (commit 7d20a4d) — without it, a fresh build with no pre-existing --- *.macs.h files in gen/ would compute wrong counts: e.g. --- `mac_format_f3_color` calls `mac_pack_color_word` which isn't --- in `wc` yet, so the lookup falls through to the "1 word" default --- and produces `WORD_COUNT(mac_format_f3_color, 1)` instead of 3. --- ---- @param c Component ---- @param components Component[] ---- @param wc table ---- @return integer -local function compute_component_word_count(c, components, wc) - -- Build component lookup table once per call. - local comp_by_name = {} - for _, cc in ipairs(components) do - comp_by_name[cc.name] = cc - end +-- ════════════════════════════════════════════════════════════════════════════ +-- Word-count computation (memoized recursive lookup) +-- ════════════════════════════════════════════════════════════════════════════ - local cache = {} - local function rec(name) - if cache[name] ~= nil then return cache[name] end - cache[name] = -1 -- mark in-progress (cycle detection) - local cc = comp_by_name[name] - local n - if cc then - local body_tokens = {} - for _, t in ipairs(split_top_level_commas(cc.body)) do - local trimmed = trim(t) - if trimmed ~= "" then body_tokens[#body_tokens + 1] = trimmed end - end - n = 0 - for _, t in ipairs(body_tokens) do - -- Read the first identifier from the token. - local ident = read_ident(t, 1) - -- Components are stored without the `mac_` prefix - -- (e.g. "format_f3_color"). The token has `mac_format_f3_color(...)`, - -- so strip the `mac_` prefix to look up the component. - local comp_name = ident - if comp_name and comp_name:sub(1, 4) == "mac_" then - comp_name = comp_name:sub(5) - end - if comp_name and comp_by_name[comp_name] then +-- Strip the `mac_` prefix from a component-call ident so we can look it +-- up against the components-by-name table. Returns the ident unchanged +-- if it doesn't start with the prefix (so a non-component ident like +-- `mask_upper` falls through to the wc-table branch). +-- @param ident string|nil +-- @return string|nil +local function strip_mac_prefix(ident) + if not ident then return nil end + if ident:sub(1, MAC_PREFIX_LEN) == MAC_PREFIX then + return ident:sub(MAC_PREFIX_LEN + 1) + end + return ident +end + +-- (internal) Recursive word-count lookup. `cache` is the memoization table +-- across all calls to `compute_component_word_count`; the in-progress +-- -1 sentinel detects cycles (A -> B -> A). +-- @param name string -- the component name (without `mac_`) +-- @param comp_by_name table +-- @param wc table +-- @param cache table +-- @return integer +local function word_count_rec(name, comp_by_name, wc, cache) + if cache[name] ~= nil then return cache[name] end + cache[name] = -1 -- mark in-progress (cycle detection) + local cc = comp_by_name[name] + local n + if cc then + n = 0 + for _, t in ipairs(duffle.split_top_level_commas(cc.body)) do + local trimmed = duffle.trim(t) + if trimmed ~= "" then + local lookup = strip_mac_prefix(duffle.read_ident(trimmed, 1)) + if lookup and comp_by_name[lookup] then -- It's a `mac_X(...)` call. Recurse. - n = n + rec(comp_name) - elseif comp_name and wc and wc[comp_name] then + n = n + word_count_rec(lookup, comp_by_name, wc, cache) + elseif lookup and wc and wc[lookup] then -- Encoding macro or pseudo-instruction (e.g. mask_upper = 2, nop2 = 2). - n = n + wc[comp_name] + n = n + wc[lookup] else -- Unrecognized token. Fall back to 1 word. n = n + 1 end end - else - -- Not a known component: assume 1 word (regular instruction). - n = 1 end - cache[name] = n - return n + else + -- Not a known component: assume 1 word (regular instruction). + n = 1 end - return rec(c.name) + cache[name] = n + return n end --- ============================================================ --- Per-component emit logic. Returns the body of lines for one --- component (signature comment, #define mac_X(...) line with --- backslash-continued tokens, then WORD_COUNT(mac_X, N) entry). --- --- Extracted from emit_component_macros_h so M.run can call it --- once per component AND extend ctx.shared.word_counts. --- ============================================================ +--- Compute the word count of a component body, accounting for macro +--- expansion. Each comma-separated entry in the body is a "slot" that +--- contributes its own word count. For most entries (regular MIPS +--- instructions) the count is 1. For `mac_Y(...)` calls, the count is +--- the word count of `mac_Y` (recursive lookup through `components`). +--- For encoding macros with a known multi-word count (e.g. `mask_upper` = 2), +--- the count is taken from `word_counts`. +--- +--- The lookup is memoized via `word_count_rec` to avoid infinite recursion +--- (e.g. if two components referenced each other). This is the same +--- algorithm as the original `tape_atom_annotation_pass.lua` (commit 7d20a4d). +--- +--- @param c Component +--- @param components Component[] +--- @param wc table +--- @return integer +local function compute_component_word_count(c, components, wc) + local comp_by_name = {} + for _, cc in ipairs(components) do comp_by_name[cc.name] = cc end + local cache = {} + return word_count_rec(c.name, comp_by_name, wc, cache) +end --- ============================================================ --- Per-component emit logic. Returns the body of lines for one --- component (signature comment, #define mac_X(...) line with --- backslash-continued tokens, then WORD_COUNT(mac_X, N) entry). --- --- Extracted from emit_component_macros_h so M.run can call it --- once per component AND extend ctx.shared.word_counts. --- ============================================================ +-- ════════════════════════════════════════════════════════════════════════════ +-- Per-component emit logic +-- ════════════════════════════════════════════════════════════════════════════ --- Split a (possibly multi-line) comment into per-line entries. --- Hand-rolled (no regex patterns used). --- @param s string --- @return string[] local function split_comment_lines(s) - local out = {} - local i = 1 - local len = #s - while i <= len do - local nl = s:find("\n", i, true) + local out = {} + local pos = 1 + local s_len = #s + while pos <= s_len do + local nl = s:find("\n", pos, true) if not nl then - out[#out + 1] = s:sub(i) + out[#out + 1] = s:sub(pos) break end - out[#out + 1] = s:sub(i, nl - 1) - i = nl + 1 + out[#out + 1] = s:sub(pos, nl - 1) + pos = nl + 1 end return out end @@ -519,8 +670,8 @@ end --- @return string[] local function tokens_from_body(body) local out = {} - for _, t in ipairs(split_top_level_commas(body)) do - local trimmed = trim(t) + for _, t in ipairs(duffle.split_top_level_commas(body)) do + local trimmed = duffle.trim(t) if trimmed ~= "" then out[#out + 1] = trimmed end end return out @@ -538,7 +689,7 @@ local function signature_from_args(args_str) return "..." end ---- Strip the trailing " \" (space + backslash) line continuation +--- Strip the trailing `" \"` (space + backslash) line continuation --- from the last body line. The last 2 chars are always that pair. local function strip_trailing_continuation(lines) local last = lines[#lines] @@ -551,17 +702,20 @@ end --- block. Converts `//` line comments to `/* */` block comments in --- each token so they don't break the C macro `\` line continuations. local function emit_macro_body(lines, c, sig, tokens) - for j = 1, #tokens do - tokens[j] = convert_line_comments_to_block(tokens[j]) + for tok_idx = 1, #tokens do + tokens[tok_idx] = convert_line_comments_to_block(tokens[tok_idx]) end lines[#lines + 1] = "#define mac_" .. c.name .. "(" .. sig .. ") \\" lines[#lines + 1] = "\t" .. tokens[1] .. " \\" - for j = 2, #tokens do - lines[#lines + 1] = ",\t" .. tokens[j] .. " \\" + for tok_idx = 2, #tokens do + lines[#lines + 1] = ",\t" .. tokens[tok_idx] .. " \\" end strip_trailing_continuation(lines) end +--- Build the list of lines for one component (signature comment, +--- `#define mac_X(...)` line with backslash-continued tokens, then +--- `WORD_COUNT(mac_X, N)` entry). --- @param c Component --- @param components Component[] --- @param wc table @@ -590,32 +744,17 @@ local function build_component_lines(c, components, wc) return lines end --- ============================================================ --- Emit a per-source .macs.h header with the mac_X macros + --- WORD_COUNT entries. Writes in BINARY mode so LF line endings --- are preserved (the git blob is LF; Windows text-mode would --- emit CRLF and break the byte-identical diff). --- --- Honors ctx.dry_run: prints the intended path but does not --- write the file. --- ============================================================ +-- ════════════════════════════════════════════════════════════════════════════ +-- Per-source emit logic +-- ════════════════════════════════════════════════════════════════════════════ ---- @param ctx PassCtx ---- @param src SourceFile ---- @param components Component[] ---- @return string|nil -- path to the written file (nil on dry-run) -local function emit_component_macros_h(ctx, src, components) - if #components == 0 then return nil end - - -- Output path: /gen/.macs.h - -- The pre-rework convention uses the *directory* basename (not - -- the source file basename) — e.g. `code/duffle/lottes_tape.h` - -- produces `code/duffle/gen/duffle.macs.h`. This matches what - -- the C codebase #includes. - local out_dir = src.dir .. "/gen" - local out_path = out_dir .. "/" .. basename_no_ext(src.dir) .. ".macs.h" - - local lines = { +-- Build the boilerplate header lines (the `#ifdef INTELLISENSE_DIRECTIVES` +-- block, the `// Auto-generated` comment, the `// Source:` line, and the +-- self-contained `WORD_COUNT` macro definition). +-- @param src SourceFile +-- @return string[] +local function header_boilerplate(src) + return { -- #pragma once wrapped in #ifdef INTELLISENSE_DIRECTIVES, matching -- the convention in lottes_tape.h. The build does manual unity -- includes (the user controls include order), so the pragma @@ -636,11 +775,45 @@ local function emit_component_macros_h(ctx, src, components) "#endif", "", } +end + +-- Compute the output path for one source's `.macs.h` file. +-- The pre-rework convention uses the *directory* basename (not the +-- source file basename) — e.g. `code/duffle/lottes_tape.h` produces +-- `code/duffle/gen/duffle.macs.h`. This matches what the C codebase +-- #includes. +-- @param src SourceFile +-- @return string -- the output directory +-- @return string -- the full output path +local function compute_macs_h_path(src) + local out_dir = src.dir .. "/" .. GEN_SUBDIR + local out_path = out_dir .. "/" .. duffle.basename_no_ext(src.dir) .. ".macs.h" + return out_dir, out_path +end + +--- Emit a per-source `.macs.h` header with the `mac_X` macros + +--- `WORD_COUNT` entries. Writes in BINARY mode so LF line endings are +--- preserved (the git blob is LF; Windows text-mode would emit CRLF and +--- break the byte-identical diff). +--- +--- Honors `ctx.dry_run`: prints the intended path but does not write +--- the file. +--- +--- @param ctx PassCtx +--- @param src SourceFile +--- @param components Component[] +--- @return string|nil -- path to the written file (nil if no components) +local function emit_component_macros_h(ctx, src, components) + if #components == 0 then return nil end + + local out_dir, out_path = compute_macs_h_path(src) + local lines = header_boilerplate(src) local wc = ctx.shared.word_counts for _, c in ipairs(components) do - local comp_lines = build_component_lines(c, components, wc) - for _, l in ipairs(comp_lines) do lines[#lines + 1] = l end + for _, l in ipairs(build_component_lines(c, components, wc)) do + lines[#lines + 1] = l + end end local content = table.concat(lines, "\n") .. "\n" @@ -650,7 +823,7 @@ local function emit_component_macros_h(ctx, src, components) return out_path end - ensure_dir(out_dir) + duffle.ensure_dir(out_dir) write_file_lf(out_path, content) print(string.format(" -> %s", out_path)) return out_path @@ -660,6 +833,17 @@ end -- Pass entry -- ════════════════════════════════════════════════════════════════════════════ +-- (internal) Extend `ctx.shared.word_counts` with this source's component +-- macros so offsets sees them without re-reading the file. +-- @param ctx PassCtx +-- @param components Component[] +local function update_shared_word_counts(ctx, components) + local wc = ctx.shared.word_counts + for _, c in ipairs(components) do + wc["mac_" .. c.name] = compute_component_word_count(c, components, wc) + end +end + --- @param ctx PassCtx --- @return PassResult function M.run(ctx) @@ -673,14 +857,8 @@ function M.run(ctx) if #components > 0 then local macs_path = emit_component_macros_h(ctx, src, components) if macs_path then - table.insert(outputs, { macs_h = macs_path }) - - -- Extend the shared word_counts table so offsets sees the - -- component macros without re-reading the file. - local wc = ctx.shared.word_counts - for _, c in ipairs(components) do - wc["mac_" .. c.name] = compute_component_word_count(c, components, wc) - end + outputs[#outputs + 1] = { macs_h = macs_path } + update_shared_word_counts(ctx, components) end end end diff --git a/scripts/passes/duffle_paths.lua b/scripts/passes/duffle_paths.lua new file mode 100644 index 0000000..957b52a --- /dev/null +++ b/scripts/passes/duffle_paths.lua @@ -0,0 +1,75 @@ +--- duffle_paths.lua — Single-line bootstrap helper for the tape-atom +--- Lua scripts. +--- +--- Each entry script (ps1_meta.lua, word_count_eval.lua, and the 5 +--- passes/*.lua files) starts with: +--- +--- ```lua +--- local duffle = dofile((arg[0]:match("(.*[/\\])") or "./") .. "duffle_paths.lua") +--- ``` +--- +--- That single line: (a) locates this helper via `arg[0]`, (b) loads +--- it (which sets `package.path` + `package.cpath` via `git rev-parse`), +--- (c) returns the `M` table (a wrapper around the setup function). +--- After this line, `require("duffle")` and `require("passes.X")` both +--- resolve normally. +--- +--- **Why a helper instead of inline?** +--- - The 8-line path-setup boilerplate was duplicated across 7 entry +--- scripts (one per file). Single source of truth here. +--- - Mirrors the build script's pattern in `build_psyq.ps1`: +--- `$path_root = split-path -Path $PSScriptRoot -Parent;` then +--- derive everything from there. +--- +--- **Why `git rev-parse --show-toplevel`?** +--- Hardcoding `C:\\projects\\Pikuma\\ps1\\...` breaks portability. Git +--- gives us the canonical repo root regardless of where the repo lives +--- on disk. + +local M = {} + +--- Resolve the repo root via git. Returns a normalized path with a +--- trailing forward-slash, or nil if not in a git repo. +--- @return string|nil +local function find_repo_root() + local p = io.popen("git rev-parse --show-toplevel 2>nul") + local root + if p then + root = p:read("*l") + p:close() + end + if not root or root == "" then return nil end + -- Normalize to forward slashes (Windows accepts both, but mixed + -- `\` + `/` confuses LuaJIT's file APIs). + root = root:gsub("\\", "/") + if not root:match("/$") then root = root .. "/" end + return root +end + +--- Set `package.path` (for `require("duffle")` + `require("passes.X")`) +--- and `package.cpath` (for `lpeg.dll` on Windows). +function M.setup() + local repo_root = find_repo_root() + if not repo_root then + io.stderr:write("[duffle_paths] git rev-parse failed -- not in a git repo?\n") + os.exit(2) + end + + local scripts_dir = repo_root .. "scripts/" + local passes_dir = repo_root .. "scripts/passes/" + package.path = scripts_dir .. "?.lua;" + .. scripts_dir .. "?/init.lua;" + .. passes_dir .. "?.lua;" + .. passes_dir .. "?/init.lua;" + .. package.path + + if package.config:sub(1, 1) == "\\" then + package.cpath = repo_root .. "toolchain/luajit-2.1/lib/lua/5.1/?.dll;" + .. package.cpath + end +end + +-- Run the setup as a side effect. +M.setup() + +return M \ No newline at end of file diff --git a/scripts/passes/offsets.lua b/scripts/passes/offsets.lua index 1ba75dc..6845d2c 100644 --- a/scripts/passes/offsets.lua +++ b/scripts/passes/offsets.lua @@ -1,286 +1,405 @@ --- passes/offsets.lua --- --- Generate /gen/.offsets.h with branch offset --- immediates for every atom_offset(F, T) reference in atom bodies. --- --- The branch offset regression we just fixed in commit 98e27c2 must --- NOT return. The fix was in duffle.lua's split_top_level_commas + --- tape_atom_annotation_pass.lua's compute_component_word_count. --- word_count_eval.count_token_words preserves the fix. --- --- THIS MODULE ALSO REQUIRES the recent fix to duffle.lua's --- split_top_level_commas (the second-half of the 98e27c2 fix): --- top-level comments must be appended to the previous token, not --- stripped, so the emit path preserves `// trailing comment` text --- for convert_line_comments_to_block to convert to `/* */`. --- --- Coding standard: tabs (1/level), EmmyLua annotations, no regex. +--- passes/offsets.lua — Branch-offset generator. +--- +--- Scans every source for `MipsAtom_(name) { ... }` (and the raw +--- `MipsCode code_ { ... }` form) declarations, computes the +--- word offset from each `atom_offset(F, T)` marker to its target +--- `atom_label(T)` declaration, and emits `.offsets.h` +--- with one `#define _atom_offset_F_T = N` per branch. +--- +--- The offset is `target_word - branch_word - 1` (the standard MIPS +--- branch-immediate encoding: branch_offset = relative_pc_in_words - 1). +--- +--- **Conventions**: tabs (1/level), EmmyLua annotations, no regex, +--- Lua 5.3 compatible. See +--- `C:\projects\Pikuma\ps1-ai\conductor\code_styleguides\lua.md`. -- ════════════════════════════════════════════════════════════════════════════ -- Module-scope requires + package.path setup -- ════════════════════════════════════════════════════════════════════════════ -local script_path = arg and arg[0] or "?" -local last_sep = 0 -for i = 1, #script_path do - local c = script_path:sub(i, i) - if c == "/" or c == "\\" then last_sep = i end -end -local script_dir = last_sep == 0 and "./" or script_path:sub(1, last_sep) -package.path = script_dir .. "../?.lua;" .. script_dir .. "../?/init.lua;" .. script_dir .. "?.lua;" .. package.path - -local duffle = require("duffle") -local trim = duffle.trim -local read_ident = duffle.read_ident -local is_space = duffle.is_space -local is_alpha = duffle.is_alpha -local is_alnum = duffle.is_alnum -local skip_ws_and_cmt = duffle.skip_ws_and_cmt -local skip_str_or_cmt = duffle.skip_str_or_cmt -local split_top_level_commas = duffle.split_top_level_commas -local read_parens = duffle.read_parens -local read_braces = duffle.read_braces -local write_file = duffle.write_file -local dirname = duffle.dirname -local basename_no_ext = duffle.basename_no_ext -local ensure_dir = duffle.ensure_dir - -local word_count_eval = require("word_count_eval") -local count_token_words = word_count_eval.count_token_words +-- Resolve `arg[0]` to an absolute-ish script directory so that +-- `require("duffle")` resolves against `scripts/` regardless of CWD. +-- Note: this boilerplate is duplicated in 6 other entry scripts; a +-- Phase-6 extraction target (`duffle.setup_package_path()`). +-- Bootstrap: see `ps1_meta.lua` for the rationale. +dofile((arg[0]:match("(.*[/\\])") or "./") .. "duffle_paths.lua") +local duffle = require("duffle") +local word_count_eval = require("word_count_eval") +local count_token_words = word_count_eval.count_token_words -- ════════════════════════════════════════════════════════════════════════════ --- Local helpers (ported from offset_gen.meta.lua lines 67-93) +-- Constants -- ════════════════════════════════════════════════════════════════════════════ +-- C qualifier keywords that may precede a `MipsAtom_` declaration +-- (and should be skipped by `skip_qualifiers`). +local QUALIFIER_KEYWORDS = { + ["static"] = true, ["const"] = true, ["volatile"] = true, + ["extern"] = true, ["register"] = true, ["auto"] = true, + ["inline"] = true, ["typedef"] = true, + ["internal"] = true, ["LP_"] = true, ["global"] = true, ["gkknown"] = true, +} + +-- Atom declaration identifiers. +local ATOM_PREFIX = "MipsAtom_" +local CODE_DECL = "MipsCode" +local CODE_RAW_PREFIX = "code_" -- raw atom form: `MipsCode code_ { ... }` +local CODE_RAW_PREFIX_LEN = 5 -- = #CODE_RAW_PREFIX + +-- Marker-call identifiers inside atom bodies. +local LABEL_MARKER = "atom_label" +local OFFSET_MARKER = "atom_offset" + +-- Offset macro/enum naming prefixes (the emitted header uses these). +local OFFSET_MACRO_PREFIX = "_atom_offset_" +local OFFSET_ENUM_PREFIX = "atom_offset_" + +-- Column width for the `#define _atom_offset_F_T = N` alignment. +local OFFSET_MACRO_COL = 44 + +-- ════════════════════════════════════════════════════════════════════════════ +-- Type declarations +-- ════════════════════════════════════════════════════════════════════════════ + +--- @class SourceFile +--- @field path string -- absolute path to the source file +--- @field text string -- the full source text +--- @field dir string -- the directory containing the source +--- @field basename string -- filename without extension + +--- @class PassCtx +--- @field sources SourceFile[] -- all source files in the build +--- @field metadata_path string -- path to word_count.metadata.h +--- @field shared table -- cross-pass shared state +--- @field shared.word_counts table -- macro name -> word count +--- @field out_root string -- output root (e.g. "build/gen") +--- @field project_root string -- project root (e.g. "code/") +--- @field upstream table -- per-pass upstream outputs +--- @field flags table -- CLI flags +--- @field dry_run boolean -- if true, compute but don't write +--- @field verbose boolean -- if true, log diagnostic info + +--- @class PassResult +--- @field outputs table[] -- {kind=, path=} entries describing emit files +--- @field errors table[] -- {line=, msg=} entries; build-stops +--- @field warnings table[] -- {line=, msg=} entries; build-succeeds + +--- @class Atom +--- @field name string -- atom name (e.g. "cube_g4_face") +--- @field body string -- the brace-delimited body (without the braces) + +--- @class BranchOffset +--- @field tag string -- the marker tag (e.g. "F" in `atom_offset(F, T)`) +--- @field target string -- the target label name (e.g. "T" in `atom_offset(F, T)`) +--- @field pos integer -- the branch's word position within the atom body +--- @field offset integer -- computed `target_word - branch_word - 1` + +--- @class AtomData +--- @field name string -- atom name +--- @field total_words integer -- total word count of the atom body +--- @field offsets BranchOffset[] -- per-branch offset list + +-- ════════════════════════════════════════════════════════════════════════════ +-- Local helpers +-- ════════════════════════════════════════════════════════════════════════════ + +-- Returns true if `s` starts with `prefix`. +-- @param s string +-- @param prefix string +-- @return boolean local function starts_with(s, prefix) if #s < #prefix then return false end - for i = 1, #prefix do - if s:sub(i, i) ~= prefix:sub(i, i) then return false end + for pos = 1, #prefix do + if s:sub(pos, pos) ~= prefix:sub(pos, pos) then return false end end return true end -local function to_upper(s) return s:upper() end - +-- Replace every non-alphanumeric char in `s` with underscore. +-- @param s string +-- @return string local function to_alnum_underscore(s) local out = "" - for i = 1, #s do - local c = s:sub(i, i) - if is_alnum(c) then out = out .. c - else out = out .. "_" end + for pos = 1, #s do + local ch = s:sub(pos, pos) + if duffle.is_alnum(ch) then out = out .. ch else out = out .. "_" end end return out end -local function pad_right(s, w) return s .. string.rep(" ", w - #s) end +-- Right-pad `s` with spaces to width `w`. If `s` is already `w` or +-- wider, no padding is added. +-- @param s string +-- @param w integer +-- @return string +local function pad_right(s, w) + return s .. string.rep(" ", math.max(0, w - #s)) +end -- ════════════════════════════════════════════════════════════════════════════ --- Marker-call helpers (ported from offset_gen.meta.lua lines 148-205) +-- Marker-call helpers -- ════════════════════════════════════════════════════════════════════════════ ---- Extract comma-separated identifier args from a parenthesized group ---- after a function-like macro call. +-- Extract comma-separated identifier args from a parenthesized group +-- after a function-like macro call. Returns (args, after_paren) where +-- `after_paren` is the position just past the closing `)`, or nil if +-- `token` did not start with `(`. +-- @param token string +-- @param after_ident integer +-- @return string[], integer|nil local function extract_ident_args(token, after_ident) - local arg_start = skip_ws_and_cmt(token, after_ident) + local arg_start = duffle.skip_ws_and_cmt(token, after_ident) if token:sub(arg_start, arg_start) ~= "(" then return {}, nil end - local inner, after_paren = read_parens(token, arg_start) + local inner, after_paren = duffle.read_parens(token, arg_start) local args = {} - local n = 1 - local len = #inner - while n <= len do - n = skip_ws_and_cmt(inner, n) - if n > len then break end - local ident, after = read_ident(inner, n) + local pos = 1 + local inner_len = #inner + while pos <= inner_len do + pos = duffle.skip_ws_and_cmt(inner, pos) + if pos > inner_len then break end + local ident, after = duffle.read_ident(inner, pos) if ident and ident ~= "" then table.insert(args, ident) - n = after + pos = after else - n = n + 1 + pos = pos + 1 end - n = skip_ws_and_cmt(inner, n) - if n <= len and inner:sub(n, n) == "," then n = n + 1 end + pos = duffle.skip_ws_and_cmt(inner, pos) + if pos <= inner_len and inner:sub(pos, pos) == "," then pos = pos + 1 end end return args, after_paren end +-- (internal) Record a `atom_label(name)` marker — `at_pos` is the +-- branch-free word position within the atom body. +-- @param labels table +-- @param args string[] +-- @param at_pos integer +local function record_label_marker(labels, args, at_pos) + if #args >= 1 then labels[args[1]] = at_pos end +end + +-- (internal) Record a `atom_offset(tag, target)` marker. +-- @param branches table[] -- list of {pos=, target=, tag=} +-- @param args string[] +-- @param at_pos integer +local function record_offset_marker(branches, args, at_pos) + if #args >= 2 then + table.insert(branches, { pos = at_pos, target = args[2], tag = args[1] }) + end +end + --- Scan a single token for atom_label/atom_offset markers, walking through --- balanced groups transparently (so nested calls are found). +--- @param token string +--- @param at_pos integer -- the branch-free word position of this token in the body +--- @param labels table +--- @param branches table[] local function scan_for_atom_markers(token, at_pos, labels, branches) - local i = 1 - local len = #token - while i <= len do - i = skip_ws_and_cmt(token, i) - if i > len then break end - local c = token:sub(i, i) - if is_alpha(c) then - local ident, after = read_ident(token, i) - if ident == "atom_label" then + local pos = 1 + local tok_len = #token + while pos <= tok_len do + pos = duffle.skip_ws_and_cmt(token, pos) + if pos > tok_len then break end + local ch = token:sub(pos, pos) + if duffle.is_alpha(ch) then + local ident, after = duffle.read_ident(token, pos) + if ident == LABEL_MARKER then local args, after_paren = extract_ident_args(token, after) - if #args >= 1 then labels[args[1]] = at_pos end - if after_paren then i = after_paren else i = after end - elseif ident == "atom_offset" then + record_label_marker(labels, args, at_pos) + pos = after_paren or after + elseif ident == OFFSET_MARKER then local args, after_paren = extract_ident_args(token, after) - if #args >= 2 then table.insert(branches, {pos = at_pos, target = args[2], tag = args[1]}) end - if after_paren then i = after_paren else i = after end + record_offset_marker(branches, args, at_pos) + pos = after_paren or after else - i = after + pos = after end else - local nx = skip_str_or_cmt(token, i) - if nx > i then i = nx else i = i + 1 end + local nx = duffle.skip_str_or_cmt(token, pos) + pos = (nx > pos) and nx or (pos + 1) end end end --- Find the end position (just past the closing ')') of the first --- atom_label/atom_offset call in `tok`. Returns 0 if no such call. +--- @param tok string +--- @return integer -- 0 if no marker call found; otherwise end-1 (just past ')') local function find_marker_call_end(tok) - local i = 1 - local len = #tok - while i <= len do - i = skip_ws_and_cmt(tok, i) - if i > len then break end - local c = tok:sub(i, i) - if is_space(c) then - i = i + 1 - elseif c == "/" then + local pos = 1 + local tok_len = #tok + while pos <= tok_len do + pos = duffle.skip_ws_and_cmt(tok, pos) + if pos > tok_len then break end + local ch = tok:sub(pos, pos) + if duffle.is_space(ch) then + pos = pos + 1 + elseif ch == "/" then -- comment — skip past it (delegated to duffle.skip_str_or_cmt) - local nx = skip_str_or_cmt(tok, i) - if nx > i then i = nx else i = i + 1 end + local nx = duffle.skip_str_or_cmt(tok, pos) + pos = (nx > pos) and nx or (pos + 1) else - local ident, after = read_ident(tok, i) - if ident == "atom_label" or ident == "atom_offset" then - local j = skip_ws_and_cmt(tok, after) - if tok:sub(j, j) == "(" then - local _, end_paren = read_parens(tok, j) + local ident, after_ident = duffle.read_ident(tok, pos) + if ident == LABEL_MARKER or ident == OFFSET_MARKER then + local open_paren = duffle.skip_ws_and_cmt(tok, after_ident) + if tok:sub(open_paren, open_paren) == "(" then + local _, end_paren = duffle.read_parens(tok, open_paren) return end_paren - 1 end return 0 end - i = after or (i + 1) + pos = after_ident or (pos + 1) end end return 0 end -- ════════════════════════════════════════════════════════════════════════════ --- Atom scanner (ported from offset_gen.meta.lua lines 245-321) +-- Atom scanner -- ════════════════════════════════════════════════════════════════════════════ ---- Skip C qualifier keywords (static, const, etc.) and return the position ---- past the last qualifier. -local function skip_qualifiers(source, i) - local keywords = { - ["static"] = true, ["const"] = true, ["volatile"] = true, - ["extern"] = true, ["register"] = true, ["auto"] = true, - ["inline"] = true, ["typedef"] = true, - ["internal"]= true, ["LP_"] = true, ["global"] = true, ["gkknown"] = true, - } +--- Skip C qualifier keywords (`static`, `const`, etc.) and return the +--- position past the last qualifier. +--- @param source string +--- @param pos integer +--- @return integer +local function skip_qualifiers(source, pos) while true do - i = skip_ws_and_cmt(source, i) - local ident, after = read_ident(source, i) - if not ident then return i end - if keywords[ident] then i = after else return i end + pos = duffle.skip_ws_and_cmt(source, pos) + local ident, after = duffle.read_ident(source, pos) + if not ident then return pos end + if QUALIFIER_KEYWORDS[ident] then pos = after else return pos end end end ---- Find every MipsAtom_(name) { ... } in a source. +-- (internal) Try to parse the wrapped atom form: `MipsAtom_() { ... }`. +-- Returns the parsed Atom (name + body + position past body), or nil if +-- the form didn't match. +-- @param source_text string +-- @param after_pos integer -- position just past `MipsAtom_` +-- @return Atom|nil +local function try_wrapped_atom(source_text, after_pos) + local paren_pos = duffle.skip_ws_and_cmt(source_text, after_pos) + if source_text:sub(paren_pos, paren_pos) ~= "(" then return nil end + local inner, after_paren = duffle.read_parens(source_text, paren_pos) + + local name_start = 1 + while name_start <= #inner and duffle.is_space(inner:sub(name_start, name_start)) do + name_start = name_start + 1 + end + local name_end = name_start + while name_end <= #inner and duffle.is_alnum(inner:sub(name_end, name_end)) do + name_end = name_end + 1 + end + local name = inner:sub(name_start, name_end - 1) + if name == "" then return nil end + + local brace_pos = duffle.scan_to_char(source_text, "{", after_paren) + if not brace_pos then return nil end + local body, after_brace = duffle.read_braces(source_text, brace_pos) + return { name = name, body = body, after_brace = after_brace } +end + +-- (internal) Try to parse the raw atom form: `MipsCode code_ { ... }`. +-- @param source_text string +-- @param after_pos integer -- position just past `MipsCode` +-- @return Atom|nil +local function try_raw_atom(source_text, after_pos) + local next_pos = duffle.skip_ws_and_cmt(source_text, after_pos) + local next_ident, next_after = duffle.read_ident(source_text, next_pos) + if not next_ident then return nil end + if not starts_with(next_ident, CODE_RAW_PREFIX) then return nil end + if #next_ident <= CODE_RAW_PREFIX_LEN then return nil end + local atom_name = next_ident:sub(CODE_RAW_PREFIX_LEN + 1) + local brace_pos = duffle.scan_to_char(source_text, "{", next_after) + if not brace_pos then return nil end + local body, after_brace = duffle.read_braces(source_text, brace_pos) + return { name = atom_name, body = body, after_brace = after_brace } +end + +--- Find every `MipsAtom_(name) { ... }` (or raw `MipsCode code_ { ... }`) +--- declaration in a source. +--- @param source_text string +--- @return Atom[] local function find_atoms(source_text) - local atoms = {} - local len = #source_text - local i = 1 + local atoms = {} + local pos = 1 + local src_len = #source_text - local function try_wrapped(after_pos) - local paren_pos = skip_ws_and_cmt(source_text, after_pos) - if source_text:sub(paren_pos, paren_pos) ~= "(" then return nil end - local inner, after_paren = read_parens(source_text, paren_pos) - local n = 1 - while n <= #inner and is_space(inner:sub(n, n)) do n = n + 1 end - local ns = n - while n <= #inner and is_alnum(inner:sub(n, n)) do n = n + 1 end - local name = inner:sub(ns, n - 1) - if name == "" then return nil end - -- Find the brace after the parens. - local brace_pos = duffle.scan_to_char(source_text, "{", after_paren) - if not brace_pos then return nil end - local body, after_brace = read_braces(source_text, brace_pos) - return {name = name, body = body, after_brace = after_brace} - end + while pos <= src_len do + pos = duffle.skip_ws_and_cmt(source_text, pos); if pos > src_len then break end + pos = skip_qualifiers(source_text, pos); if pos > src_len then break end - local function try_raw(after_pos) - local next_pos = skip_ws_and_cmt(source_text, after_pos) - local next_ident, next_after = read_ident(source_text, next_pos) - if not next_ident then return nil end - if not starts_with(next_ident, "code_") then return nil end - if #next_ident <= 5 then return nil end - local atom_name = next_ident:sub(6) - local brace_pos = duffle.scan_to_char(source_text, "{", next_after) - if not brace_pos then return nil end - local body, after_brace = read_braces(source_text, brace_pos) - return {name = atom_name, body = body, after_brace = after_brace} - end - - while i <= len do - i = skip_ws_and_cmt(source_text, i); if i > len then break end - i = skip_qualifiers(source_text, i); if i > len then break end - local ident, after = read_ident(source_text, i) + local ident, after = duffle.read_ident(source_text, pos) if not ident then - i = i + 1 - elseif ident == "MipsAtom_" then - local atom = try_wrapped(after) + pos = pos + 1 + elseif ident == ATOM_PREFIX then + local atom = try_wrapped_atom(source_text, after) if atom then - table.insert(atoms, {name = atom.name, body = atom.body}) - i = atom.after_brace + atoms[#atoms + 1] = { name = atom.name, body = atom.body } + pos = atom.after_brace else - i = i + 1 + pos = pos + 1 end - elseif ident == "MipsCode" then - local atom = try_raw(after) + elseif ident == CODE_DECL then + local atom = try_raw_atom(source_text, after) if atom then - table.insert(atoms, {name = atom.name, body = atom.body}) - i = atom.after_brace + atoms[#atoms + 1] = { name = atom.name, body = atom.body } + pos = atom.after_brace else - i = after + pos = after end else - i = after + pos = after end end return atoms end -- ════════════════════════════════════════════════════════════════════════════ --- Per-atom body scan (ported from offset_gen.meta.lua lines 207-239) +-- Per-atom body scan -- ════════════════════════════════════════════════════════════════════════════ +-- (internal) Count words emitted by the rest of `tok` after a marker call +-- (the marker call itself emits 0 words, but the source pattern may bundle +-- the marker with the next instruction on the same line, separated by no +-- top-level comma). Returns the word count contributed by that rest. +-- @param tok string +-- @param word_counts table +-- @return integer +local function count_marker_rest(tok, word_counts) + local marker_end = find_marker_call_end(tok) + if marker_end <= 0 or marker_end >= #tok then return 0 end + local rest = duffle.trim(tok:sub(marker_end + 1)) + if rest == "" then return 0 end + return count_token_words(rest, word_counts) +end + +-- (internal) Is this token a marker call (`atom_label` or `atom_offset`)? +-- @param tok string +-- @return boolean +local function is_marker_token(tok) + local leading_ident = duffle.read_ident(tok, 1) + return leading_ident == LABEL_MARKER or leading_ident == OFFSET_MARKER +end + --- Scan an atom body for labels + branches, count total words. --- Returns (labels, branches, total_words). +--- @param body string +--- @param word_counts table +--- @return table, table[], integer local function scan_atom_body(body, word_counts) local pos = 0 local labels = {} local branches = {} - for _, tok in ipairs(split_top_level_commas(body)) do - local k = 1 - local tlen = #tok - while k <= tlen and is_space(tok:sub(k, k)) do k = k + 1 end - local leading_ident = read_ident(tok, k) - if leading_ident == "atom_label" or leading_ident == "atom_offset" then + for _, tok in ipairs(duffle.split_top_level_commas(body)) do + if is_marker_token(tok) then -- Marker call: record at the current pos, do NOT advance pos. - -- But the source pattern may bundle the marker with the next - -- instruction on a new line (no top-level comma between them). - -- In that case, the rest of `tok` after the marker call is - -- a real instruction that must still be counted. scan_for_atom_markers(tok, pos, labels, branches) - local marker_end = find_marker_call_end(tok) - if marker_end > 0 and marker_end < #tok then - local rest = trim(tok:sub(marker_end + 1)) - if rest ~= "" then - local rest_words = count_token_words(rest, word_counts) - pos = pos + rest_words - end - end + pos = pos + count_marker_rest(tok, word_counts) else local words = count_token_words(tok, word_counts) scan_for_atom_markers(tok, pos, labels, branches) @@ -291,10 +410,14 @@ local function scan_atom_body(body, word_counts) end -- ════════════════════════════════════════════════════════════════════════════ --- Offset computation + header generation (ported lines 327-383) +-- Offset computation + header generation -- ════════════════════════════════════════════════════════════════════════════ ---- Compute branch offsets as (target_word - branch_word - 1). +-- Compute branch offsets as `target_word - branch_word - 1` (the +-- standard MIPS branch-immediate encoding). +-- @param labels table +-- @param branches table[] +-- @return BranchOffset[] local function compute_offsets(labels, branches) local results = {} for _, br in ipairs(branches) do @@ -302,17 +425,55 @@ local function compute_offsets(labels, branches) if not target then error("Branch target '" .. br.target .. "' has no atom_label (at word " .. br.pos .. ")") end - table.insert(results, {target = br.target, tag = br.tag, offset = target - br.pos - 1}) + results[#results + 1] = { target = br.target, tag = br.tag, offset = target - br.pos - 1 } end return results end ---- Generate the per-source .offsets.h header. +-- (internal) Build a constant-table entry `{macro_name, enum_name, value}` +-- from a BranchOffset. +-- @param r BranchOffset +-- @return table +local function make_offset_const(r) + return { + macro_name = OFFSET_MACRO_PREFIX .. r.tag .. "_" .. r.target, + enum_name = OFFSET_ENUM_PREFIX .. r.tag .. "_" .. r.target, + value = r.offset, + } +end + +-- (internal) Emit one atom's offset constants + enum into the lines buffer. +-- @param add fun(s: string) +-- @param atom AtomData +local function emit_atom_offsets(add, atom) + if #atom.offsets == 0 then return end + add("// --- atom: " .. atom.name .. " (" .. atom.total_words .. " words) ---") + add("") + local consts = {} + for _, r in ipairs(atom.offsets) do + consts[#consts + 1] = make_offset_const(r) + end + for _, c in ipairs(consts) do + add("#define " .. pad_right(c.macro_name, OFFSET_MACRO_COL) .. " " .. c.value) + end + add("") + add("enum {") + for _, c in ipairs(consts) do + add(" " .. c.enum_name .. " = " .. c.macro_name .. ",") + end + add("};") + add("") +end + +-- Generate the per-source .offsets.h header. +-- @param source_path string +-- @param atoms_data AtomData[] +-- @return string local function generate_header(source_path, atoms_data) - local basename = basename_no_ext(source_path) + local basename = duffle.basename_no_ext(source_path) local lines = {} - local function add(s) table.insert(lines, s) end + local function add(s) lines[#lines + 1] = s end add("// Auto-generated by ps1_meta.lua (passes/offsets.lua) — DO NOT EDIT") add("// Source: " .. source_path) @@ -322,28 +483,7 @@ local function generate_header(source_path, atoms_data) add("") add("") for _, atom in ipairs(atoms_data) do - if #atom.offsets > 0 then - add("// --- atom: " .. atom.name .. " (" .. atom.total_words .. " words) ---") - add("") - local consts = {} - for _, r in ipairs(atom.offsets) do - table.insert(consts, { - macro_name = "_atom_offset_" .. r.tag .. "_" .. r.target, - enum_name = "atom_offset_" .. r.tag .. "_" .. r.target, - value = r.offset, - }) - end - for _, c in ipairs(consts) do - add("#define " .. pad_right(c.macro_name, 44) .. " " .. c.value) - end - add("") - add("enum {") - for _, c in ipairs(consts) do - add(" " .. c.enum_name .. " = " .. c.macro_name .. ",") - end - add("};") - add("") - end + emit_atom_offsets(add, atom) end add("#pragma endregion " .. basename) add("") @@ -351,13 +491,41 @@ local function generate_header(source_path, atoms_data) end -- ════════════════════════════════════════════════════════════════════════════ --- M.run — orchestrator entry +-- M — module exports -- ════════════════════════════════════════════════════════════════════════════ ---- @class M - local M = {} +-- (internal) Process one source: find atoms, scan bodies, write header. +-- Returns the offsets_h path if a header was written, or nil. +-- @param ctx PassCtx +-- @param src SourceFile +-- @return string|nil -- the offsets_h path +local function process_source(ctx, src) + local atoms = find_atoms(src.text) + if #atoms == 0 then return nil end + + local atoms_data = {} + for _, atom in ipairs(atoms) do + local labels, branches, total = scan_atom_body(atom.body, ctx.shared.word_counts) + atoms_data[#atoms_data + 1] = { + name = atom.name, + total_words = total, + offsets = compute_offsets(labels, branches), + } + end + + local out_path = src.dir .. "/gen/" .. duffle.basename_no_ext(src.dir) .. ".offsets.h" + if not ctx.dry_run then + duffle.ensure_dir(duffle.dirname(out_path)) + duffle.write_file(out_path, generate_header(src.path, atoms_data)) + end + return out_path +end + +--- Run the offsets pass. For each source, emits a per-module +--- `.offsets.h` containing `#define _atom_offset_F_T = N` +--- constants for every `atom_offset(F, T)` reference in the source's atoms. --- @param ctx PassCtx --- @return PassResult function M.run(ctx) @@ -366,25 +534,9 @@ function M.run(ctx) local warnings = {} for _, src in ipairs(ctx.sources) do - local atoms = find_atoms(src.text) - if #atoms > 0 then - local atoms_data = {} - for _, atom in ipairs(atoms) do - local labels, branches, total = scan_atom_body(atom.body, ctx.shared.word_counts) - local offsets = compute_offsets(labels, branches) - table.insert(atoms_data, { - name = atom.name, - total_words = total, - offsets = offsets, - }) - end - - local out_path = src.dir .. "/gen/" .. basename_no_ext(src.dir) .. ".offsets.h" - if not ctx.dry_run then - ensure_dir(dirname(out_path)) - write_file(out_path, generate_header(src.path, atoms_data)) - end - table.insert(outputs, { offsets_h = out_path }) + local out_path = process_source(ctx, src) + if out_path then + outputs[#outputs + 1] = { offsets_h = out_path } end end diff --git a/scripts/passes/report.lua b/scripts/passes/report.lua index 43b8e55..d215da4 100644 --- a/scripts/passes/report.lua +++ b/scripts/passes/report.lua @@ -1,66 +1,174 @@ --- passes/report.lua --- --- Render the per-project summary (build/gen/annotation_validation.txt) --- + the per-MODULE annotation reports (build/gen/.annotations.txt). --- Aggregates errors + warnings from upstream annotation pass results. --- --- The annotation pass stashes its per-MODULE results in --- ctx.flags._annot_results (set by passes/annotation.lua). This report --- pass renders them. Each entry contains a `dir`, `dir_basename`, and --- `atoms_count`; the detailed per-source data still lives in the --- annotation pass's internal result objects, accessible via the shared --- `ctx.sources` list (re-validated by reference, not by value). --- --- Coding standard: tabs (1/level), EmmyLua annotations, no regex. +--- passes/report.lua — Per-MODULE annotation report renderer + +--- project-wide summary writer. +--- +--- Two output files per build: +--- - `build/gen/.annotations.txt` — one per source-directory +--- containing atoms; aggregates across all sources in the directory. +--- - `build/gen/annotation_validation.txt` — the project summary. +--- +--- The annotation pass stashes per-MODULE summary entries in +--- `ctx.flags._annot_results` (set by `passes/annotation.lua`). This +--- pass re-validates each source via `annotation.validate()` to get +--- the detailed per-source results needed for the report. The cost is +--- acceptable: `validate()` is fast (~5ms per source) and runs once. +--- +--- **Conventions**: tabs (1/level), EmmyLua annotations, no regex, +--- Lua 5.3 compatible. See +--- `C:\projects\Pikuma\ps1-ai\conductor\code_styleguides\lua.md`. -- ════════════════════════════════════════════════════════════════════════════ -- Module-scope requires + package.path setup -- ════════════════════════════════════════════════════════════════════════════ -local script_path = arg and arg[0] or "?" -local last_sep = 0 -for i = 1, #script_path do - local c = script_path:sub(i, i) - if c == "/" or c == "\\" then last_sep = i end -end -local script_dir = last_sep == 0 and "./" or script_path:sub(1, last_sep) -package.path = script_dir .. "../?.lua;" .. script_dir .. "../?/init.lua;" .. script_dir .. "?.lua;" .. package.path +-- Resolve `arg[0]` to an absolute-ish script directory so that +-- `require("duffle")` resolves against `scripts/` regardless of CWD. +-- Note: this boilerplate is duplicated in 6 other entry scripts; a +-- Phase-6 extraction target (`duffle.setup_package_path()`). +-- Bootstrap: see `ps1_meta.lua` for the rationale. +dofile((arg[0]:match("(.*[/\\])") or "./") .. "duffle_paths.lua") +local duffle = require("duffle") -local duffle = require("duffle") -local ensure_dir = duffle.ensure_dir -local write_file = duffle.write_file +-- ════════════════════════════════════════════════════════════════════════════ +-- Constants +-- ════════════════════════════════════════════════════════════════════════════ + +-- Section separators used in the rendered text reports. The thin rules +-- are hand-tuned to align with the per-section content width; do not +-- change without also checking the section renderers below. +local RULE_THICK = "========================================================" +local SECTION_HEADER_ATOMS = "── Atoms ────────────────────────────────────────────────" +local SECTION_HEADER_ANNOTS = "── Annotations ──────────────────────────────────────────" +local SECTION_HEADER_BINDS = "── Binds_* structs ──────────────────────────────────────" +local SECTION_HEADER_MACROS = "── Macro word-count declarations ─────────────────────────" +local SECTION_HEADER_ERRORS = "── Errors ──────────────────────────────────────────────" +local SECTION_HEADER_WARNINGS = "── Warnings ────────────────────────────────────────────" + +-- Lua pattern that captures the basename (last path segment) of a +-- forward- or back-slash separated path. +local BASENAME_PATTERN = "([^/\\]+)$" + +-- Debug flag name — set to truthy in `_G` to enable verbose logging. +local DEBUG_FLAG = "_DEBUG_REPORT" + +-- Pass identifier for log messages. +local PASS_NAME = "report" + +-- ════════════════════════════════════════════════════════════════════════════ +-- Type declarations +-- ════════════════════════════════════════════════════════════════════════════ + +--- @class SourceFile +--- @field path string -- absolute path to the source file +--- @field text string -- the full source text +--- @field dir string -- the directory containing the source +--- @field basename string -- filename without extension + +--- @class PassCtx +--- @field sources SourceFile[] -- all source files in the build +--- @field metadata_path string -- path to word_count.metadata.h +--- @field shared table -- cross-pass shared state +--- @field out_root string -- output root (e.g. "build/gen") +--- @field project_root string -- project root (e.g. "code/") +--- @field upstream table -- per-pass upstream outputs +--- @field flags table -- CLI flags + per-pass stash +--- @field flags._annot_results ModuleEntry[] -- stashed by annotation pass +--- @field dry_run boolean -- if true, compute but don't write +--- @field verbose boolean -- if true, log diagnostic info + +--- @class PassResult +--- @field outputs table[] -- {kind=, path=} entries describing emit files +--- @field errors table[] -- {line=, msg=} entries; build-stops +--- @field warnings table[] -- {line=, msg=} entries; build-succeeds + +-- Shapes produced by `passes/annotation.lua`'s `M.validate()`. + +--- @class AtomEntry +--- @field name string -- atom name (e.g. "cube_g4_face") +--- @field line integer -- source line of the atom declaration + +--- @class AnnotEntry +--- @field line integer -- source line +--- @field macro string -- the macro name (e.g. "atom_reads") +--- @field name string -- the atom name (if a `name(...)` was given) +--- @field kind string -- "atom_info" | "atom_bind" | ... +--- @field binds string|nil -- Binds_X name if any +--- @field reads string[] -- R_* names (read targets) +--- @field writes string[] -- R_* names (write targets) +--- @field error string|nil -- error message if annotation was malformed + +--- @class BindsField +--- @field name string -- field name +--- @field offset integer -- byte offset within the Binds_X struct + +--- @class BindsStruct +--- @field name string -- struct name (e.g. "Binds_Floor") +--- @field line integer -- source line of the typedef +--- @field bytes integer -- total byte size +--- @field fields BindsField[] -- the field list + +--- @class MacroEntry +--- @field name string -- macro name (e.g. "WORD_COUNT(my_macro, 4)") +--- @field line integer -- source line +--- @field words integer -- declared word count + +--- @class Finding +--- @field line integer -- source line +--- @field msg string -- finding message + +--- @class AnnotationResult +--- @field source string -- set by this pass; original source path +--- @field atoms AtomEntry[] -- atom declarations in this source +--- @field annots AnnotEntry[] -- annotation entries +--- @field macros MacroEntry[] -- macro word-count declarations +--- @field binds BindsStruct[] -- Binds_* struct declarations +--- @field errors Finding[] -- errors from validation +--- @field warnings Finding[] -- warnings from validation +--- @field info table -- info summary (not rendered here) + +--- @class ModuleEntry +--- @field dir string -- absolute directory path +--- @field dir_basename string -- basename (e.g. "duffle", "gte_hello") +--- @field atoms_count integer -- pre-counted atoms for filtering + +--- @class ModuleReport +--- @field dir string -- module directory +--- @field sources SourceFile[] -- sources in this module +--- @field results AnnotationResult[] -- per-source validate() results + +--- @class ProjectReport +--- @field results AnnotationResult[] -- all per-source results -- ════════════════════════════════════════════════════════════════════════════ -- Per-MODULE annotation report (aggregated across all sources in a dir) -- ════════════════════════════════════════════════════════════════════════════ --- --- We no longer re-validate the source here; the annotation pass has --- already done the work and stored the per-source results internally --- (re-validating here would double the work). Instead, this pass reads --- the per-source `validate()` results via a callback re-validation: --- the annotation pass stashes the per-module summary, and we re-validate --- each source once more to render the per-module report. The cost is --- acceptable: validate() is fast (~5ms per source for our 19 sources) --- and the report pass runs once at the end of the build. --- --- In a future refactor we could pass the validate() result directly --- through ctx.upstream.annotation to avoid the re-validation, but the --- current approach keeps the passes loosely coupled. -local function render_module_report(dir, sources, results) - local lines = {} - local function add(s) lines[#lines + 1] = s end +-- Extract the basename (last path segment) of a forward- or back-slash +-- separated path. Returns the input unchanged if no separator is found. +-- @param path string +-- @return string +local function source_basename(path) + return path:match(BASENAME_PATTERN) or path +end - add("========================================================") - add("ANNOTATION PASS — module " .. dir:match("([^/\\]+)$") or dir) - add("========================================================") - add(string.format("Sources: %d", #sources)) - for _, s in ipairs(sources) do - add(" " .. s.path) +-- (internal) Format a single annotation entry as one rendered line. +-- @param a AnnotEntry +-- @param src_name string +-- @return string +local function format_annot_line(a, src_name) + if a.error then + return string.format(" ✗ line %d %s [ERROR: %s] [%s]", a.line, a.macro or "?", a.error, src_name) end - add("") + local line = string.format(" ● line %d %s [%s]", a.line, a.name, src_name) + if a.binds then line = line .. " binds=" .. a.binds end + if #a.reads > 0 then line = line .. " reads={" .. table.concat(a.reads, ",") .. "}" end + if #a.writes > 0 then line = line .. " writes={" .. table.concat(a.writes, ",") .. "}" end + return line +end - -- Tally totals across the module +-- (internal) Tally totals across all results in a module. +-- @param results AnnotationResult[] +-- @return integer, integer, integer, integer, integer, integer +local function tally_module_totals(results) local total_atoms, total_annots, total_binds, total_macros = 0, 0, 0, 0 local total_errors, total_warnings = 0, 0 for _, r in ipairs(results) do @@ -71,41 +179,38 @@ local function render_module_report(dir, sources, results) total_errors = total_errors + #r.errors total_warnings = total_warnings + #r.warnings end - add(string.format("Atoms: %d Annotations: %d Binds structs: %d Macro decls: %d", - total_atoms, total_annots, total_binds, total_macros)) - add("") + return total_atoms, total_annots, total_binds, total_macros, total_errors, total_warnings +end - -- Per-source breakdown (each source's atoms + annnots + binds) - -- Aggregated under the module's "── Atoms ──" section. - add("── Atoms ────────────────────────────────────────────────") +-- (internal) Section renderer: per-source atom declarations. +local function render_module_atoms_section(add, results) + add(SECTION_HEADER_ATOMS) for _, r in ipairs(results) do - local src_name = r.source:match("([^/\\]+)$") or r.source + local src_name = source_basename(r.source) for _, a in ipairs(r.atoms) do add(string.format(" MipsAtom_(%s) line %d [%s]", a.name, a.line, src_name)) end end add("") +end - add("── Annotations ──────────────────────────────────────────") +-- (internal) Section renderer: per-source annotation entries. +local function render_module_annots_section(add, results) + add(SECTION_HEADER_ANNOTS) for _, r in ipairs(results) do - local src_name = r.source:match("([^/\\]+)$") or r.source + local src_name = source_basename(r.source) for _, a in ipairs(r.annots) do - if a.error then - add(string.format(" ✗ line %d %s [ERROR: %s] [%s]", a.line, a.macro or "?", a.error, src_name)) - else - local line = string.format(" ● line %d %s [%s]", a.line, a.name, src_name) - if a.binds then line = line .. " binds=" .. a.binds end - if #a.reads > 0 then line = line .. " reads={" .. table.concat(a.reads, ",") .. "}" end - if #a.writes > 0 then line = line .. " writes={" .. table.concat(a.writes, ",") .. "}" end - add(line) - end + add(format_annot_line(a, src_name)) end end add("") +end - add("── Binds_* structs ──────────────────────────────────────") +-- (internal) Section renderer: per-source Binds_* struct declarations. +local function render_module_binds_section(add, results) + add(SECTION_HEADER_BINDS) for _, r in ipairs(results) do - local src_name = r.source:match("([^/\\]+)$") or r.source + local src_name = source_basename(r.source) for _, b in ipairs(r.binds) do add(string.format(" %s line %d %d bytes [%s]", b.name, b.line, b.bytes, src_name)) for _, f in ipairs(b.fields) do @@ -114,35 +219,79 @@ local function render_module_report(dir, sources, results) end end add("") +end - add("── Macro word-count declarations ─────────────────────────") +-- (internal) Section renderer: per-source macro word-count declarations. +local function render_module_macros_section(add, results) + add(SECTION_HEADER_MACROS) for _, r in ipairs(results) do - local src_name = r.source:match("([^/\\]+)$") or r.source + local src_name = source_basename(r.source) for _, m in ipairs(r.macros) do add(string.format(" %s line %d words=%d [%s]", m.name, m.line, m.words, src_name)) end end add("") +end - add("── Errors ──────────────────────────────────────────────") - if total_errors == 0 then add(" (none)") end - for _, r in ipairs(results) do - local src_name = r.source:match("([^/\\]+)$") or r.source - for _, e in ipairs(r.errors) do - add(string.format(" ✗ line %d %s [%s]", e.line, e.msg, src_name)) +-- (internal) Section renderer: per-source errors (one-line + "(none)" if empty). +local function render_module_errors_section(add, results, total_errors) + add(SECTION_HEADER_ERRORS) + if total_errors == 0 then + add(" (none)") + else + for _, r in ipairs(results) do + local src_name = source_basename(r.source) + for _, e in ipairs(r.errors) do + add(string.format(" ✗ line %d %s [%s]", e.line, e.msg, src_name)) + end end end add("") +end - add("── Warnings ────────────────────────────────────────────") - if total_warnings == 0 then add(" (none)") end - for _, r in ipairs(results) do - local src_name = r.source:match("([^/\\]+)$") or r.source - for _, w in ipairs(r.warnings) do - add(string.format(" ⚠ line %d %s [%s]", w.line, w.msg, src_name)) +-- (internal) Section renderer: per-source warnings (one-line + "(none)" if empty). +local function render_module_warnings_section(add, results, total_warnings) + add(SECTION_HEADER_WARNINGS) + if total_warnings == 0 then + add(" (none)") + else + for _, r in ipairs(results) do + local src_name = source_basename(r.source) + for _, w in ipairs(r.warnings) do + add(string.format(" ⚠ line %d %s [%s]", w.line, w.msg, src_name)) + end end end add("") +end + +--- Render the per-MODULE annotation report (one `.annotations.txt`). +--- @param dir string -- module directory path +--- @param sources SourceFile[] -- sources in this module +--- @param results AnnotationResult[] -- per-source validate() results +--- @return string -- the rendered report text +local function render_module_report(dir, sources, results) + local lines = {} + local function add(s) lines[#lines + 1] = s end + + add(RULE_THICK) + add("ANNOTATION PASS — module " .. source_basename(dir)) + add(RULE_THICK) + add(string.format("Sources: %d", #sources)) + for _, s in ipairs(sources) do add(" " .. s.path) end + add("") + + local total_atoms, total_annots, total_binds, total_macros, total_errors, total_warnings = tally_module_totals(results) + add(string.format("Atoms: %d Annotations: %d Binds structs: %d Macro decls: %d", + total_atoms, total_annots, total_binds, total_macros)) + add("") + + render_module_atoms_section(add, results) + render_module_annots_section(add, results) + render_module_binds_section(add, results) + render_module_macros_section(add, results) + render_module_errors_section(add, results, total_errors) + render_module_warnings_section(add, results, total_warnings) return table.concat(lines, "\n") .. "\n" end @@ -151,13 +300,17 @@ end -- Per-project summary (ported from tape_atom_annotation_pass.lua:1488-1528) -- ════════════════════════════════════════════════════════════════════════════ +--- Render the per-project summary (`build/gen/annotation_validation.txt`). +--- Aggregates totals across all sources; lists per-source error counts +--- if any source has errors. +--- @param all_results AnnotationResult[] +--- @return string local function render_project_report(all_results) local lines = {} local function add(s) lines[#lines + 1] = s end local total_atoms, total_annots, total_macros, total_binds = 0, 0, 0, 0 local total_errors, total_warnings = 0, 0 - for _, r in ipairs(all_results) do total_atoms = total_atoms + #r.atoms total_annots = total_annots + #r.annots @@ -167,9 +320,9 @@ local function render_project_report(all_results) total_warnings = total_warnings + #r.warnings end - add("========================================================") + add(RULE_THICK) add("ANNOTATION VALIDATION — project summary") - add("========================================================") + add(RULE_THICK) add("") add(string.format("Atoms: %d", total_atoms)) add(string.format("Annotations: %d", total_annots)) @@ -184,7 +337,7 @@ local function render_project_report(all_results) add("Per-source error counts:") for _, r in ipairs(all_results) do if #r.errors > 0 then - local src_name = r.source:match("([^/\\]+)$") or r.source + local src_name = source_basename(r.source) add(string.format(" %s : %d error(s)", src_name, #r.errors)) end end @@ -195,13 +348,72 @@ local function render_project_report(all_results) end -- ════════════════════════════════════════════════════════════════════════════ --- M.run — orchestrator entry +-- Orchestration helpers -- ════════════════════════════════════════════════════════════════════════════ ---- @class M +-- Group source files by their `dir` field. Used to mirror the +-- per-DIRECTORY partitioning the annotation pass uses. +-- @param sources SourceFile[] +-- @return table -- map of dir -> sources in that dir +local function group_sources_by_dir(sources) + local by_dir = {} + for _, src in ipairs(sources) do + by_dir[src.dir] = by_dir[src.dir] or {} + table.insert(by_dir[src.dir], src) + end + return by_dir +end + +-- (internal) Validate each source in `dir_sources` via the annotation pass, +-- tagging each result with `result.source = src.path` for downstream rendering. +-- Returns the list of module results + the flat list of all results (for the +-- project-wide summary). +-- @param ctx PassCtx +-- @param dir_sources SourceFile[] +-- @return AnnotationResult[], AnnotationResult[] +local function validate_module_sources(ctx, dir_sources) + local annotation = require("passes.annotation") + local module_results = {} + local all_results = {} + for _, src in ipairs(dir_sources) do + local result = annotation.validate(ctx, src) + result.source = src.path + module_results[#module_results + 1] = result + all_results[#all_results + 1] = result + end + return module_results, all_results +end + +-- (internal) Does this module's results contain anything worth emitting? +-- @param module_results AnnotationResult[] +-- @return boolean +local function module_has_content(module_results) + for _, r in ipairs(module_results) do + if #r.atoms > 0 or #r.annots > 0 or #r.binds > 0 + or #r.macros > 0 or #r.errors > 0 or #r.warnings > 0 then + return true + end + end + return false +end + +-- (internal) Log a debug message if `_G[DEBUG_FLAG]` is truthy. +-- @param fmt string +local function debug_log(fmt, ...) + if _G[DEBUG_FLAG] then + io.stderr:write(string.format("[%s] " .. fmt, PASS_NAME, ...)) + end +end + +-- ════════════════════════════════════════════════════════════════════════════ +-- M — module exports +-- ════════════════════════════════════════════════════════════════════════════ local M = {} +--- Run the report pass. Renders one `.annotations.txt` +--- per source-directory that has content, plus the project-wide +--- `annotation_validation.txt` summary. --- @param ctx PassCtx --- @return PassResult function M.run(ctx) @@ -209,67 +421,39 @@ function M.run(ctx) local errors = {} local warnings = {} - -- The annotation pass stashes per-MODULE summary entries in - -- ctx.flags._annot_results. Each entry has { dir, dir_basename, - -- atoms_count }. To render the per-module report we need the - -- detailed per-source `validate()` results, which we re-compute - -- here by calling annotation.validate() on each source. The cost - -- is acceptable (validate() is fast, no GTE/GPU work). local module_entries = (ctx.flags and ctx.flags._annot_results) or {} + local by_dir = group_sources_by_dir(ctx.sources) - -- Group sources by dir (same partitioning the annotation pass uses). - local by_dir = {} - for _, src in ipairs(ctx.sources) do - by_dir[src.dir] = by_dir[src.dir] or {} - table.insert(by_dir[src.dir], src) - end + if not ctx.dry_run then duffle.ensure_dir(ctx.out_root) end - -- Render per-MODULE reports. - if not ctx.dry_run then ensure_dir(ctx.out_root) end local all_results_for_summary = {} for _, entry in ipairs(module_entries) do - local dir = entry.dir - local dir_basename = entry.dir_basename - local dir_sources = by_dir[dir] or {} - if _G._DEBUG_REPORT then - io.stderr:write(string.format("[report] entry: dir=%s basename=%s atoms_count=%d dir_sources=%d\n", - dir, dir_basename, entry.atoms_count, #dir_sources)) - end - -- Skip dirs with zero atoms AND zero annotations - if entry.atoms_count > 0 or #dir_sources > 0 then - -- Re-validate each source to get the detailed results. - -- (We could pass the per-source results through ctx instead; - -- current approach is simpler and the re-validation is fast.) - local annotation = require("passes.annotation") - local module_results = {} - local has_content = false - for _, src in ipairs(dir_sources) do - local result = annotation.validate(ctx, src) - result.source = src.path - module_results[#module_results + 1] = result - all_results_for_summary[#all_results_for_summary + 1] = result - if #result.atoms > 0 or #result.annots > 0 or #result.binds > 0 - or #result.macros > 0 or #result.errors > 0 or #result.warnings > 0 then - has_content = true - end + debug_log("entry: dir=%s basename=%s atoms_count=%d dir_sources=%d\n", + entry.dir, entry.dir_basename, entry.atoms_count, #(by_dir[entry.dir] or {})) + + if entry.atoms_count > 0 or #(by_dir[entry.dir] or {}) > 0 then + local dir_sources = by_dir[entry.dir] or {} + local module_results, all_results = validate_module_sources(ctx, dir_sources) + for _, r in ipairs(all_results) do + all_results_for_summary[#all_results_for_summary + 1] = r end - if has_content then - local out_path = ctx.out_root .. "/" .. dir_basename .. ".annotations.txt" + + if module_has_content(module_results) then + local out_path = ctx.out_root .. "/" .. entry.dir_basename .. ".annotations.txt" if not ctx.dry_run then - write_file(out_path, render_module_report(dir, dir_sources, module_results)) + duffle.write_file(out_path, render_module_report(entry.dir, dir_sources, module_results)) end - table.insert(outputs, { annotations_txt = out_path }) - elseif _G._DEBUG_REPORT then - io.stderr:write(string.format("[report] -> no content; skipping\n")) + outputs[#outputs + 1] = { annotations_txt = out_path } + else + debug_log(" -> no content; skipping\n") end end end - -- Render project summary (across all sources). if not ctx.dry_run and #all_results_for_summary > 0 then local summary_path = ctx.out_root .. "/annotation_validation.txt" - write_file(summary_path, render_project_report(all_results_for_summary)) - table.insert(outputs, { summary_txt = summary_path }) + duffle.write_file(summary_path, render_project_report(all_results_for_summary)) + outputs[#outputs + 1] = { summary_txt = summary_path } end return { outputs = outputs, errors = errors, warnings = warnings } diff --git a/scripts/passes/static_analysis.lua b/scripts/passes/static_analysis.lua index d69eb93..ea7583c 100644 --- a/scripts/passes/static_analysis.lua +++ b/scripts/passes/static_analysis.lua @@ -1,67 +1,138 @@ --- passes/static_analysis.lua --- --- Per-atom static-analysis checks for the tape-atom build pipeline. --- Currently ships Phase 1 checks (GTE pipeline-fill + mac_yield --- uniformity). Phases 2/3 (ABI handoff discipline, GPU port-store --- shape, per-atom cycle budget) extend this file. --- --- Workspace boundary: same conventions as annotation.lua --- - primitives from duffle.lua (read_parens, read_braces, scan_to_char, LineIndex) --- - LPeg not needed: pure hand-rolled string scanning --- - 5.3-compatible (no , no continue keyword) --- - no :match/:gmatch --- - tab indent, EmmyLua @class/@param annotations --- --- The orchestrator (ps1_meta.lua) wires this module in via the --- PASSES table: --- ["static-analysis"] = { --- module = "passes.static_analysis", --- kind = "validation", -- errors stop the build --- deps = {"word-counts", "components"}, --- out = { { kind = "report", --- path_template = "/.static_analysis.txt" } }, --- } +--- passes/static_analysis.lua — Per-atom static-analysis checks. +--- +--- The 5 checks currently shipped: +--- 1. **GTE pipeline-fill** — every `gte_cmdw_*` invocation must be +--- preceded by the minimum number of `nop` words (per +--- `duffle.duffle.GTE_PIPELINE_LATENCY`) so the COP2 pipeline latency is +--- fully retired before the command issues. +--- 2. **mac_yield uniformity** — every atom body must contain exactly +--- one `mac_yield()` call (control transfer pattern). +--- 3. **ABI handoff** — every `atom_bind(Binds_X)` must reference a +--- `typedef Struct_(Binds_X) { ... }` declaration. +--- 4. **GPU port-store shape** — per-shape (`f3`/`f4`/`g4`/etc.) the +--- sum of `mac_format_X_color` + `mac_gte_store_X_*` + +--- `mac_insert_ot_tag_X` words must equal the GP0 cmd's expected +--- packet size. +--- 5. **per-atom cycle budget** — sum each atom body's instruction +--- latencies (per `duffle.duffle.INSTRUCTION_LATENCY`); report total. +--- +--- The orchestrator (`ps1_meta.lua`) wires this module in via the +--- PASSES table: +--- `["static-analysis"] = { module = "passes.static_analysis", +--- kind = "validation", +--- deps = {"word-counts", "components"}, +--- out = { { kind = "report", +--- path_template = "/.static_analysis.txt" } } }` +--- +--- **Conventions**: tabs (1/level), EmmyLua annotations, no regex, +--- Lua 5.3 compatible. See +--- `C:\projects\Pikuma\ps1-ai\conductor\code_styleguides\lua.md`. -- ════════════════════════════════════════════════════════════════════════════ -- Module-scope requires + package.path setup -- ════════════════════════════════════════════════════════════════════════════ -local script_path = arg and arg[0] or "?" -local last_sep = 0 -for i = 1, #script_path do - local c = script_path:sub(i, i) - if c == "/" or c == "\\" then last_sep = i end -end -local script_dir = last_sep == 0 and "./" or script_path:sub(1, last_sep) -package.path = script_dir .. "../?.lua;" .. script_dir .. "../?/init.lua;" .. script_dir .. "?.lua;" .. package.path +-- `duffle.setup_package_path()` resolves `arg[0]` and prepends `scripts/` +-- (and `scripts/passes/`) to `package.path`, so `require("duffle")` +-- resolves regardless of CWD. See `duffle.lua` for the implementation. -local duffle = require("duffle") -local read_ident = duffle.read_ident -local skip_ws_and_cmt = duffle.skip_ws_and_cmt -local read_parens = duffle.read_parens -local read_braces = duffle.read_braces -local read_brackets = duffle.read_brackets -local scan_to_char = duffle.scan_to_char -local split_top_level_commas = duffle.split_top_level_commas -local trim = duffle.trim -local ensure_dir = duffle.ensure_dir -local write_file = duffle.write_file -local basename_no_ext = duffle.basename_no_ext +-- Bootstrap: see `ps1_meta.lua` for the rationale. +dofile((arg[0]:match("(.*[/\\])") or "./") .. "duffle_paths.lua") +local duffle = require("duffle") --- Latency table lives in duffle.lua (shared between this pass + future --- per-atom cycle-budget pass). Lazily read on first use. -local GTE_PIPELINE_LATENCY = duffle.GTE_PIPELINE_LATENCY +-- Domain tables (single source of truth in duffle.lua). --- GP0 domain tables (Phase 2 check #4). Same source as GTE_PIPELINE_LATENCY. -local GP0_CMD_SIZE = duffle.GP0_CMD_SIZE -local GP0_CMD_BY_SHAPE = duffle.GP0_CMD_BY_SHAPE -local GP0_MACRO_CONTRIB = duffle.GP0_MACRO_CONTRIB --- Instruction latency table (Phase 3). Per-macro cycle cost in the --- best-case (no-stall) scenario. Unknown macros default to --- UNKNOWN_INSTRUCTION_CYCLES with a warning. -local INSTRUCTION_LATENCY = duffle.INSTRUCTION_LATENCY -local UNKNOWN_INSTRUCTION_CYCLES = duffle.UNKNOWN_INSTRUCTION_CYCLES + + + + +-- ════════════════════════════════════════════════════════════════════════════ +-- Constants +-- ════════════════════════════════════════════════════════════════════════════ + +-- Atom declaration + component declaration identifiers. +local ATOM_DECL = "MipsAtom_" +local ATOM_COMP = "MipsAtomComp_" +local ATOM_COMP_PROC = "MipsAtomComp_Proc_" + +-- Marker-call identifiers inside atom bodies. +local ATOM_LABEL = "atom_label" +local ATOM_OFFSET = "atom_offset" +local ATOM_INFO = "atom_info" +local ATOM_BIND = "atom_bind" +local ATOM_READS = "atom_reads" +local ATOM_WRITES = "atom_writes" +local ATOM_YIELD = "mac_yield" +local WORD_COUNT_PRAGMA = "WORD_COUNT(" + +-- ASCII byte values used in tokenization. +local BYTE_NEWLINE = 10 +local BYTE_HASH = 35 -- '#' +local BYTE_OPEN_PAREN = 40 +local BYTE_OPEN_BRACE = 123 +local BYTE_OPEN_BRACK = 91 +local BYTE_SEMI = 59 + +-- Per-check output paths (relative to ctx.out_root). +local OUTPUT_EXTENSION = ".static_analysis.txt" + +-- ════════════════════════════════════════════════════════════════════════════ +-- Type declarations +-- ════════════════════════════════════════════════════════════════════════════ + +--- @class SourceFile +--- @field path string -- absolute path to the source file +--- @field text string -- the full source text +--- @field dir string -- the directory containing the source +--- @field basename string -- filename without extension + +--- @class PassCtx +--- @field sources SourceFile[] +--- @field metadata_path string +--- @field shared table +--- @field shared.word_counts table +--- @field out_root string +--- @field project_root string +--- @field upstream table +--- @field flags table +--- @field dry_run boolean +--- @field verbose boolean + +--- @class PassResult +--- @field outputs table[] +--- @field errors table[] +--- @field warnings table[] + +--- @alias AtomName string -- lower_snake_case atom name +--- @alias MacroName string -- lower_snake_case macro identifier +--- @alias CheckName string -- "gte_pipeline_fill" | "mac_yield_uniformity" | "abi_handoff" | "gpu_port_store_shape" | "per_atom_cycle_budget" + +--- @class AtomBody +--- @field line integer -- source line of the atom declaration +--- @field name AtomName -- atom name (e.g. "cube_g4_face") +--- @field body string -- the brace-delimited body (without the braces) +--- @field body_off integer -- char offset of body[1] in source +--- @field kind string -- "atom" | "comp_bare" | "comp_proc" + +--- @class Token +--- @field tok string -- the raw token text (trimmed) +--- @field line integer -- source line of the token's start +--- @field ident string|nil -- the leading ident of the token (if any) +--- @field kind string -- "n_words" | "mac_yield" | "gte_cmdw" | "mac_format" | "mac_gte_store" | "mac_insert_ot_tag" | "atom_label" | "atom_offset" | "other" + +--- @class Finding +--- @field line integer -- source line of the finding +--- @field atom AtomName -- the atom this finding is for (or "") +--- @field check CheckName -- the check identifier +--- @field kind string -- "error" | "warning" | "info" +--- @field msg string -- the finding message + +--- @class AtomAnalysis +--- @field atom AtomBody +--- @field tokens Token[] -- the tokens in the atom body, annotated +--- @field findings Finding[] -- findings for this atom +--- @field total_cycles integer -- sum of token cycle costs (Phase 3) -- ════════════════════════════════════════════════════════════════════════════ -- Source walkers @@ -94,18 +165,18 @@ local function find_atom_bodies(source_text) local len = #source_text local i = 1 while i <= len do - i = skip_ws_and_cmt(source_text, i); if i > len then break end + i = duffle.skip_ws_and_cmt(source_text, i); if i > len then break end -- Skip preprocessor directives (#define / #include / #pragma / -- etc). Otherwise the `#define MipsAtom_(sym) ...` definition -- in lottes_tape.h gets matched as an atom named "sym" and -- its `body` swallows the next real atom declaration via - -- scan_to_char("{", ...). + -- duffle.scan_to_char("{", ...). if source_text:sub(i, i) == "#" then local j = i while j <= len and source_text:byte(j) ~= 10 do j = j + 1 end i = j + 1 else - local ident, after = read_ident(source_text, i) + local ident, after = duffle.read_ident(source_text, i) if not ident then i = i + 1 elseif ident == "MipsAtom_" @@ -119,11 +190,11 @@ local function find_atom_bodies(source_text) else kind = "comp_proc" end - local open = skip_ws_and_cmt(source_text, after) + local open = duffle.skip_ws_and_cmt(source_text, after) if source_text:sub(open, open) ~= "(" then i = open + 1 else - local inner, after_paren = read_parens(source_text, open) + local inner, after_paren = duffle.read_parens(source_text, open) if kind == "comp_proc" then -- MipsAtomComp_Proc_(sym, { body }) @@ -152,11 +223,11 @@ local function find_atom_bodies(source_text) if depth == 0 then break end j = j + 1 elseif c == 40 then - local _, a = read_parens(inner, j); j = a + local _, a = duffle.read_parens(inner, j); j = a elseif c == 91 then - local _, a = read_brackets(inner, j); j = a + local _, a = duffle.read_brackets(inner, j); j = a elseif c == 34 or c == 39 then - j = duffle.skip_str_or_cmt(inner, j) + 1 + j = duffle.duffle.skip_str_or_cmt(inner, j) + 1 else j = j + 1 end @@ -197,9 +268,9 @@ local function find_atom_bodies(source_text) if name == "" then i = open + 1 else - local brace = scan_to_char(source_text, "{", after_paren) + local brace = duffle.scan_to_char(source_text, "{", after_paren) if brace then - local body, after_brace = read_braces(source_text, brace) + local body, after_brace = duffle.read_braces(source_text, brace) local body_off = brace + 1 out[#out + 1] = { line = line_of(i), @@ -262,11 +333,11 @@ end -- the trailing comma, and `nop` becomes its own token. So no special -- handling is needed here.) local function nop_word_count(token) - local s = trim(token) + local s = duffle.trim(token) -- strip trailing comma(s) (defensive against raw text via, but our -- tokenize_body already strips them; this is a safety net) s = s:gsub(",$", "") - s = trim(s) + s = duffle.trim(s) if s == "nop" then return 1 end if s == "nop2" then return 2 end return 0 @@ -283,7 +354,7 @@ local function tokenize_body(body) local rel = 1 while rel <= len do -- Find next non-whitespace, non-comment start - local ws_end = skip_ws_and_cmt(body, rel) + local ws_end = duffle.skip_ws_and_cmt(body, rel) if ws_end > rel then rel = ws_end end @@ -299,19 +370,19 @@ local function tokenize_body(body) if c == 10 then break end -- '\n' if c == 59 then break end -- ';' if c == 40 then -- '(' - local _, a = read_parens(body, i); i = a + local _, a = duffle.read_parens(body, i); i = a elseif c == 123 then -- '{' - local _, a = read_braces(body, i); i = a + local _, a = duffle.read_braces(body, i); i = a elseif c == 91 then -- '[' - local _, a = read_brackets(body, i); i = a + local _, a = duffle.read_brackets(body, i); i = a elseif c == 34 or c == 39 then -- '"' or '\'' - i = duffle.skip_str_or_cmt(body, i) + 1 + i = duffle.duffle.skip_str_or_cmt(body, i) + 1 else i = i + 1 end end -- Extract token [rel .. i-1] - local tok = trim(body:sub(rel, i - 1)) + local tok = duffle.trim(body:sub(rel, i - 1)) if tok ~= "" then out[#out + 1] = { tok = tok, rel = rel } end @@ -319,7 +390,7 @@ local function tokenize_body(body) if i <= len then i = i + 1 -- Also skip whitespace before next token - local w = skip_ws_and_cmt(body, i) + local w = duffle.skip_ws_and_cmt(body, i) if w > i then i = w end end rel = i @@ -333,7 +404,7 @@ end --- Walk the token list. Whenever we hit a `gte_cmdw_` token, count --- consecutive nop words starting at the next token. If count < the ---- minimum declared in `GTE_PIPELINE_LATENCY[X]`, record a finding. +--- minimum declared in `duffle.GTE_PIPELINE_LATENCY[X]`, record a finding. --- --- Aliases (`gte_cmdw_rotate_translate_perspective_single` etc.) are --- resolved against the lookup table directly; if a macro name is not @@ -344,7 +415,7 @@ local function check_gte_pipeline_fill(atoms, findings, line_of) -- count consecutive `nop` words IMMEDIATELY PRECEDING it (the -- source-level `nop2, gte_cmdw_X` idiom provides the pre-pipeline -- fill that gte.h's wrapper functions provide internally). If - -- count < `GTE_PIPELINE_LATENCY[X]`, record a finding. + -- count < `duffle.GTE_PIPELINE_LATENCY[X]`, record a finding. -- -- We count nops going backwards from the cmdw token, stopping at -- the first non-nop token. Tokens like `mem_share` or `port_write` @@ -363,7 +434,7 @@ local function check_gte_pipeline_fill(atoms, findings, line_of) or tok:match("^(gte_cmdw_[%w_]+)%s*$") if cmdw_full then local variant = cmdw_full:match("^gte_cmdw_(.+)$") - local need = GTE_PIPELINE_LATENCY[cmdw_full] + local need = duffle.GTE_PIPELINE_LATENCY[cmdw_full] if need == nil then -- alias or new gte_cmdw_ not yet in latency table local line = a.line + line_in_body[tokens[ti].rel] @@ -373,7 +444,7 @@ local function check_gte_pipeline_fill(atoms, findings, line_of) check = "gte_pipeline_fill", kind = "warning", msg = string.format( - "%s at line %d uses `gte_cmdw_%s` but that macro is not in duffle.GTE_PIPELINE_LATENCY -- add a min_nops entry", + "%s at line %d uses `gte_cmdw_%s` but that macro is not in duffle.duffle.GTE_PIPELINE_LATENCY -- add a min_nops entry", a.name, line, variant), } ti = ti + 1 @@ -539,42 +610,42 @@ local function find_binds_structs(source_text) local len = #source_text local i = 1 while i <= len do - i = skip_ws_and_cmt(source_text, i); if i > len then break end + i = duffle.skip_ws_and_cmt(source_text, i); if i > len then break end if source_text:sub(i, i) == "#" then local j = i while j <= len and source_text:byte(j) ~= 10 do j = j + 1 end i = j + 1 else - local ident, after = read_ident(source_text, i) + local ident, after = duffle.read_ident(source_text, i) if not ident then i = i + 1 elseif ident == "typedef" then - local j = skip_ws_and_cmt(source_text, after) - local id2, after2 = read_ident(source_text, j) + local j = duffle.skip_ws_and_cmt(source_text, after) + local id2, after2 = duffle.read_ident(source_text, j) if id2 ~= "Struct_" then i = after2 or (j + 1) else - local open = skip_ws_and_cmt(source_text, after2) + local open = duffle.skip_ws_and_cmt(source_text, after2) if source_text:sub(open, open) ~= "(" then i = open + 1 else - local inner, after_paren = read_parens(source_text, open) - local name = trim(inner) - local brace = scan_to_char(source_text, "{", after_paren) + local inner, after_paren = duffle.read_parens(source_text, open) + local name = duffle.trim(inner) + local brace = duffle.scan_to_char(source_text, "{", after_paren) if not brace then i = open + 1 else - local body, after_brace = read_braces(source_text, brace) + local body, after_brace = duffle.read_braces(source_text, brace) local fields = {} local byte_off = 0 local k = 1 while k <= #body do - k = skip_ws_and_cmt(body, k); if k > #body then break end - local tid, tafter = read_ident(body, k) + k = duffle.skip_ws_and_cmt(body, k); if k > #body then break end + local tid, tafter = duffle.read_ident(body, k) if not tid then k = k + 1 elseif tid == "U4" then - local fid, fafter = read_ident(body, skip_ws_and_cmt(body, tafter)) + local fid, fafter = duffle.read_ident(body, duffle.skip_ws_and_cmt(body, tafter)) if fid then fields[#fields + 1] = { name = fid, offset = byte_off } byte_off = byte_off + 4 @@ -610,60 +681,60 @@ local function find_atom_info(source_text) local len = #source_text local i = 1 while i <= len do - i = skip_ws_and_cmt(source_text, i); if i > len then break end + i = duffle.skip_ws_and_cmt(source_text, i); if i > len then break end if source_text:sub(i, i) == "#" then local j = i while j <= len and source_text:byte(j) ~= 10 do j = j + 1 end i = j + 1 else - local ident, after = read_ident(source_text, i) + local ident, after = duffle.read_ident(source_text, i) if not ident then i = i + 1 elseif ident == "MipsAtom_" then - local open = skip_ws_and_cmt(source_text, after) + local open = duffle.skip_ws_and_cmt(source_text, after) if source_text:sub(open, open) ~= "(" then i = open + 1 else - local inner, after_paren = read_parens(source_text, open) + local inner, after_paren = duffle.read_parens(source_text, open) local a = 1 while a <= #inner and inner:sub(a, a):match("[%s]") do a = a + 1 end local b = a while b <= #inner and inner:sub(b, b):match("[%w_]") do b = b + 1 end local atom_name = inner:sub(a, b - 1) - local lookahead = skip_ws_and_cmt(source_text, after_paren) - local look_ident, look_after = read_ident(source_text, lookahead) + local lookahead = duffle.skip_ws_and_cmt(source_text, after_paren) + local look_ident, look_after = duffle.read_ident(source_text, lookahead) if look_ident == "atom_info" then - local info_open = skip_ws_and_cmt(source_text, look_after) + local info_open = duffle.skip_ws_and_cmt(source_text, look_after) if source_text:sub(info_open, info_open) == "(" then - local info_inner, info_after = read_parens(source_text, info_open) + local info_inner, info_after = duffle.read_parens(source_text, info_open) local binds, reads, writes = nil, nil, nil local j = 1 while j <= #info_inner do - j = skip_ws_and_cmt(info_inner, j); if j > #info_inner then break end - local sub_ident, sub_after = read_ident(info_inner, j) + j = duffle.skip_ws_and_cmt(info_inner, j); if j > #info_inner then break end + local sub_ident, sub_after = duffle.read_ident(info_inner, j) if not sub_ident then j = j + 1 elseif sub_ident == "atom_bind" then - local sub_open = skip_ws_and_cmt(info_inner, sub_after) + local sub_open = duffle.skip_ws_and_cmt(info_inner, sub_after) if info_inner:sub(sub_open, sub_open) == "(" then - local sub_inner, sub_after2 = read_parens(info_inner, sub_open) - binds = trim(sub_inner) + local sub_inner, sub_after2 = duffle.read_parens(info_inner, sub_open) + binds = duffle.trim(sub_inner) j = sub_after2 else j = sub_open + 1 end elseif sub_ident == "atom_reads" or sub_ident == "atom_writes" then local kind = sub_ident - local sub_open = skip_ws_and_cmt(info_inner, sub_after) + local sub_open = duffle.skip_ws_and_cmt(info_inner, sub_after) if info_inner:sub(sub_open, sub_open) == "(" then - local sub_inner, sub_after2 = read_parens(info_inner, sub_open) + local sub_inner, sub_after2 = duffle.read_parens(info_inner, sub_open) local regs = {} local p = 1 while p <= #sub_inner do - p = skip_ws_and_cmt(sub_inner, p); if p > #sub_inner then break end - local pid, pa = read_ident(sub_inner, p) + p = duffle.skip_ws_and_cmt(sub_inner, p); if p > #sub_inner then break end + local pid, pa = duffle.read_ident(sub_inner, p) if pid then - regs[#regs + 1] = trim(pid) + regs[#regs + 1] = duffle.trim(pid) p = pa else p = p + 1 @@ -816,14 +887,14 @@ end --- For every baked atom body, detect which GP0 primitive it's emitting --- (first `mac_format__color` call). Sum contributions from --- `mac_format_X_color` + `mac_gte_store_X_post_*` + `mac_insert_ot_tag_X`. ---- Compare to GP0_CMD_SIZE[cmd_byte]. Mismatch = error. +--- Compare to duffle.GP0_CMD_SIZE[cmd_byte]. Mismatch = error. --- --- Soft behavior (warnings): --- - Atoms emitting a primitive via raw `store_word(R_PrimCursor, ...)` --- (no `mac_format_X_color` call) emit a "manual packet assembly" --- advisory. Cannot auto-validate. --- - Atoms containing a `mac_(...)` call whose name is not in ---- GP0_MACRO_CONTRIB emit a "new macro; update GP0_MACRO_CONTRIB" +--- duffle.GP0_MACRO_CONTRIB emit a "new macro; update duffle.GP0_MACRO_CONTRIB" --- advisory. --- --- Applies only to `kind = "atom"` (baked atoms). Components don't @@ -844,24 +915,24 @@ local function check_gpu_portstore_shape(atoms, findings) -- to get the bare shape suffix (f3 / g4 / etc). local shape = tok:match("^mac_format_([%w_]+)_color%s*%(") or tok:match("^mac_format_([%w_]+)_color%s*$") - if shape and GP0_CMD_BY_SHAPE[shape] then + if shape and duffle.GP0_CMD_BY_SHAPE[shape] then if not cmd_byte then - cmd_byte = GP0_CMD_BY_SHAPE[shape] + cmd_byte = duffle.GP0_CMD_BY_SHAPE[shape] cmd_line = a.line + line_in_body[t.rel] end saw_format = true local contrib_key = "mac_format_" .. shape .. "_color" - local n = GP0_MACRO_CONTRIB[contrib_key] + local n = duffle.GP0_MACRO_CONTRIB[contrib_key] if n then contrib = contrib + n end end local gte_store = tok:match("^mac_gte_store_[%w_]+") if gte_store then - local n = GP0_MACRO_CONTRIB[gte_store] + local n = duffle.GP0_MACRO_CONTRIB[gte_store] if n then contrib = contrib + n end end local ot_tag = tok:match("^mac_insert_ot_tag_([%w_]+)") if ot_tag then - local n = GP0_MACRO_CONTRIB["mac_insert_ot_tag_" .. ot_tag] + local n = duffle.GP0_MACRO_CONTRIB["mac_insert_ot_tag_" .. ot_tag] if n then contrib = contrib + n end end if tok:match("^store_word%s*%(") and tok:find("R_PrimCursor", 1, true) then @@ -879,7 +950,7 @@ local function check_gpu_portstore_shape(atoms, findings) } end else - local expected = GP0_CMD_SIZE[cmd_byte] + local expected = duffle.GP0_CMD_SIZE[cmd_byte] if contrib ~= expected then findings[#findings + 1] = { atom = a.name, line = cmd_line or a.line, @@ -899,20 +970,20 @@ end --- Compute the cycle cost of one token. The token is a string like --- `add_ui(R_T0, R_T1, 4)` or `nop2` or `gte_cmdw_rtpt`. Returns: ---- cycles - integer cycle cost (from INSTRUCTION_LATENCY, or ---- UNKNOWN_INSTRUCTION_CYCLES if not in the table) +--- cycles - integer cycle cost (from duffle.INSTRUCTION_LATENCY, or +--- duffle.UNKNOWN_INSTRUCTION_CYCLES if not in the table) --- macro_name - the bare ident (e.g. `add_ui`, `gte_cmdw_rtpt`, --- `nop2`, `mac_yield`) ---- unknown - true iff the macro wasn't in INSTRUCTION_LATENCY +--- unknown - true iff the macro wasn't in duffle.INSTRUCTION_LATENCY --- The function strips trailing `()` from function-call style macros --- so `mac_yield()` and `mac_yield` resolve identically. local function token_cycles(tok) -- Extract the leading ident. Tolerate `(...)` args. local ident = tok:match("^([%w_]+)") - if not ident then return UNKNOWN_INSTRUCTION_CYCLES, "?", true end - local cost = INSTRUCTION_LATENCY[ident] + if not ident then return duffle.UNKNOWN_INSTRUCTION_CYCLES, "?", true end + local cost = duffle.INSTRUCTION_LATENCY[ident] if cost == nil then - return UNKNOWN_INSTRUCTION_CYCLES, ident, true + return duffle.UNKNOWN_INSTRUCTION_CYCLES, ident, true end return cost, ident, false end @@ -968,7 +1039,7 @@ end --- mac_yield or end-of-body) --- has_loops - true iff a path re-entered a token it had visited --- (warning; loop bodies aren't supported) ---- unknown_macros - list of unique macro names not in INSTRUCTION_LATENCY +--- unknown_macros - list of unique macro names not in duffle.INSTRUCTION_LATENCY --- cycles_full - sum of ALL token costs (the previous "best case" --- value; included for backward-compat; double-counts --- the BD-slot nop relative to cycles_min/max) @@ -1155,7 +1226,7 @@ end --- Per-source check that emits one finding per unknown macro seen --- (deduplicated across atoms so the warning section doesn't get ---- spammed with N copies of "macro X not in INSTRUCTION_LATENCY"). +--- spammed with N copies of "macro X not in duffle.INSTRUCTION_LATENCY"). local function check_per_atom_cycle_budget(atoms, findings) local unknown_seen = {} for _, a in ipairs(atoms) do @@ -1166,8 +1237,8 @@ local function check_per_atom_cycle_budget(atoms, findings) findings[#findings + 1] = { atom = a.name, line = a.line, check = "per_atom_cycle_budget", kind = "warning", - msg = string.format("%s at line %d uses macro `%s` which is not in INSTRUCTION_LATENCY; cycle count will be +%d per call (best-case). Add an entry to duffle.M.INSTRUCTION_LATENCY.", - a.name, a.line, name, UNKNOWN_INSTRUCTION_CYCLES), + msg = string.format("%s at line %d uses macro `%s` which is not in duffle.INSTRUCTION_LATENCY; cycle count will be +%d per call (best-case). Add an entry to duffle.M.duffle.INSTRUCTION_LATENCY.", + a.name, a.line, name, duffle.UNKNOWN_INSTRUCTION_CYCLES), } end end @@ -1284,7 +1355,7 @@ end local function emit_static_analysis_txt(ctx, src, result) local out_path = ctx.out_root .. "/" .. src.basename .. ".static_analysis.txt" if ctx.dry_run then return out_path end - ensure_dir(ctx.out_root) + duffle.ensure_dir(ctx.out_root) local lines = {} local function add(s) lines[#lines + 1] = s end @@ -1352,7 +1423,7 @@ local function emit_static_analysis_txt(ctx, src, result) add(string.format(" %s", i_.msg)) end - write_file(out_path, table.concat(lines, "\n") .. "\n") + duffle.write_file(out_path, table.concat(lines, "\n") .. "\n") return out_path end @@ -1370,7 +1441,7 @@ local function emit_module_static_analysis_txt(ctx, dir, dir_sources, atoms, fin local dir_basename = dir:match("([^/\\]+)$") or dir local out_path = ctx.out_root .. "/" .. dir_basename .. ".static_analysis.txt" if ctx.dry_run then return out_path end - ensure_dir(ctx.out_root) + duffle.ensure_dir(ctx.out_root) local lines = {} local function add(s) lines[#lines + 1] = s end @@ -1545,7 +1616,7 @@ local function emit_module_static_analysis_txt(ctx, dir, dir_sources, atoms, fin end end - write_file(out_path, table.concat(lines, "\n") .. "\n") + duffle.write_file(out_path, table.concat(lines, "\n") .. "\n") return out_path end diff --git a/scripts/word_count_eval.lua b/scripts/passes/word_count_eval.lua similarity index 96% rename from scripts/word_count_eval.lua rename to scripts/passes/word_count_eval.lua index 975f04e..09b8119 100644 --- a/scripts/word_count_eval.lua +++ b/scripts/passes/word_count_eval.lua @@ -23,16 +23,8 @@ -- `require("duffle")` resolves against `scripts/` regardless of CWD. -- Note: this boilerplate is duplicated in 6 other entry scripts; a -- Phase-6 extraction target (`duffle.setup_package_path()`). -local script_path = arg and arg[0] or "?" -local last_sep = 0 -for sep_pos = 1, #script_path do - local ch = script_path:sub(sep_pos, sep_pos) - if ch == "/" or ch == "\\" then last_sep = sep_pos end -end -local script_dir = last_sep == 0 and "./" or script_path:sub(1, last_sep) -package.path = script_dir .. "?.lua;" .. script_dir .. "?/init.lua;" .. package.path -package.cpath = "C:\\projects\\Pikuma\\ps1\\toolchain\\luajit-2.1\\lib\\lua\\5.1\\?.dll;" .. package.cpath - +-- Bootstrap: see `ps1_meta.lua` for the rationale. +dofile((arg[0]:match("(.*[/\\])") or "./") .. "duffle_paths.lua") local duffle = require("duffle") -- ════════════════════════════════════════════════════════════════════════════ diff --git a/scripts/ps1_meta.lua b/scripts/ps1_meta.lua index 3e9ef10..5514359 100644 --- a/scripts/ps1_meta.lua +++ b/scripts/ps1_meta.lua @@ -1,76 +1,105 @@ --- ps1_meta.lua --- --- Orchestrator entry point for the tape-atom metaprogram pipeline. --- Dispatches to pass modules under scripts/passes/, resolving dependencies --- topologically. Single CLI surface (`--` flags + auto-dep + --dry-run). --- --- Coding standard: tabs (1/level), EmmyLua annotations, no regex, --- Lua 5.3 compatible. - +--- ps1_meta.lua — Orchestrator entry point for the tape-atom metaprogram pipeline. +--- +--- Dispatches to pass modules under `scripts/passes/`, resolving +--- dependencies topologically (Kahn's algorithm + cycle detection). +--- Single CLI surface (`--` flags + auto-dep expansion + --dry-run). +--- +--- **Architecture**: +--- - **PASSES table** — declarative dep graph (data, not code). +--- - **FLAG_HANDLERS table** — per-flag CLI dispatchers (handler-map +--- pattern; replaces an 8-way if/elseif chain). +--- - **parse_args** → **build_ctx** → **topo_sort** → **dispatch_passes**. +--- +--- **Conventions**: tabs (1/level), EmmyLua annotations, no regex, +--- Lua 5.3 compatible. +--- -- ════════════════════════════════════════════════════════════════════════════ -- Module-scope requires + package.path setup -- ════════════════════════════════════════════════════════════════════════════ -local script_path = arg and arg[0] or "?" -local last_sep = 0 -for i = 1, #script_path do - local c = script_path:sub(i, i) - if c == "/" or c == "\\" then last_sep = i end -end -local script_dir = last_sep == 0 and "./" or script_path:sub(1, last_sep) -package.path = script_dir .. "?.lua;" .. script_dir .. "?/init.lua;" .. package.path -package.cpath = "C:\\projects\\Pikuma\\ps1\\toolchain\\luajit-2.1\\lib\\lua\\5.1\\?.dll;" .. package.cpath - +-- Resolve `arg[0]` to an absolute-ish script directory so that `require("duffle")` resolves against `scripts/` regardless of CWD. +-- Note: this boilerplate is duplicated in 6 other entry scripts; a +-- Phase-6 extraction target (`duffle.setup_package_path()`). +-- Bootstrap: load `duffle_paths.lua` (uses `git rev-parse` to find the repo root, then sets package.path + package.cpath). +-- After this line, `require("duffle")` and `require("passes.X")` both resolve. +dofile((arg[0]:match("(.*[/\\])") or "./") .. "duffle_paths.lua") local duffle = require("duffle") -local dirname = duffle.dirname -local basename_no_ext = duffle.basename_no_ext + +-- ════════════════════════════════════════════════════════════════════════════ +-- Constants +-- ════════════════════════════════════════════════════════════════════════════ + +-- Exit codes (per the --help text and the post-build summary convention). +local EXIT_OK = 0 +local EXIT_VALIDATION_ERRORS = 1 +local EXIT_INTERNAL_ERROR = 2 + +-- Default --out-root value if not provided. +local DEFAULT_OUT_ROOT = "build/gen" + +-- Sentinel for "all passes" in `PASS_FLAG_TO_NAME`. Distinguishes `--all` from the per-pass flags (which map to individual pass names). +local ALL_PASSES_SENTINEL = "__all__" + +-- Sentinel key for the pass-flag dispatcher in `FLAG_HANDLERS`. +-- The actual pass names are looked up via `PASS_FLAG_TO_NAME`, not direct dispatch, so this key never matches a real flag. +local PASS_FLAG_DISPATCH_KEY = "__pass__" -- ════════════════════════════════════════════════════════════════════════════ -- Type declarations -- ════════════════════════════════════════════════════════════════════════════ --- @class PassDescriptor ---- @field module string -- module name passed to require() ---- @field kind string -- "shared" | "header-output" | "validation" | "report" ---- @field deps string[] -- names of upstream passes ---- @field desc string -- human description (used by --help + ASCII graph) ---- @field out PassOutput[] -- output paths (used by --dry-run + report) +--- @field module string -- module name passed to require() +--- @field kind string -- "shared" | "header-output" | "validation" | "report" +--- @field deps string[] -- names of upstream passes +--- @field desc string -- human description (used by --help + ASCII graph) +--- @field out PassOutput[] -- output paths (used by --dry-run + report) --- @class PassOutput --- @field kind string -- "header" | "report" --- @field path_template string -- e.g. "/gen/.macs.h" --- @class SourceFile ---- @field path string ---- @field text string ---- @field dir string ---- @field basename string +--- @field path string -- absolute path to the source file +--- @field text string -- the full source text +--- @field dir string -- the directory containing the source +--- @field basename string -- filename without extension --- @class PassCtx ---- @field sources SourceFile[] ---- @field metadata_path string ---- @field shared table ---- @field shared.word_counts table ---- @field out_root string ---- @field project_root string ---- @field upstream table ---- @field flags table ---- @field dry_run boolean ---- @field verbose boolean +--- @field sources SourceFile[] -- all source files in the build +--- @field metadata_path string -- path to word_count.metadata.h +--- @field shared table -- cross-pass shared state +--- @field shared.word_counts table -- populated by word-counts pass +--- @field out_root string -- output root (e.g. "build/gen") +--- @field project_root string -- project root (e.g. "code/") +--- @field upstream table -- per-pass output accumulator +--- @field flags table -- CLI flags + per-pass stash +--- @field dry_run boolean -- if true, compute but don't write +--- @field verbose boolean -- if true, log diagnostic info + +--- @class PassOutputEntry +--- @field [string] string -- dynamic shape; key is the output kind + -- (e.g. "macs_h", "offsets_h", "errors_h", + -- "annotations_txt", "static_analysis_txt", + -- "summary_txt"), value is the path + +--- @class Finding +--- @field line integer -- source line (or 0 for pass-level) +--- @field msg string -- finding message --- @class PassResult ---- @field outputs table[] ---- @field errors table[] ---- @field warnings table[] +--- @field outputs PassOutputEntry[] -- emitted file paths +--- @field errors Finding[] -- build-stops (per-pass kind policy) +--- @field warnings Finding[] -- informational --- @class ParsedArgs ---- @field requested_set string[] -- pass names to run (explicit --all expanded) ---- @field sources string[] -- --source values ---- @field metadata string -- --metadata value ---- @field out_root string -- --out-root value (default "build/gen") ---- @field project_root string -- --project-root value (default dirname(metadata)) ---- @field dry_run boolean ---- @field verbose boolean +--- @field requested_set string[] -- pass names to run (explicit --all expanded) +--- @field sources string[] -- --source values +--- @field metadata string -- --metadata value +--- @field out_root string -- --out-root value (default "build/gen") +--- @field project_root string -- --project-root value (default dirname(metadata)) +--- @field dry_run boolean -- if true, compute but don't write +--- @field verbose boolean -- if true, log diagnostic info -- ════════════════════════════════════════════════════════════════════════════ -- PASSES table (data, not code) — the orchestrator's dep graph @@ -78,7 +107,7 @@ local basename_no_ext = duffle.basename_no_ext local PASSES = { ["word-counts"] = { - module = "word_count_eval", + module = "passes.word_count_eval", kind = "shared", deps = {}, desc = "Build the shared metadata table (metadata.h + .macs.h)", @@ -140,7 +169,7 @@ local PASS_FLAG_TO_NAME = { ["--offsets"] = "offsets", ["--static-analysis"] = "static-analysis", ["--report"] = "report", - ["--all"] = "__all__", + ["--all"] = ALL_PASSES_SENTINEL, } local ALL_PASS_NAMES = { @@ -150,6 +179,7 @@ local ALL_PASS_NAMES = { --- Append every pass name to args.requested_set. Used by --all and --- by the "default to --all if no pass flags were given" fallback. +--- @param args ParsedArgs local function request_all_passes(args) for _, n in ipairs(ALL_PASS_NAMES) do args.requested_set[#args.requested_set + 1] = n @@ -247,9 +277,9 @@ end -- Pass-flag handler. Reads the closed-set table, expands --all, -- appends to requested_set. Single-statement, no nesting. -FLAG_HANDLERS["__pass__"] = function(args, a) +FLAG_HANDLERS[PASS_FLAG_DISPATCH_KEY] = function(args, a) local name = PASS_FLAG_TO_NAME[a] - if name == "__all__" then + if name == ALL_PASSES_SENTINEL then request_all_passes(args) return end @@ -265,26 +295,26 @@ local function parse_args(argv) requested_set = {}, sources = {}, metadata = nil, - out_root = "build/gen", + out_root = DEFAULT_OUT_ROOT, project_root = nil, dry_run = false, verbose = false, } - local i = 1 - while i <= #argv do - local a = argv[i] + local pos = 1 + while pos <= #argv do + local a = argv[pos] local handler = FLAG_HANDLERS[a] if handler then - i = handler(args, argv, i) or i + pos = handler(args, argv, pos) or pos elseif PASS_FLAG_TO_NAME[a] then - FLAG_HANDLERS["__pass__"](args, a) + FLAG_HANDLERS[PASS_FLAG_DISPATCH_KEY](args, a) else io.stderr:write("ps1_meta: unknown flag '" .. a .. "'\n") io.stderr:write("Run with --help for usage.\n") - os.exit(2) + os.exit(EXIT_INTERNAL_ERROR) end - i = i + 1 + pos = pos + 1 end -- Default: --all if no explicit pass flags. @@ -294,20 +324,20 @@ local function parse_args(argv) -- Defaults: project_root = dirname(metadata). if args.metadata and not args.project_root then - local d = dirname(args.metadata) + local d = duffle.dirname(args.metadata) if #d > 0 and (d:sub(-1) == "/" or d:sub(-1) == "\\") then d = d:sub(1, -2) end - args.project_root = dirname(d) + args.project_root = duffle.dirname(d) end if not args.metadata then io.stderr:write("ps1_meta: --metadata PATH is required\n") - os.exit(2) + os.exit(EXIT_INTERNAL_ERROR) end if #args.sources == 0 then io.stderr:write("ps1_meta: at least one --source FILE is required\n") - os.exit(2) + os.exit(EXIT_INTERNAL_ERROR) end return args @@ -328,13 +358,13 @@ local function build_ctx(args) local f = io.open(path, "r") if not f then io.stderr:write("ps1_meta: cannot open --source " .. path .. "\n") - os.exit(2) + os.exit(EXIT_INTERNAL_ERROR) end local text = f:read("*a") f:close() - local dir = dirname(path) - local basename = basename_no_ext(path) + local dir = duffle.dirname(path) + local basename = duffle.basename_no_ext(path) if #dir > 0 and (dir:sub(-1) == "/" or dir:sub(-1) == "\\") then dir = dir:sub(1, -2) end @@ -364,14 +394,13 @@ end -- Topological sort (Kahn's algorithm + cycle detection) -- ════════════════════════════════════════════════════════════════════════════ ---- Topologically sort the requested pass set, augmented with all transitive deps. ---- Detects cycles and errors out with details. +--- Compute the dep-closure of `requested_set`: include every pass name +--- transitively required by the requested set. --- --- @param passes table --- @param requested_set string[] ---- @return string[] -- execution order -local function topo_sort(passes, requested_set) - -- Step 1: dep-closure. +--- @return table -- set of pass names needed (including transitive deps) +local function dep_closure(passes, requested_set) local needed = {} for _, name in ipairs(requested_set) do needed[name] = true end local changed = true @@ -390,55 +419,99 @@ local function topo_sort(passes, requested_set) end end end + return needed +end - -- Step 2: Kahn's algorithm. +--- Count entries in a hash table (Lua's `#t` doesn't work for hash tables). +--- @param t table +--- @return integer +local function count_entries(t) + local n = 0 + for _ in pairs(t) do n = n + 1 end + return n +end + +--- Compute in-degrees for the Kahn sort: for each pass in `needed`, +--- the number of its deps that are also in `needed`. +--- +--- @param passes table +--- @param needed table +--- @return table +local function compute_in_degrees(passes, needed) local in_degree = {} for name, _ in pairs(needed) do in_degree[name] = 0 end for name, _ in pairs(needed) do for _, dep in ipairs(passes[name].deps) do if needed[dep] then - in_degree[name] = (in_degree[name] or 0) + 1 + in_degree[name] = in_degree[name] + 1 end end end + return in_degree +end - -- Seed with passes that have no unmet deps. Sort for determinism. +--- Seed the Kahn ready queue with passes whose in-degree is 0, sorted +--- alphabetically for deterministic execution order. +--- +--- @param in_degree table +--- @return string[] +local function seed_ready_queue(in_degree) local ready = {} for name, deg in pairs(in_degree) do if deg == 0 then ready[#ready + 1] = name end end table.sort(ready) + return ready +end - local order = {} - while #ready > 0 do - local n = table.remove(ready, 1) - order[#order + 1] = n - for name, _ in pairs(needed) do - if name ~= n then - for _, dep in ipairs(passes[name].deps) do - if dep == n then - in_degree[name] = in_degree[name] - 1 - if in_degree[name] == 0 then - ready[#ready + 1] = name - table.sort(ready) - end +-- (internal) Pop the next ready pass, decrement the in-degree of every +-- remaining pass that depended on it (inserting newly-zero-degree passes +-- back into the ready queue), and append to `order`. Keeps `ready` sorted. +-- @param passes table +-- @param needed table +-- @param in_degree table +-- @param ready string[] +-- @param order string[] +local function process_next_ready(passes, needed, in_degree, ready, order) + local just_finished = table.remove(ready, 1) + order[#order + 1] = just_finished + for name, _ in pairs(needed) do + if name ~= just_finished then + for _, dep in ipairs(passes[name].deps) do + if dep == just_finished then + in_degree[name] = in_degree[name] - 1 + if in_degree[name] == 0 then + ready[#ready + 1] = name + table.sort(ready) end end end end end +end + +--- Topologically sort the requested pass set, augmented with all transitive deps. +--- Detects cycles and errors out with details. +--- +--- @param passes table +--- @param requested_set string[] +--- @return string[] -- execution order +local function topo_sort(passes, requested_set) + local needed = dep_closure(passes, requested_set) + local in_degree = compute_in_degrees(passes, needed) + local ready = seed_ready_queue(in_degree) + + local order = {} + while #ready > 0 do + process_next_ready(passes, needed, in_degree, ready, order) + end -- Cycle detection: if order doesn't include all needed passes, -- some are stuck with in_degree > 0 (the cycle closed on itself -- before Kahn could process them). Without this check, a fully- -- closed cycle (e.g. A -> B -> A) would silently return an empty -- order list, leaving the orchestrator to dispatch nothing. - -- - -- Note: `#needed` returns 0 for hash tables (needed is a set, - -- not a sequence), so we count entries explicitly. - local needed_count = 0 - for _ in pairs(needed) do needed_count = needed_count + 1 end - if #order ~= needed_count then + if #order ~= count_entries(needed) then for name, deg in pairs(in_degree) do if deg > 0 then error("dependency cycle detected involving pass '" .. name .. "'") @@ -518,6 +591,68 @@ end -- Main orchestrator -- ════════════════════════════════════════════════════════════════════════════ +-- (internal) Push a pass's outputs + warnings into `ctx.upstream[name]` +-- for downstream passes to consume. +-- @param ctx PassCtx +-- @param pass_name string +-- @param result PassResult +local function accumulate_pass_result(ctx, pass_name, result) + ctx.upstream[pass_name] = ctx.upstream[pass_name] or {} + for _, out in ipairs(result.outputs or {}) do + table.insert(ctx.upstream[pass_name], out) + end + for _, warn in ipairs(result.warnings or {}) do + table.insert(ctx.upstream[pass_name], warn) + end +end + +-- (internal) If the pass's kind is in PASS_KIND_STOP_ON_ERROR and it +-- reported errors, write each error to stderr. Returns true if any +-- validation errors were reported. +-- @param pass_name string +-- @param pass PassDescriptor +-- @param result PassResult +-- @return boolean +local function report_validation_errors(pass_name, pass, result) + local has_errors = result.errors and #result.errors > 0 + if not (has_errors and PASS_KIND_STOP_ON_ERROR[pass.kind]) then + return false + end + for _, e in ipairs(result.errors) do + io.stderr:write(string.format("[%s] line %d: %s\n", + pass_name, e.line or 0, e.msg or "")) + end + return true +end + +-- (internal) Run each pass in `order` in topological sequence. Tracks +-- `had_errors` instead of os.exit()ing mid-loop so the report pass (and +-- any other downstream pass) still runs and writes its per-module files. +-- The legacy behavior was os.exit(1) on the first error, which left +-- downstream per-module reports un-emitted; the 2026-07-10 change to +-- per-module aggregation made that visible to the user (build/gen had +-- only the partial reports from the failing pass), so we now complete +-- all passes and set the exit code at the end. +--- +-- @param ctx PassCtx +-- @param order string[] +-- @return boolean -- true if any validation errors were reported +local function dispatch_passes(ctx, order) + ctx.shared = {} + local had_errors = false + for _, pass_name in ipairs(order) do + local pass = PASSES[pass_name] + local mod = require(pass.module) + local result = mod.run(ctx) + + accumulate_pass_result(ctx, pass_name, result) + if report_validation_errors(pass_name, pass, result) then + had_errors = true + end + end + return had_errors +end + --- Main entry point. Runs the requested passes in dep-topological order. --- @param argv string[] local function main(argv) @@ -525,63 +660,25 @@ local function main(argv) local args = parse_args(argv) local ctx = build_ctx(args) - -- 1. Compute requested set + dep-closed set. local requested = args.requested_set local closed = topo_sort(PASSES, requested) - -- 2. --dry-run: print dep order + ASCII graph, exit 0. + -- --dry-run: print dep order + ASCII graph, exit OK. if args.dry_run then io.write(render_dep_graph(PASSES, requested, closed)) - os.exit(0) + os.exit(EXIT_OK) end - -- 3. Run passes in topological order. We track a `had_errors` - -- flag instead of os.exit()'ing mid-loop, so the report pass - -- (and any other downstream pass) still runs and writes its - -- per-module reports. The exit code at the end is set to 1 - -- if any pass reported errors. - ctx.shared = {} - local had_errors = false - for _, pass_name in ipairs(closed) do - local pass = PASSES[pass_name] - local mod = require(pass.module) - local result = mod.run(ctx) - - -- Collect outputs + warnings into ctx.upstream. - ctx.upstream[pass_name] = ctx.upstream[pass_name] or {} - for _, out in ipairs(result.outputs or {}) do - table.insert(ctx.upstream[pass_name], out) - end - for _, warn in ipairs(result.warnings or {}) do - table.insert(ctx.upstream[pass_name], warn) - end - - -- Record errors for the exit code, but DON'T os.exit() here: - -- the report pass (kind="report") and any other downstream - -- pass still needs to run to emit its per-module files. - -- The legacy behavior was os.exit(1) on the first error, - -- which left downstream per-module reports un-emitted; the - -- 2026-07-10 change to per-module aggregation made that - -- visible to the user (build/gen had only the partial - -- reports from the failing pass), so we now complete - -- all passes and set the exit code at the end. - if (result.errors and #result.errors > 0) and PASS_KIND_STOP_ON_ERROR[pass.kind] then - for _, e in ipairs(result.errors) do - io.stderr:write(string.format("[%s] line %d: %s\n", - pass_name, e.line or 0, e.msg or "")) - end - had_errors = true - end - end - if had_errors then os.exit(1) end + local had_errors = dispatch_passes(ctx, closed) + if had_errors then os.exit(EXIT_VALIDATION_ERRORS) end end) if not ok then io.stderr:write("[ps1_meta] internal error: " .. tostring(err) .. "\n") - os.exit(2) + os.exit(EXIT_INTERNAL_ERROR) end - os.exit(0) + os.exit(EXIT_OK) end main({...})