From 072231c46b4fbfbb9fd210f2d5fc6495b1c6b21d Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sat, 11 Jul 2026 13:45:22 -0400 Subject: [PATCH] Lua Metaprogram: Scan codepaths collapse + more reviews. --- scripts/duffle.lua | 37 +- scripts/passes/annotation.lua | 711 +++-------------------------- scripts/passes/components.lua | 201 ++------ scripts/passes/offsets.lua | 226 ++------- scripts/passes/report.lua | 15 +- scripts/passes/scan_source.lua | 474 +++++++++++++++++++ scripts/passes/static_analysis.lua | 538 +++++++--------------- scripts/ps1_meta.lua | 112 +++-- scripts/update_deps.ps1 | 13 - 9 files changed, 870 insertions(+), 1457 deletions(-) create mode 100644 scripts/passes/scan_source.lua diff --git a/scripts/duffle.lua b/scripts/duffle.lua index 5499385..7e0e4e5 100644 --- a/scripts/duffle.lua +++ b/scripts/duffle.lua @@ -295,6 +295,36 @@ function M.write_file(path, content) f:write(content); f:close() end +-- Write content to disk in binary mode so LF line endings are preserved on Windows +-- (text mode would convert LF -> CRLF, breaking byte-identical diffs against git-tracked gen/*.h files which are stored as LF). +-- @param path string +-- @param content string +function M.write_file_lf(path, content) + local f = io.open(path, "wb") + if not f then error("Cannot write " .. path) end + f:write(content); f:close() +end + +-- Convert a (possibly relative) path to an absolute path, using CWD if needed. +-- Normalizes forward slashes to backslashes on Windows. +-- Used for byte-identical emit: the // Source: comment line uses the absolute path. +-- @param path string +-- @return string +function M.to_absolute_path(path) + if #path >= 2 and path:sub(2, 2) == ":" then + -- Already absolute; normalize slashes for consistency. + return (path:gsub("/", "\\")) + end + local p = io.popen("cd") + if not p then return path end + local cwd = p:read("*l") + p:close() + if not cwd then return path end + cwd = cwd:gsub("/", "\\") + local tail = (path:gsub("/", "\\")) + return cwd .. "\\" .. tail +end + -- Cache of directories already verified to exist in this process. -- Each ensure_dir() call may otherwise spawn a `cmd.exe mkdir` (50-100ms per call on Windows) — calling it inside per-source loops added 1.5+ -- seconds to the report pass. Cache makes ensure_dir idempotent within the process lifetime. @@ -534,9 +564,11 @@ function M.load_word_counts(metadata_path) return counts end + -- ════════════════════════════════════════════════════════════════════════════ +-- ══════════════════════════════════════════════════ -- Section 6: LineIndex (perf fix — replaces the per-call rescan line_of) --- ════════════════════════════════════════════════════════════════════════════ +-- ══════════════════════════════════════════════════ function M.LineIndex(source) local positions = {} @@ -547,7 +579,7 @@ function M.LineIndex(source) positions[n] = pos end end - -- (internal) Binary-search for the line number containing `query_pos`. + -- (internal) Binary-search for the line number containing query_pos. local function line_of(query_pos) local lo, hi = 1, n while lo <= hi do @@ -560,7 +592,6 @@ function M.LineIndex(source) return line_of end --- ════════════════════════════════════════════════════════════════════════════ -- Section 7: domain tables -- ════════════════════════════════════════════════════════════════════════════ diff --git a/scripts/passes/annotation.lua b/scripts/passes/annotation.lua index 8352a9c..80af908 100644 --- a/scripts/passes/annotation.lua +++ b/scripts/passes/annotation.lua @@ -3,6 +3,10 @@ --- Validates `MipsAtom_(name) atom_info(atom_bind(Binds_X), atom_reads(...), atom_writes(...)) { ... }` declarations in source files. --- Also reads: `Binds_*` struct declarations (`typedef Struct_(Binds_X) { ... };`) --- +--- Source scanning: done ONCE upstream by `duffle.scan_source()` (ps1_meta.lua pre-scans each +--- source and stashes the result in `src.scan`). This pass is pure: read from the scan, run +--- checks, emit findings. No source re-walking. +--- --- Writes: --- - `/.errors.h` — one per module, with `#error` directives on findings (the C compile will surface the error) --- - The annotations.txt report is rendered by `passes/report.lua` from the per-module results stashed in `ctx.flags._annot_results` @@ -18,23 +22,8 @@ local _src = debug.getinfo(1, "S").source:sub(2) local _dir = _src:match("(.*[/\\])") or "./" dofile(_dir .. "../duffle_paths.lua") local duffle = require("duffle") -local is_space = duffle.is_space -local is_alpha = duffle.is_alpha -local is_alnum = duffle.is_alnum -local trim = duffle.trim -local find_byte = duffle.find_byte -local read_file = duffle.read_file local write_file = duffle.write_file local ensure_dir = duffle.ensure_dir -local dirname = duffle.dirname -local basename_no_ext = duffle.basename_no_ext -local skip_str_or_cmt = duffle.skip_str_or_cmt -local skip_ws_and_cmt = duffle.skip_ws_and_cmt -local read_ident = duffle.read_ident -local read_parens = duffle.read_parens -local read_braces = duffle.read_braces -local scan_to_char = duffle.scan_to_char -local split_top_level_commas = duffle.split_top_level_commas -- Domain tables (single source of truth in duffle.lua). local WAVE_CONTEXT_REGS = duffle.WAVE_CONTEXT_REGS @@ -42,43 +31,6 @@ local TAPE_ATOM_MACROS = duffle.TAPE_ATOM_MACROS local function is_wave_context_reg(n) return WAVE_CONTEXT_REGS[n] ~= nil end --- ════════════════════════════════════════════════════════════════════════════ --- Constants --- ════════════════════════════════════════════════════════════════════════════ - --- Atom declaration + annotation identifiers. -local ATOM_DECL = "MipsAtom_" -local ATOM_INFO = "atom_info" -local STRUCT_TYPE = "Struct_" -local PRAGMA_IDENT = "pragma" -local PRAGMA_OPERATOR = "_Pragma" - --- Struct-name prefix + byte size of U4 fields. -local BINDS_PREFIX = "Binds_" -local BINDS_PREFIX_LEN = 6 -- = #BINDS_PREFIX -local U4_TYPE = "U4" -local U4_BYTES = 4 -- sizeof(U4) -local BINDS_FIELD_PREFIX = "R_" -- wave-context register name prefix - --- TAPE_WORDS pragma keys (the third token after #pragma). -local WORDS_KEY = "words" -local WORDS_KEY_PREFIX = "words=" -- the per-macro `words=N` form -local WORDS_KEY_PREFIX_LEN = 6 -- = #WORDS_KEY_PREFIX -local TAPE_ATOM_WORDS_KEY = "tape_atom words" -- the _Pragma form - --- ASCII byte values used in tokenization. -local BYTE_NEWLINE = 10 -local BYTE_SPACE = 32 -local BYTE_DQUOTE = 34 -local BYTE_EQUALS = 61 -local BYTE_OPEN_PAREN = 40 -local BYTE_OPEN_BRACE = 123 -local BYTE_OPEN_BRACK = 91 -local BYTE_CLOSE_PAREN = 41 -local BYTE_CLOSE_BRACE = 125 -local BYTE_CLOSE_BRACK = 93 -local BYTE_COMMA = 44 - -- ════════════════════════════════════════════════════════════════════════════ -- Type declarations -- ════════════════════════════════════════════════════════════════════════════ @@ -88,6 +40,7 @@ local BYTE_COMMA = 44 --- @field text string -- the full source text --- @field dir string -- the directory containing the source --- @field basename string -- filename without extension +--- @field scan table -- pre-scanned SourceScan payload (from duffle.scan_source) --- @class PassCtx --- @field sources SourceFile[] @@ -107,10 +60,6 @@ local BYTE_COMMA = 44 --- @field errors table[] --- @field warnings table[] ---- @class Atom ---- @field line integer -- source line of the MipsAtom_ declaration ---- @field name string -- atom name (e.g. "cube_g4_face") - --- @class AtomAnnotation --- @field line integer -- source line of the atom_info call --- @field macro string -- the macro name (always "atom_info" in the new shape) @@ -119,599 +68,64 @@ local BYTE_COMMA = 44 --- @field binds string|nil -- Binds_X name if any --- @field reads string[] -- R_* names (read targets) --- @field writes string[] -- R_* names (write targets) ---- @field error string|nil -- error message if annotation was malformed ---- @field errors string[] -- nested errors from per-arg validation - ---- @class BindsField ---- @field name string -- field name ---- @field offset integer -- byte offset within the Binds_X struct - ---- @class BindsStruct ---- @field name string -- struct name (e.g. "Binds_Floor") ---- @field line integer -- source line of the typedef ---- @field bytes integer -- total byte size ---- @field fields BindsField[] -- the field list - ---- @class MacroEntry ---- @field name string -- macro name (e.g. "mac_format_f3_color") ---- @field line integer -- source line of the TAPE_WORDS pragma ---- @field words integer -- declared word count --- @class Finding --- @field line integer -- source line (or 0 for pass-level) --- @field msg string -- finding message --- @class AnnotatedResult ---- @field atoms Atom[] +--- @field atoms AtomEntry[] --- @field annots AtomAnnotation[] --- @field macros MacroEntry[] ---- @field binds BindsStruct[] +--- @field binds BindsEntry[] --- @field errors Finding[] --- @field warnings Finding[] --- @field info Finding[] ---- ════════════════════════════════════════════════════════════════════════════ --- split helpers --- ════════════════════════════════════════════════════════════════════════════ - ---- Split a string at top-level commas. Used inside TAPE_ATOM_* macro ---- bodies where nested parens/braces/brackets are possible. ---- @param s string ---- @return string[] -local function split_csv_top(s) - local tokens = {} - local pos = 1 - local chunk_a = 1 - local depth = 0 - local str_len = #s - while pos <= str_len do - local ch = s:byte(pos) - if ch == BYTE_OPEN_PAREN or ch == BYTE_OPEN_BRACE or ch == BYTE_OPEN_BRACK then - depth = depth + 1 - pos = pos + 1 - elseif ch == BYTE_CLOSE_PAREN or ch == BYTE_CLOSE_BRACE or ch == BYTE_CLOSE_BRACK then - depth = depth - 1 - pos = pos + 1 - elseif ch == BYTE_COMMA and depth == 0 then - tokens[#tokens + 1] = s:sub(chunk_a, pos - 1) - pos = pos + 1 - chunk_a = pos - else - pos = pos + 1 - end - end - local last = s:sub(chunk_a) - if trim(last) ~= "" then tokens[#tokens + 1] = last end - return tokens -end - ---- Split a string into whitespace-separated tokens. ---- @param s string ---- @return string[] -local function split_ws(s) - local tokens = {} - local pos = 1 - local n = 1 - local len = #s - while pos <= len do - -- Skip whitespace. - while pos <= len and is_space(s:sub(pos, pos)) do pos = pos + 1 end - if pos > len then break end - local chunk_a = pos - -- Take non-whitespace run. - while pos <= len and not is_space(s:sub(pos, pos)) do pos = pos + 1 end - tokens[n] = s:sub(chunk_a, pos - 1) - n = n + 1 - end - return tokens -end - --- ════════════════════════════════════════════════════════════════════════════ --- Parse TAPE_ATOM_ANNOT(...) calls --- ════════════════════════════════════════════════════════════════════════════ - --- Recognize a `atom_bind(...)`, `atom_reads(...)`, or `atom_writes(...)` sub-call embedded inside an atom_info arg list. --- Returns the kind ("atom_bind" / "atom_reads" / "atom_writes") and the inner content, or nil if the token isn't a recognized sub-call form. --- Flattened via a prefix lookup instead of a nested if/elseif chain. -local REGS_CALL_PREFIX = { - ["atom_writes("] = { kind = "atom_writes", inner_offset = 13 }, - ["atom_reads("] = { kind = "atom_reads", inner_offset = 12 }, - ["atom_bind("] = { kind = "atom_bind", inner_offset = 11, single_ident = true }, -} - -local function parse_regs_call(s) - if s:sub(-1) ~= ")" then return nil end - -- Try longest prefix first so "atom_writes(" wins over "atom_reads(" when both 12-char prefixes would otherwise match. - -- Lengths: - -- atom_writes( = 12 chars, offset 13 - -- atom_reads( = 11 chars, offset 12 - -- atom_bind( = 10 chars, offset 11 - local spec = REGS_CALL_PREFIX[s:sub(1, 12)] - if not spec then spec = REGS_CALL_PREFIX[s:sub(1, 11)] end - if not spec then spec = REGS_CALL_PREFIX[s:sub(1, 10)] end - if not spec then return nil end - local inner = s:sub(spec.inner_offset, -2) - if spec.single_ident then - -- atom_bind takes a single Binds_* type ident. Trim and pass through. - return spec.kind, trim(inner) - end - return spec.kind, inner -end - --- Parse a comma-separated inner content (e.g. inside atom_reads(...)) into a list of trimmed identifiers. -local function parse_regs_list(inner) - local out = {} - for _, r in ipairs(split_csv_top(inner)) do - local trimmed = trim(r) - if trimmed ~= "" then out[#out + 1] = trimmed end - end - return out -end - --- Parse a single token (from split_csv_top) into an arg entry. --- Three forms: register-list call, bare identifier, "other" (preserved as text). -local function parse_arg_token(s) - local kind, inner = parse_regs_call(s) - if kind then - if kind == "atom_bind" then - return { kind = kind, value = inner } -- single ident, not a list - end - return { kind = kind, value = parse_regs_list(inner) } - end - local id = read_ident(s, 1) - if id and trim(s) == id then - return { kind = "ident", value = id } - end - return { kind = "other", value = s } -end - ---- Extract identifier args from a parenthesized group. ---- Returns a list of {kind, value} pairs where kind is one of: ---- "ident" -- a bare identifier (e.g. a reserved token) ---- "atom_reads" -- an atom_reads(...) call: value is the register list ---- "atom_writes" -- an atom_writes(...) call: value is the register list ---- "other" -- something we can't classify (preserved as text) -local function parse_atom_info_args(inner) - local args = {} - for _, tok in ipairs(split_csv_top(inner)) do - local s = trim(tok) - if s ~= "" then - args[#args + 1] = parse_arg_token(s) - end - end - return args -end - --- ════════════════════════════════════════════════════════════════════════════ --- Parse TAPE_WORDS(mac_X, N) pragma directives --- ════════════════════════════════════════════════════════════════════════════ - ---- Skip preprocessor directives (lines starting with `#`). ---- Returns the position past the newline at the end of the line. ---- @param source string ---- @param pos integer ---- @return integer ---- Parse `_Pragma("mac_X tape_atom words=N")` (operator form). ---- @param source string ---- @param ident_pos integer -- position of the `_Pragma` ident ---- @param after_ident integer -- position just past the ident ---- @return MacroEntry|nil, integer -- (entry or nil, new source position) -local function parse_pragma_operator(source, ident_pos, after_ident) - local open_paren = skip_ws_and_cmt(source, after_ident) - if source:byte(open_paren) ~= BYTE_OPEN_PAREN then - return nil, open_paren + 1 - end - local str, str_end = read_parens(source, open_paren) - -- scan: _Pragma() - str = trim(str) - if str:sub(1, 1) ~= '"' or str:sub(-1) ~= '"' then - return nil, str_end - end - local inner = str:sub(2, -2) - local space = find_byte(inner, BYTE_SPACE, 1) - if not space then return nil, str_end end - local name = inner:sub(1, space - 1) - local rest = inner:sub(space + 1) - local eq = find_byte(rest, BYTE_EQUALS, 1) - if not eq then return nil, str_end end - local key = trim(rest:sub(1, eq - 1)) - local val = trim(rest:sub(eq + 1)) - if key ~= TAPE_ATOM_WORDS_KEY and key ~= WORDS_KEY then return nil, str_end end - return { - line = source:sub(1, ident_pos) and 0 or 0, -- see line_of below - name = name, - words = tonumber(val) or 0, - }, str_end -end - ---- Parse `#pragma mac_X tape_atom words=N` (directive form). ---- @param source string ---- @param ident_pos integer -- position of the `pragma` ident ---- @param after_ident integer -- position just past the ident ---- @return MacroEntry|nil, integer -- (entry or nil, new source position) -local function parse_pragma_directive(source, ident_pos, after_ident) - local str_len = #source - local rest_start = skip_ws_and_cmt(source, after_ident) - local eol = rest_start - while eol <= str_len and source:byte(eol) ~= BYTE_NEWLINE do - eol = eol + 1 - end - local line_text = trim(source:sub(rest_start, eol - 1)) - -- scan: #pragma tape_atom words= - local tokens = split_ws(line_text) - local entry - if #tokens >= 3 and tokens[2] == "tape_atom" and tokens[3]:sub(1, WORDS_KEY_PREFIX_LEN) == WORDS_KEY_PREFIX then - entry = { - name = tokens[1], - words = tonumber(tokens[3]:sub(WORDS_KEY_PREFIX_LEN + 1)) or 0, - } - elseif #tokens >= 2 and tokens[2]:sub(1, WORDS_KEY_PREFIX_LEN) == WORDS_KEY_PREFIX then - entry = { - name = tokens[1], - words = tonumber(tokens[2]:sub(WORDS_KEY_PREFIX_LEN + 1)) or 0, - } - end - if entry then - local line_of = duffle.LineIndex(source) - entry.line = line_of(ident_pos) - end - return entry, eol -end - ---- Find every `TAPE_WORDS(mac_X, N)` pragma in source. ---- Accepts both forms: ---- `_Pragma("mac_X tape_atom words=N")` (operator form) ---- `#pragma mac_X tape_atom words=N` (directive form) ---- @param source string ---- @return MacroEntry[] -local function find_macro_word_annotations(source) - local out = {} - local pos = 1 - local str_len = #source - while pos <= str_len do - pos = skip_ws_and_cmt(source, pos) - if pos > str_len then break end - - -- Skip preprocessor directives (lines starting with #). - -- (_Pragma is an operator, not a directive — it doesn't start with #.) - if source:byte(pos) == 35 then -- '#' - pos = duffle.skip_preprocessor_line(source, pos) - goto continue - end - - local ident, after_ident = read_ident(source, pos) - -- scan: - if not ident then pos = pos + 1; goto continue end - - if ident == PRAGMA_OPERATOR then - -- scan: _Pragma(...) - local entry, new_pos = parse_pragma_operator(source, pos, after_ident) - if entry then - local line_of = duffle.LineIndex(source) - entry.line = line_of(pos) - out[#out + 1] = entry - end - pos = new_pos - elseif ident == PRAGMA_IDENT then - -- scan: #pragma tape_atom words= - local entry, new_pos = parse_pragma_directive(source, pos, after_ident) - if entry then out[#out + 1] = entry end - pos = new_pos - else - pos = after_ident - end - - ::continue:: - end - return out -end - --- ════════════════════════════════════════════════════════════════════════════ --- Parse `typedef Struct_(Binds_X) { ... };` declarations --- ════════════════════════════════════════════════════════════════════════════ - ---- Walk a `Binds_X` body string and extract U4 fields (name + byte offset). ---- Only U4 fields are tracked (Binds_* are always word arrays in this ---- codebase -- pointers stored as U4, indices as U4, etc.). ---- @param body string -- the brace-delimited body (without the braces) ---- @return BindsField[] -- field list ---- @return integer -- total byte size -local function parse_binds_body(body) - local fields = {} - local byte_off = 0 - local pos = 1 - local body_len = #body - while pos <= body_len do - pos = skip_ws_and_cmt(body, pos) - if pos > body_len then break end - local type_ident, type_after = read_ident(body, pos) - if not type_ident then - pos = pos + 1 - elseif type_ident == U4_TYPE then - local field_after = skip_ws_and_cmt(body, type_after) - local fid, fafter = read_ident(body, field_after) - if fid then - fields[#fields + 1] = { name = fid, offset = byte_off } - byte_off = byte_off + U4_BYTES - end - pos = fafter or (type_after + 1) - else - pos = type_after + 1 - end - end - return fields, byte_off -end - ---- Try to parse a `typedef Struct_(Binds_X) { ... };` declaration. ---- Returns the parsed BindsStruct (if the form matched) and the new source position. ---- If the form didn't match, returns nil + a position to continue scanning from. ---- @param source string ---- @param ident_pos integer -- position of the `typedef` ident start ---- @param after_typedef integer -- position just past `typedef` ---- @param line_of fun(pos: integer): integer ---- @return BindsStruct|nil, integer -local function parse_typedef_binds(source, ident_pos, after_typedef, line_of) - local after_type = skip_ws_and_cmt(source, after_typedef) - local type_ident, after_type_ident = read_ident(source, after_type) - -- scan: typedef - if type_ident ~= STRUCT_TYPE then - return nil, after_type_ident or (after_type + 1) - end - - local open_paren = skip_ws_and_cmt(source, after_type_ident) - if source:byte(open_paren) ~= BYTE_OPEN_PAREN then - return nil, open_paren + 1 - end - - local inner, after_paren = read_parens(source, open_paren) - -- scan: typedef Struct_() - local name = trim(inner) - - local brace = scan_to_char(source, "{", after_paren) - -- scan: typedef Struct_() { - if not brace then return nil, open_paren + 1 end - - local body, after_brace = read_braces(source, brace) - -- scan: typedef Struct_() { } - local fields, bytes = parse_binds_body(body) - - -- Only emit Binds_* structs (other Struct_ typedefs are ignored). - if name:sub(1, BINDS_PREFIX_LEN) ~= BINDS_PREFIX then - return nil, after_brace - end - - return { - line = line_of(ident_pos), - name = name, - fields = fields, - bytes = bytes, - }, after_brace -end - ---- Find every `Binds_*` struct declaration. ---- @param source string ---- @return BindsStruct[] -local function find_binds_structs(source) - local line_of = duffle.LineIndex(source) - local out = {} - local pos = 1 - local str_len = #source - while pos <= str_len do - pos = skip_ws_and_cmt(source, pos) - if pos > str_len then break end - - if source:byte(pos) == 35 then -- '#' - pos = duffle.skip_preprocessor_line(source, pos) - goto continue - end - - local ident, after_ident = read_ident(source, pos) - -- scan: - if not ident then pos = pos + 1; goto continue end - if ident ~= "typedef" then pos = after_ident; goto continue end - - -- scan: typedef Struct_() { } - local binds_struct, new_pos = parse_typedef_binds(source, pos, after_ident, line_of) - if binds_struct then out[#out + 1] = binds_struct end - pos = new_pos - - ::continue:: - end - return out -end - --- ════════════════════════════════════════════════════════════════════════════ --- Find every MipsAtom_(name) { ... } declaration in source --- ════════════════════════════════════════════════════════════════════════════ - ---- Read the next identifier token from `s` starting at `pos`, where the identifier is a contiguous run of `[a-zA-Z0-9_]` ---- characters (no underscore-starting alpha-only constraint). ---- Returns the ident + the position just past it, or nil + pos if no identifier starts there. ---- @param s string ---- @param pos integer ---- @return string|nil, integer -local function read_alnum_ident(s, pos) - local str_len = #s; while pos <= str_len and is_space(s:sub(pos, pos)) do pos = pos + 1 end - local start = pos; while pos <= str_len and is_alnum(s:sub(pos, pos)) do pos = pos + 1 end - if pos == start then return nil, pos end - return s:sub(start, pos - 1), pos -end - ---- Find every `MipsAtom_(name)` declaration in source. ---- (Just the name + source line; the body is parsed separately by `parse_mips_atom`.) ---- @param source string ---- @return Atom[] -local function find_atom_names(source) - local line_of = duffle.LineIndex(source) - local out = {} - local pos = 1 - local str_len = #source - while pos <= str_len do - pos = skip_ws_and_cmt(source, pos) - if pos > str_len then break end - - local ident, after_ident = read_ident(source, pos) - -- scan: - if not ident then pos = pos + 1; goto continue end - if ident ~= ATOM_DECL then pos = after_ident; goto continue end - - local open_paren = skip_ws_and_cmt(source, after_ident) - if source:byte(open_paren) ~= BYTE_OPEN_PAREN then pos = open_paren + 1; goto continue end - - local inner, after_paren = read_parens(source, open_paren) - -- scan: MipsAtom_() - local name, _ = read_alnum_ident(inner, 1) - if name and name ~= "" then - out[#out + 1] = { line = line_of(pos), name = name } - end - local brace = scan_to_char(source, "{", after_paren) - -- scan: MipsAtom_() { - if brace then - local _, after_brace = read_braces(source, brace) - -- scan: MipsAtom_() { } - pos = after_brace - else - pos = open_paren + 1 - end - - ::continue:: - end - return out -end - --- ════════════════════════════════════════════════════════════════════════════ --- Find atom annotations (atom_info + atom_bind / atom_reads / atom_writes) --- ════════════════════════════════════════════════════════════════════════════ - ---- True iff the parsed arg is a register-list call (any recognized form). -local function is_regs_arg(a) return a and (a.kind == "atom_reads" or a.kind == "atom_writes" or a.kind == "regs") end - ---- Per-macro arg-shape handlers. Each takes (entry, args) and mutates Per-atom_info sub-call dispatch. ---- Each takes (entry, args) and mutates entry.{reads, writes, binds, errors}. ---- All sub-calls are order-independent; each is dispatched on its `kind` (atom_bind / atom_reads / atom_writes) when parsed. -local ANNOT_ARG_HANDLERS = {} - --- atom_info(atom_bind(Binds_X), atom_reads(...), atom_writes(...)) -function ANNOT_ARG_HANDLERS.info(entry, args) - for _, arg in ipairs(args) do - if arg.kind == "atom_bind" then entry.binds = arg.value - elseif arg.kind == "atom_reads" then entry.reads = arg.value - elseif arg.kind == "atom_writes" then entry.writes = arg.value - elseif arg.kind == "ident" then - -- Reserved for future use. Currently ignored. - else - entry.errors[#entry.errors + 1] = string.format("unexpected atom_info arg kind=%s value=%s", arg.kind, tostring(arg.value)) - end - end -end - --- Build a new annotation entry with the standard shape. -local function new_annot_entry(line, ident, name, kind) - return { - line = line, - macro = ident, - name = name, - kind = kind, - binds = nil, - reads = {}, - writes = {}, - errors = {}, - } -end - ---- Try to parse an `atom_info(...)` call right after the `MipsAtom_(name)` parens. ---- Returns the annotation entry (if present) and the new source position past the atom_info call. ---- Returns nil if no atom_info follows. ---- @param source string ---- @param atom_name string ---- @param after_mipsatom_paren integer -- position past the MipsAtom_(...) close paren ---- @param line_of fun(pos: integer): integer ---- @return AtomAnnotation|nil, integer -- (entry or nil, new source position) -local function parse_atom_info_call(source, atom_name, after_mipsatom_paren, line_of) - local lookahead = skip_ws_and_cmt(source, after_mipsatom_paren) - local look_ident, look_after = read_ident(source, lookahead) - -- scan: MipsAtom_() - if look_ident ~= ATOM_INFO then return nil, after_mipsatom_paren end - - local info_open = skip_ws_and_cmt(source, look_after) - if source:byte(info_open) ~= BYTE_OPEN_PAREN then return nil, info_open + 1 end - - local info_inner, info_after = read_parens(source, info_open) - -- scan: MipsAtom_() atom_info(, , ) - local args = parse_atom_info_args(info_inner) - local entry = new_annot_entry(line_of(lookahead), ATOM_INFO, atom_name, "info") - ANNOT_ARG_HANDLERS.info(entry, args) - return entry, info_after -end - ---- Find every `MipsAtom_(name) atom_info(...) { ... };` annotation in source. ---- Returns a list of annotation entries. Atoms without a following `atom_info(...)` call produce NO entry ---- (atoms without annotations are valid in the new minimal shape). ---- @param source string ---- @return AtomAnnotation[] -local function find_atom_annotations(source) - local line_of = duffle.LineIndex(source) - local annots = {} - local pos = 1 - local str_len = #source - while pos <= str_len do - pos = skip_ws_and_cmt(source, pos) - if pos > str_len then break end - - -- Skip preprocessor directives (lines starting with #). - if source:byte(pos) == 35 then -- '#' - pos = duffle.skip_preprocessor_line(source, pos) - goto continue - end - - local ident, after_ident = read_ident(source, pos) - -- scan: - if not ident then pos = pos + 1; goto continue end - if ident ~= ATOM_DECL then pos = after_ident; goto continue end - - local open_paren = skip_ws_and_cmt(source, after_ident) - if source:byte(open_paren) ~= BYTE_OPEN_PAREN then pos = open_paren + 1; goto continue end - - local inner, after_paren = read_parens(source, open_paren) - -- scan: MipsAtom_() - local name, _ = read_alnum_ident(inner, 1) - - local entry, new_pos = parse_atom_info_call(source, name, after_paren, line_of) - -- scan: MipsAtom_() atom_info(, , ) - if entry then annots[#annots + 1] = entry end - pos = new_pos - - -- Skip past the body { ... } if present. - local brace = scan_to_char(source, "{", pos) - -- scan: MipsAtom_() atom_info(...) { - if brace then - local _, after_brace = read_braces(source, brace) - -- scan: MipsAtom_() atom_info(...) { } - pos = after_brace - end - - ::continue:: - end - return annots -end - -- ════════════════════════════════════════════════════════════════════════════ -- Validation -- ════════════════════════════════════════════════════════════════════════════ +-- +-- Pure check: read from src.scan, run validations, emit findings. +-- No source walking; no parsing. The scan was done once upstream. +--- Validate one source against its pre-scanned SourceScan payload. +--- @param ctx PassCtx +--- @param src SourceFile +--- @return AnnotatedResult local function validate(ctx, src) - local source = src.text + local scan = src.scan - local annots = find_atom_annotations(source) - local macros = find_macro_word_annotations(source) - local binds = find_binds_structs(source) - local atoms = find_atom_names(source) + -- Project the pre-scanned atoms to the AtomEntry shape this pass needs. + local atoms = {} + for _, a in ipairs(scan.atoms) do + if a.kind == "atom" then + atoms[#atoms + 1] = { line = a.line, name = a.raw_name } + end + end + -- Project the pre-scanned atom_infos to AtomAnnotation shape. + local annots = {} + for _, info in ipairs(scan.atom_infos) do + annots[#annots + 1] = { + line = info.info_line, + macro = "atom_info", + name = info.atom_name, + kind = "info", + binds = info.binds, + reads = info.reads or {}, + writes = info.writes or {}, + errors = {}, + } + end + + -- Index atoms by name for lookup. local atom_index = {} for _, a in ipairs(atoms) do atom_index[a.name] = a end + -- Index binds by name for lookup. local binds_index = {} - for _, b in ipairs(binds) do binds_index[b.name] = b end + for _, b in ipairs(scan.binds) do binds_index[b.name] = b end local errors = {} local warnings = {} @@ -719,9 +133,7 @@ local function validate(ctx, src) -- 1. Every annotated atom must exist as a real MipsAtom_ declaration. for _, a in ipairs(annots) do - if a.error then - errors[#errors + 1] = {line = a.line, msg = a.error} - elseif not atom_index[a.name] then + if not atom_index[a.name] then errors[#errors + 1] = { line = a.line, msg = string.format("annotation for '%s' has no matching MipsAtom_(%s) { ... }", a.name, a.name), @@ -734,11 +146,11 @@ local function validate(ctx, src) end end - -- 2. Every atom may have AT MOST ONE annotation (no duplicates). + -- 2. Every atom may have AT MOST ONE annotation (no duplicates). -- (Atoms with ZERO annotations are valid in the new minimal shape.) local count_per_atom = {} for _, a in ipairs(annots) do - if a.name and not a.error then + if a.name then count_per_atom[a.name] = (count_per_atom[a.name] or 0) + 1 end end @@ -751,8 +163,7 @@ local function validate(ctx, src) end end - -- 3. (Phase validity check DROPPED. Phases were removed from the annotation DSL. - -- They may be reintroduced later as sub-calls of atom_info, at which point ordering checks will go here.) + -- 3. (Phase validity check DROPPED. Phases were removed from the annotation DSL.) -- 4. BIND atoms must reference a real Binds_* struct. for _, a in ipairs(annots) do @@ -760,7 +171,7 @@ local function validate(ctx, src) if not binds_index[a.binds] then -- Demoted from error to warning (2026-07-10): the same condition is now caught by passes/static_analysis.lua's -- check_abi_handoff() as an error. Emitting a warning here keeps the annotation pass from being stop-on-error - -- for the common test-fixture case, while still surfacing the issue in the report. + -- for the common test-fixture case, while still surfacing the issue in the report. -- The static-analysis report remains the source of truth for build-stopping errors. warnings[#warnings + 1] = { line = a.line, @@ -774,8 +185,6 @@ local function validate(ctx, src) for _, a in ipairs(annots) do if a.binds and binds_index[a.binds] then local bs = binds_index[a.binds] - local field_names = {} - for _, f in ipairs(bs.fields) do field_names[f.name] = true end for _, f in ipairs(bs.fields) do local candidate = "R_" .. f.name @@ -812,7 +221,6 @@ local function validate(ctx, src) -- 7. TAPE_WORDS(mac_X, N) ↔ WORD_COUNT(mac_X, N) drift. -- Three outcomes: missing (error), mismatch (error), match (info). - -- Flattened via early-return-style helper instead of 3-way elseif. local function check_macro_drift(m, declared) if not declared then errors[#errors + 1] = { @@ -833,7 +241,7 @@ local function validate(ctx, src) msg = string.format("OK: %s = %d words", m.name, m.words), } end - for _, m in ipairs(macros) do + for _, m in ipairs(scan.macros) do check_macro_drift(m, ctx.shared.word_counts[m.name]) end @@ -841,14 +249,14 @@ local function validate(ctx, src) info[#info + 1] = { line = 0, msg = string.format("scanned: %d atom(s), %d annotation(s), %d macro-word-decl(s), %d binds struct(s)", - #atoms, #annots, #macros, #binds), + #atoms, #annots, #scan.macros, #scan.binds), } return { atoms = atoms, annots = annots, - macros = macros, - binds = binds, + macros = scan.macros, + binds = scan.binds, errors = errors, warnings = warnings, info = info, @@ -858,18 +266,12 @@ end -- ════════════════════════════════════════════════════════════════════════════ -- Per-DIRECTORY (per-module) output: errors.h + annotations.txt -- ════════════════════════════════════════════════════════════════════════════ --- --- Per-source reports were the old behavior; each source in the same directory produced its own .errors.h + .annotations.txt, --- which flooded build/gen/ with one report per header. --- Aggregates per-DIRECTORY (one errors.h + one annotations.txt per module basename). --- Directories with zero atoms/annotations are skipped (no file emitted). ---- Render `.errors.h` with `#error` directives for every error found across all sources in the directory. +--- Render `.errors.h` with `#error` directives for every error found across all sources in the directory. --- Empty directories (no errors, no atoms) produce no file. local function emit_module_errors_h(ctx, dir_basename, atoms_count, errors, sources) if ctx.dry_run then return nil end if atoms_count == 0 and #errors == 0 then - -- Skip dirs with nothing to report return nil end local out_path = ctx.out_root .. "/" .. dir_basename .. ".errors.h" @@ -882,8 +284,6 @@ local function emit_module_errors_h(ctx, dir_basename, atoms_count, errors, sour if #errors == 0 then lines[#lines + 1] = "// annotation pass OK" else - -- Prefix each error with the source basename for traceability - -- in the C compile log. for _, e in ipairs(errors) do local src_tag = "" if e.source then @@ -907,7 +307,6 @@ local function emit_module_annotations_stub(ctx, dir, dir_basename, atoms_count) dir_basename = dir_basename, atoms_count = atoms_count, } - -- annotations.txt is written by report.lua end -- ════════════════════════════════════════════════════════════════════════════ @@ -919,7 +318,6 @@ end local M = {} -- Expose `validate` for downstream passes (e.g. report.lua) that need to re-render the per-source results into a per-MODULE report. --- Keeping it as a single shared function avoids the duplication that an earlier version of report.lua had. M.validate = validate --- @param ctx PassCtx @@ -929,9 +327,8 @@ function M.run(ctx) local errors = {} local warnings = {} - -- Per-DIRECTORY (per-module) aggregation. Group sources by `src.dir`, validate every source in the dir, then emit ONE errors.h per dir - -- (skipping dirs with no atoms AND no errors). - -- The actual annotations.txt is rendered by passes/report.lua from the stashed per-module results below. + -- Per-DIRECTORY (per-module) aggregation. Group sources by `src.dir`, + -- validate every source in the dir, then emit ONE errors.h per dir. local by_dir = {} for _, src in ipairs(ctx.sources) do by_dir[src.dir] = by_dir[src.dir] or {} @@ -939,10 +336,8 @@ function M.run(ctx) end for dir, dir_sources in pairs(by_dir) do - -- Dir basename = last component of `dir` ("code/duffle" -> "duffle"). local dir_basename = dir:match("([^/\\]+)$") or dir - -- Aggregate validate() results across the directory. local dir_atoms = 0 local dir_errors = {} local dir_warnings = {} @@ -959,13 +354,11 @@ function M.run(ctx) end end - -- Emit one errors.h per dir. local err_path = emit_module_errors_h(ctx, dir_basename, dir_atoms, dir_errors, dir_sources) if err_path then table.insert(outputs, { errors_h = err_path }) end - -- Stash for report pass. emit_module_annotations_stub(ctx, dir, dir_basename, dir_atoms) end diff --git a/scripts/passes/components.lua b/scripts/passes/components.lua index 30f7912..dc96844 100644 --- a/scripts/passes/components.lua +++ b/scripts/passes/components.lua @@ -1,8 +1,12 @@ --- passes/components.lua — Component-macro header generator. --- ---- Walks every source for `MipsAtomComp_(ac_X) { body }` (and the function-form `MipsAtomComp_Proc_(ac_X, { body })`) declarations and ---- emits a per-directory `.macs.h` containing one `#define mac_X(sig) \` macro per component + `WORD_COUNT(mac_X, N)` ---- entries for downstream offset computation. +--- Reads the pre-scanned SourceScan payload (produced once upstream by `duffle.scan_source`) +--- for `MipsAtomComp_(ac_X)` and `MipsAtomComp_Proc_(ac_X, { body })` declarations, then does +--- per-source backward lookups for the function-args string (from the preceding `FI_ MipsAtom ac_X(...)` +--- function declaration) and the preceding comment block (for LSP/IntelliSense signature docs). +--- +--- Emits a per-directory `.macs.h` containing one `#define mac_X(sig) \` macro per component +--- + `WORD_COUNT(mac_X, N)` entries for downstream offset computation. --- --- **Conventions**: tabs (1/level), EmmyLua annotations, no regex, --- Lua 5.3 compatible. @@ -20,7 +24,7 @@ -- Module-scope requires + package.path setup -- ════════════════════════════════════════════════════════════════════════════ --- Resolve `arg[0]` to an absolute-ish script directory so that `require("duffle")` resolves against `scripts/` regardless of CWD. +-- Bootstrap: same as entry scripts. See `ps1_meta.lua` for the rationale. -- Bootstrap: load `scripts/duffle_paths.lua` (sets package.path + package.cpath). -- Uses `debug.getinfo` to find this file's own directory, so it works -- both standalone and when require'd from the orchestrator. @@ -36,7 +40,6 @@ local word_count_eval = require("word_count_eval") -- Atom component declaration identifiers. local ATOM_COMP_PROC = "MipsAtomComp_Proc_" -local ATOM_COMP = "MipsAtomComp_" local MIPS_ATOM = "MipsAtom" -- prefix on the function declaration that wraps an AtomComp_Proc_ -- Component-name prefixes. @@ -61,6 +64,7 @@ local GEN_SUBDIR = "gen" --- @field text string -- the full source text --- @field dir string -- the directory containing the source --- @field basename string -- filename without extension +--- @field scan table -- pre-scanned SourceScan payload (from duffle.scan_source) --- @class PassCtx --- @field sources SourceFile[] -- all source files in the build @@ -72,7 +76,7 @@ local GEN_SUBDIR = "gen" --- @field upstream table -- per-pass upstream outputs --- @field flags table -- CLI flags --- @field dry_run boolean -- if true, compute but don't write ---- @field verbose boolean -- if true, log diagnostic info +--- @field verbose boolean -- log diagnostic info --- @class PassResult --- @field outputs table[] -- {kind=, path=} entries describing emit files @@ -90,37 +94,6 @@ local GEN_SUBDIR = "gen" -- Local helpers (file I/O + path normalization) -- ════════════════════════════════════════════════════════════════════════════ --- Write content to disk in binary mode so LF line endings are preserved on Windows --- (text mode would convert LF -> CRLF, breaking byte-identical diffs against git-tracked gen/*.macs.h files which are stored as LF). --- @param path string --- @param content string -local function write_file_lf(path, content) - local f = io.open(path, "wb") - if not f then error("Cannot write " .. path) end - f:write(content); f:close() -end - --- Convert a (possibly relative) path to an absolute Windows path. --- The pre-rework output's "// Source:" comment line used the absolute path (e.g. "C:\projects\Pikuma\ps1\code\duffle\lottes_tape.h"); --- If we want byte-identical output, we must normalize relative -> absolute before emitting that comment. --- @param path string --- @return string -local function to_absolute_path(path) - if #path >= 2 and path:sub(2, 2) == ":" then - -- Already absolute; normalize slashes for consistency. - return (path:gsub("/", "\\")) - end - local p = io.popen("cd") - if not p then return path end - local cwd = p:read("*l") - p:close() - if not cwd then return path end - -- Normalize forward slashes to backslashes (Windows convention) on both the cwd AND the relative path tail, so the join is uniform. - cwd = cwd:gsub("/", "\\") - local tail = (path:gsub("/", "\\")) - return cwd .. "\\" .. tail -end - local M = {} -- ════════════════════════════════════════════════════════════════════════════ @@ -372,118 +345,28 @@ local function extract_arg_names(args_str) end -- ════════════════════════════════════════════════════════════════════════════ --- Component scanner (bare + function forms) +-- Component projection (read from pre-scanned SourceScan) -- ════════════════════════════════════════════════════════════════════════════ --- Parse the inner content of an `AtomComp_(name, ...)` call. --- Returns (name, body_or_nil) — `body_or_nil` is non-nil iff this is the function-form `MipsAtomComp_Proc_(name, { body })` invocation. --- @param inner string -- the content between ( and ) of the AtomComp_ call ---- @return string|nil, string|nil -local function parse_atomcomp_inner(inner) - local tokens = duffle.split_top_level_commas(inner) - if #tokens == 1 then - return duffle.trim(tokens[1]), nil - elseif #tokens == 2 then - local name = duffle.trim(tokens[1]) - local body_raw = duffle.trim(tokens[2]) - -- Strip leading { and trailing } if present. - local body - if #body_raw >= 2 and body_raw:sub(1, 1) == "{" and body_raw:sub(-1) == "}" then - body = duffle.trim(body_raw:sub(2, -2)) - else - body = body_raw +-- Project pre-scanned MipsAtomComp_ / MipsAtomComp_Proc_ entries into Component shape. +-- Does per-source backward lookups for args (preceding function decl) and comment (preceding comment block). +-- @param source string -- the full source text (needed for backward lookups) +-- @param scan table -- SourceScan from duffle.scan_source +-- @return Component[] +local function project_components(source, scan) + local out = {} + for _, a in ipairs(scan.atoms) do + if a.kind == "comp_bare" or a.kind == "comp_proc" then + local args = find_function_args_for(source, a.raw_name, a.ident_pos) + local comment = preceding_comment_block(source, a.ident_pos) + out[#out + 1] = { + line = a.line, + name = a.name, + body = a.body, + args = args, + comment = comment, + } end - return name, body - end - return nil, nil -end - --- (internal) Try to extract a bare-form `MipsAtomComp_(ac_X)` declaration. --- Bare form: `MipsAtomComp_(ac_X) { body }` — body comes from the brace block AFTER the parens. --- @param source string --- @param name string -- the `ac_X` ident from the parens ---- @param ident_pos integer -- position of the `MipsAtomComp_` ident start ---- @param after_paren integer -- position just past the closing `)` ---- @param line_of fun(pos: integer): integer ---- @param args string|nil -- function-args from preceding function decl ---- @param comment string -- preceding comment block ---- @return Component|nil, integer -- the component + new source position -local function make_bare_component(source, name, ident_pos, after_paren, line_of, args, comment) - local brace = duffle.scan_to_char(source, "{", after_paren) - -- scan: () { - if not brace then return nil, after_paren + 1 end - local body, after_brace = duffle.read_braces(source, brace) - -- scan: () { } - return { - line = line_of(ident_pos), - name = name:sub(AC_PREFIX_LEN + 1), -- strip "ac_" prefix - body = body, - args = args, - comment = comment, - }, after_brace -end - --- (internal) Build the function-form `MipsAtomComp_Proc_` component. Body --- came from inside the parens; no following brace block. -local function make_proc_component(name, body, ident_pos, line_of, args, comment) - return { - line = line_of(ident_pos), - name = name:sub(AC_PREFIX_LEN + 1), - body = body, - args = args, - comment = comment, - } -end - ---- Find every `MipsAtomComp_(ac_) { body }` declaration in source. ---- Supports BOTH the bare form and the function form: ---- Bare: `MipsAtomComp_(ac_X) { body }` ---- Function: `MipsAtomComp_Proc_(ac_X, { body })` (with a preceding ---- `"FI_ MipsAtom ac_X(args)"` function declaration) ---- ---- @param source string ---- @return Component[] -local function find_component_atoms(source) - local line_of = duffle.LineIndex(source) - local out = {} - local pos = 1 - local src_len = #source - while pos <= src_len do - pos = duffle.skip_ws_and_cmt(source, pos) - if pos > src_len then break end - - local ident, after_ident = duffle.read_ident(source, pos) - -- scan: - local is_comp = ident == ATOM_COMP or ident == ATOM_COMP_PROC - if not ident then pos = pos + 1; goto continue end - if not is_comp then pos = after_ident; goto continue end - - local open_paren = duffle.skip_ws_and_cmt(source, after_ident) - if source:sub(open_paren, open_paren) ~= "(" then pos = open_paren + 1; goto continue end - - local inner, after_paren = duffle.read_parens(source, open_paren) - -- scan: () - local name, body = parse_atomcomp_inner(inner) - -- scan: () OR (, { }) - if not name or name:sub(1, AC_PREFIX_LEN) ~= AC_PREFIX then pos = open_paren + 1; goto continue end - - local args = find_function_args_for(source, name, open_paren) - local comment = preceding_comment_block(source, pos) - if body == nil then - -- Bare form: body comes from the brace block after the parens. - -- scan: () { - local comp, new_pos = make_bare_component(source, name, pos, after_paren, line_of, args, comment) - -- scan: () { } - if comp then out[#out + 1] = comp end - pos = new_pos - else - -- Function form: body was inside the parens. - -- scan: (, { }) - out[#out + 1] = make_proc_component(name, body, pos, line_of, args, comment) - pos = after_paren - end - - ::continue:: end return out end @@ -494,7 +377,7 @@ end -- Convert `//` line comments to `/* */` block comments in a token. -- --- C macros use `\` line-continuations; a `//` comment before `\` would consume the continuation, +-- C macros use `\` line-continuations; a `//` comment before `\` would consume the continuation, -- breaking the macro. We convert `//` to `/* */` so the multi-line macro structure is preserved. -- -- Skips `//` sequences that are inside string or character literals @@ -548,7 +431,7 @@ local function strip_mac_prefix(ident) return ident end --- (internal) Recursive word-count lookup. `cache` is the memoization table across all calls to `compute_component_word_count`; +-- (internal) Recursive word-count lookup. `cache` is the memoization table across all calls to `compute_component_word_count`; -- the in-progress -1 sentinel detects cycles (A -> B -> A). -- @param name string -- the component name (without `mac_`) -- @param comp_by_name table @@ -588,7 +471,7 @@ end --- Compute the word count of a component body, accounting for macro expansion. --- Each comma-separated entry in the body is a "slot" that contributes its own word count. ---- For most entries (regular MIPS instructions) the count is 1. +--- For most entries (regular MIPS instructions) the count is 1. --- For `mac_Y(...)` calls, the count is the word count of `mac_Y` (recursive lookup through `components`). --- For encoding macros with a known multi-word count (e.g. `mask_upper` = 2), --- the count is taken from `word_counts`. @@ -677,7 +560,7 @@ local function emit_macro_body(lines, c, sig, tokens) strip_trailing_continuation(lines) end ---- Build the list of lines for one component +--- Build the list of lines for one component --- (signature comment, `#define mac_X(...)` line with backslash-continued tokens, then `WORD_COUNT(mac_X, N)` entry). --- @param c Component --- @param components Component[] @@ -711,19 +594,19 @@ end -- Per-source emit logic -- ════════════════════════════════════════════════════════════════════════════ --- Build the boilerplate header lines (the `#ifdef INTELLISENSE_DIRECTIVES` block, +-- Build the boilerplate header lines (the `#ifdef INTELLISENSE_DIRECTIVES` block, -- the `// Auto-generated` comment, the `// Source:` line, and the self-contained `WORD_COUNT` macro definition). -- @param src SourceFile -- @return string[] local function header_boilerplate(src) return { - -- #pragma once wrapped in #ifdef INTELLISENSE_DIRECTIVES, matching the convention in lottes_tape.h. + -- #pragma once wrapped in #ifdef INTELLISENSE_DIRECTIVES, matching the convention in lottes_tape.h. -- The build does manual unity includes (the user controls include order), so the pragma is only active for IDE/tooling. "#ifdef INTELLISENSE_DIRECTIVES", "#pragma once", "#endif", "// Auto-generated by ps1_meta.lua — DO NOT EDIT", - "// Source: " .. to_absolute_path(src.path), + "// Source: " .. duffle.to_absolute_path(src.path), "// Component atoms (MipsAtomComp_(ac_*)) -> macro variants (mac_*)", "", -- Self-contained: define WORD_COUNT if not already defined. @@ -737,8 +620,8 @@ local function header_boilerplate(src) end -- Compute the output path for one source's `.macs.h` file. --- The pre-rework convention uses the *directory* basename --- (not the source file basename) — e.g. `code/duffle/lottes_tape.h` produces `code/duffle/gen/duffle.macs.h`. +-- The pre-rework convention uses the *directory* basename +-- (not the source file basename) — e.g. `code/duffle/lottes_tape.h` produces `code/duffle/gen/duffle.macs.h`. -- This matches what the C codebase #includes. -- @param src SourceFile -- @return string -- the output directory @@ -779,7 +662,7 @@ local function emit_component_macros_h(ctx, src, components) end duffle.ensure_dir(out_dir) - write_file_lf(out_path, content) + duffle.write_file_lf(out_path, content) print(string.format(" -> %s", out_path)) return out_path end @@ -788,7 +671,7 @@ end -- Pass entry -- ════════════════════════════════════════════════════════════════════════════ --- (internal) Extend `ctx.shared.word_counts` with this source's component macros +-- (internal) Extend `ctx.shared.word_counts` with this source's component macros -- so offsets sees them without re-reading the file. -- @param ctx PassCtx -- @param components Component[] @@ -807,8 +690,8 @@ function M.run(ctx) local warnings = {} for _, src in ipairs(ctx.sources) do - -- find_component_atoms operates on src.text - local components = find_component_atoms(src.text) + -- project_components reads from src.scan + does backward lookups on src.text + local components = project_components(src.text, src.scan) if #components > 0 then local macs_path = emit_component_macros_h(ctx, src, components) if macs_path then diff --git a/scripts/passes/offsets.lua b/scripts/passes/offsets.lua index 2eaae2e..65e89e9 100644 --- a/scripts/passes/offsets.lua +++ b/scripts/passes/offsets.lua @@ -1,8 +1,9 @@ --- passes/offsets.lua — Branch-offset generator. --- ---- Scans every source for `MipsAtom_(name) { ... }` (and the raw `MipsCode code_ { ... }` form) declarations, ---- computes the word offset from each `atom_offset(F, T)` marker to its target `atom_label(T)` declaration, ---- and emits `.offsets.h` with one `#define _atom_offset_F_T = N` per branch. +--- Reads the pre-scanned SourceScan payload (produced once upstream by `duffle.scan_source`) +--- for `MipsAtom_(name)` and `MipsCode code_` declarations, computes the word offset +--- from each `atom_offset(F, T)` marker to its target `atom_label(T)` declaration, and emits +--- `.offsets.h` with one `#define _atom_offset_F_T = N` per branch. --- --- The offset is `target_word - branch_word - 1` (the standard MIPS branch-immediate encoding: branch_offset = relative_pc_in_words - 1). --- @@ -13,8 +14,7 @@ -- Module-scope requires + package.path setup -- ════════════════════════════════════════════════════════════════════════════ --- Resolve `arg[0]` to an absolute-ish script directory so that `require("duffle")` resolves against `scripts/` regardless of CWD. --- Bootstrap: see `ps1_meta.lua` for the rationale. +-- Bootstrap: same as entry scripts. See `ps1_meta.lua` for the rationale. -- Bootstrap: load `scripts/duffle_paths.lua` (sets package.path + package.cpath). -- Uses `debug.getinfo` to find this file's own directory, so it works -- both standalone and when require'd from the orchestrator. @@ -29,21 +29,6 @@ local count_token_words = word_count_eval.count_token_words -- Constants -- ════════════════════════════════════════════════════════════════════════════ --- C qualifier keywords that may precede a `MipsAtom_` declaration --- (and should be skipped by `skip_qualifiers`). -local QUALIFIER_KEYWORDS = { - ["static"] = true, ["const"] = true, ["volatile"] = true, - ["extern"] = true, ["register"] = true, ["auto"] = true, - ["inline"] = true, ["typedef"] = true, - ["internal"] = true, ["LP_"] = true, ["global"] = true, ["gkknown"] = true, -} - --- Atom declaration identifiers. -local ATOM_PREFIX = "MipsAtom_" -local CODE_DECL = "MipsCode" -local CODE_RAW_PREFIX = "code_" -- raw atom form: `MipsCode code_ { ... }` -local CODE_RAW_PREFIX_LEN = 5 -- = #CODE_RAW_PREFIX - -- Marker-call identifiers inside atom bodies. local LABEL_MARKER = "atom_label" local OFFSET_MARKER = "atom_offset" @@ -64,6 +49,7 @@ local OFFSET_MACRO_COL = 44 --- @field text string -- the full source text --- @field dir string -- the directory containing the source --- @field basename string -- filename without extension +--- @field scan table -- pre-scanned SourceScan payload (from duffle.scan_source) --- @class PassCtx --- @field sources SourceFile[] -- all source files in the build @@ -75,17 +61,13 @@ local OFFSET_MACRO_COL = 44 --- @field upstream table -- per-pass upstream outputs --- @field flags table -- CLI flags --- @field dry_run boolean -- if true, compute but don't write ---- @field verbose boolean -- if true, log diagnostic info +--- @field verbose boolean -- log diagnostic info --- @class PassResult --- @field outputs table[] -- {kind=, path=} entries describing emit files --- @field errors table[] -- {line=, msg=} entries; build-stops --- @field warnings table[] -- {line=, msg=} entries; build-succeeds ---- @class Atom ---- @field name string -- atom name (e.g. "cube_g4_face") ---- @field body string -- the brace-delimited body (without the braces) - --- @class BranchOffset --- @field tag string -- the marker tag (e.g. "F" in `atom_offset(F, T)`) --- @field target string -- the target label name (e.g. "T" in `atom_offset(F, T)`) @@ -98,44 +80,7 @@ local OFFSET_MACRO_COL = 44 --- @field offsets BranchOffset[] -- per-branch offset list -- ════════════════════════════════════════════════════════════════════════════ --- Local helpers --- ════════════════════════════════════════════════════════════════════════════ - --- Returns true if `s` starts with `prefix`. --- @param s string --- @param prefix string --- @return boolean -local function starts_with(s, prefix) - if #s < #prefix then return false end - for pos = 1, #prefix do - if s:sub(pos, pos) ~= prefix:sub(pos, pos) then return false end - end - return true -end - --- Replace every non-alphanumeric char in `s` with underscore. --- @param s string --- @return string -local function to_alnum_underscore(s) - local out = "" - for pos = 1, #s do - local ch = s:sub(pos, pos) - if duffle.is_alnum(ch) then out = out .. ch else out = out .. "_" end - end - return out -end - --- Right-pad `s` with spaces to width `w`. If `s` is already `w` or --- wider, no padding is added. --- @param s string --- @param w integer --- @return string -local function pad_right(s, w) - return s .. string.rep(" ", math.max(0, w - #s)) -end - --- ════════════════════════════════════════════════════════════════════════════ --- Marker-call helpers +-- Per-token marker-call helpers (atom_label / atom_offset inside bodies) -- ════════════════════════════════════════════════════════════════════════════ -- Extract comma-separated identifier args from a parenthesized group after a function-like macro call. @@ -254,123 +199,12 @@ local function find_marker_call_end(tok) end -- ════════════════════════════════════════════════════════════════════════════ --- Atom scanner +-- Per-atom body scan (for atom_label / atom_offset markers) -- ════════════════════════════════════════════════════════════════════════════ ---- Skip C qualifier keywords (`static`, `const`, etc.) and return the position past the last qualifier. ---- @param source string ---- @param pos integer ---- @return integer -local function skip_qualifiers(source, pos) - while true do - pos = duffle.skip_ws_and_cmt(source, pos) - local ident, after = duffle.read_ident(source, pos) - if not ident then return pos end - if QUALIFIER_KEYWORDS[ident] then pos = after else return pos end - end -end - --- (internal) Try to parse the wrapped atom form: `MipsAtom_() { ... }`. --- Returns the parsed Atom (name + body + position past body), or nil if the form didn't match. --- @param source_text string --- @param after_pos integer -- position just past `MipsAtom_` --- @return Atom|nil -local function try_wrapped_atom(source_text, after_pos) - local paren_pos = duffle.skip_ws_and_cmt(source_text, after_pos) - if source_text:sub(paren_pos, paren_pos) ~= "(" then return nil end - local inner, after_paren = duffle.read_parens(source_text, paren_pos) - -- scan: MipsAtom_() - - local name_start = 1 - while name_start <= #inner and duffle.is_space(inner:sub(name_start, name_start)) do - name_start = name_start + 1 - end - local name_end = name_start - while name_end <= #inner and duffle.is_alnum(inner:sub(name_end, name_end)) do - name_end = name_end + 1 - end - local name = inner:sub(name_start, name_end - 1) - if name == "" then return nil end - - local brace_pos = duffle.scan_to_char(source_text, "{", after_paren) - -- scan: MipsAtom_() { - if not brace_pos then return nil end - local body, after_brace = duffle.read_braces(source_text, brace_pos) - -- scan: MipsAtom_() { } - return { name = name, body = body, after_brace = after_brace } -end - --- (internal) Try to parse the raw atom form: `MipsCode code_ { ... }`. --- @param source_text string --- @param after_pos integer -- position just past `MipsCode` --- @return Atom|nil -local function try_raw_atom(source_text, after_pos) - local next_pos = duffle.skip_ws_and_cmt(source_text, after_pos) - local next_ident, next_after = duffle.read_ident(source_text, next_pos) - -- scan: MipsCode - if not next_ident then return nil end - if not starts_with(next_ident, CODE_RAW_PREFIX) then return nil end - if #next_ident <= CODE_RAW_PREFIX_LEN then return nil end - local atom_name = next_ident:sub(CODE_RAW_PREFIX_LEN + 1) - -- scan: MipsCode code_ - local brace_pos = duffle.scan_to_char(source_text, "{", next_after) - -- scan: MipsCode code_ { - if not brace_pos then return nil end - local body, after_brace = duffle.read_braces(source_text, brace_pos) - -- scan: MipsCode code_ { } - return { name = atom_name, body = body, after_brace = after_brace } -end - ---- Find every `MipsAtom_(name) { ... }` (or raw `MipsCode code_ { ... }`) declaration in a source. ---- @param source_text string ---- @return Atom[] -local function find_atoms(source_text) - local atoms = {} - local pos = 1 - local src_len = #source_text - - while pos <= src_len do - pos = duffle.skip_ws_and_cmt(source_text, pos); if pos > src_len then break end - pos = skip_qualifiers(source_text, pos); if pos > src_len then break end - - local ident, after = duffle.read_ident(source_text, pos) - -- scan: - if not ident then pos = pos + 1; goto continue end - - if ident == ATOM_PREFIX then - -- scan: MipsAtom_() { } - local atom = try_wrapped_atom(source_text, after) - if atom then - atoms[#atoms + 1] = { name = atom.name, body = atom.body } - pos = atom.after_brace - else - pos = pos + 1 - end - elseif ident == CODE_DECL then - -- scan: MipsCode code_ { } - local atom = try_raw_atom(source_text, after) - if atom then - atoms[#atoms + 1] = { name = atom.name, body = atom.body } - pos = atom.after_brace - else - pos = after - end - else - pos = after - end - - ::continue:: - end - return atoms -end - --- ════════════════════════════════════════════════════════════════════════════ --- Per-atom body scan --- ════════════════════════════════════════════════════════════════════════════ - --- (internal) Count words emitted by the rest of `tok` after a marker call --- (the marker call itself emits 0 words, but the source pattern may bundle the marker with the next instruction on the same line, --- separated by no top-level comma). +-- (internal) Count words emitted by the rest of `tok` after a marker call +-- (the marker call itself emits 0 words, but the source pattern may bundle the marker with the next instruction on the same line, +-- separated by no top-level comma). -- Returns the word count contributed by that rest. -- @param tok string -- @param word_counts table @@ -435,8 +269,15 @@ local function compute_offsets(labels, branches) return results end --- (internal) Build a constant-table entry `{macro_name, enum_name, value}` --- from a BranchOffset. +-- Right-pad `s` with spaces to width `w`. If `s` is already `w` or wider, no padding is added. +-- @param s string +-- @param w integer +-- @return string +local function pad_right(s, w) + return s .. string.rep(" ", math.max(0, w - #s)) +end + +-- (internal) Build a constant-table entry `{macro_name, enum_name, value}` from a BranchOffset. -- @param r BranchOffset -- @return table local function make_offset_const(r) @@ -501,13 +342,28 @@ end local M = {} --- (internal) Process one source: find atoms, scan bodies, write header. +-- Project the pre-scanned SourceScan entries into the {name, body} shape this pass needs. +-- MipsAtom_ entries have kind="atom"; MipsCode code_ entries have kind="raw_atom". +-- @param scan table -- SourceScan from duffle.scan_source +-- @return table[] -- list of {name=, body=} +local function project_atoms(scan) + local out = {} + for _, a in ipairs(scan.atoms) do + out[#out + 1] = { name = a.raw_name, body = a.body } + end + for _, a in ipairs(scan.raw_atoms) do + out[#out + 1] = { name = a.name, body = a.body } + end + return out +end + +-- (internal) Process one source: project atoms from scan, scan bodies, write header. -- Returns the offsets_h path if a header was written, or nil. -- @param ctx PassCtx -- @param src SourceFile -- @return string|nil -- the offsets_h path local function process_source(ctx, src) - local atoms = find_atoms(src.text) + local atoms = project_atoms(src.scan) if #atoms == 0 then return nil end local atoms_data = {} @@ -528,9 +384,9 @@ local function process_source(ctx, src) return out_path end ---- Run the offsets pass. ---- For each source, emits a per-module `.offsets.h` containing `#define _atom_offset_F_T = N` constants for every `atom_offset(F, T)` reference ---- in the source's atoms. +--- Run the offsets pass. +--- For each source, emits a per-module `.offsets.h` containing `#define _atom_offset_F_T = N` constants +--- for every `atom_offset(F, T)` reference in the source's atoms. --- @param ctx PassCtx --- @return PassResult function M.run(ctx) diff --git a/scripts/passes/report.lua b/scripts/passes/report.lua index 31335c7..1ab0b87 100644 --- a/scripts/passes/report.lua +++ b/scripts/passes/report.lua @@ -2,19 +2,14 @@ --- project-wide summary writer. --- --- Two output files per build: ---- - `build/gen/.annotations.txt` — one per source-directory ---- containing atoms; aggregates across all sources in the directory. +--- - `build/gen/.annotations.txt` — one per source-directory containing atoms; aggregates across all sources in the directory. --- - `build/gen/annotation_validation.txt` — the project summary. --- ---- The annotation pass stashes per-MODULE summary entries in ---- `ctx.flags._annot_results` (set by `passes/annotation.lua`). This ---- pass re-validates each source via `annotation.validate()` to get ---- the detailed per-source results needed for the report. The cost is ---- acceptable: `validate()` is fast (~5ms per source) and runs once. +--- The annotation pass stashes per-MODULE summary entries in `ctx.flags._annot_results` (set by `passes/annotation.lua`). +--- This pass re-validates each source via `annotation.validate()` to get the detailed per-source results needed for the report. --- --- **Conventions**: tabs (1/level), EmmyLua annotations, no regex, ---- Lua 5.3 compatible. See ---- `C:\projects\Pikuma\ps1-ai\conductor\code_styleguides\lua.md`. +--- Lua 5.3 compatible. -- ════════════════════════════════════════════════════════════════════════════ -- Module-scope requires + package.path setup @@ -298,7 +293,7 @@ local function render_module_report(dir, sources, results) end -- ════════════════════════════════════════════════════════════════════════════ --- Per-project summary (ported from tape_atom_annotation_pass.lua:1488-1528) +-- Per-project summary -- ════════════════════════════════════════════════════════════════════════════ --- Render the per-project summary (`build/gen/annotation_validation.txt`). diff --git a/scripts/passes/scan_source.lua b/scripts/passes/scan_source.lua new file mode 100644 index 0000000..029f050 --- /dev/null +++ b/scripts/passes/scan_source.lua @@ -0,0 +1,474 @@ +--- passes/scan_source.lua — Source pre-scan pass (the "mega entity" pass). +--- +--- Single source-walk pass that produces the fat `SourceScan` payload consumed by all downstream passes. Walks each `ctx.sources` entry once, +--- extracting every construct type the metaprograms need: +--- +--- MipsAtom_ (kind = "atom", with optional atom_info inner) +--- MipsAtomComp_ (kind = "comp_bare") +--- MipsAtomComp_Proc_ (kind = "comp_proc", body inside last {}) +--- MipsCode code_ (kind = "raw_atom", offsets pass only) +--- typedef Struct_(Binds_X) { fields } +--- #pragma mac_X tape_atom words=N + _Pragma("...") +--- +--- The result is attached to each `src.scan` so downstream passes can read from `src.scan.atoms` / `src.scan.binds` / etc. without re-walking the source. +--- This is the first pass in the dep graph (no deps). +--- Every other pass that reads source structure depends on this one — see `ps1_meta.lua :: PASSES`. +--- +--- **Conventions**: tabs (1/level), EmmyLua annotations, no regex, +--- Lua 5.3 compatible. + +-- Bootstrap: same as entry scripts. See `ps1_meta.lua` for the rationale. +-- Bootstrap: load `scripts/duffle_paths.lua` (sets package.path + package.cpath). +-- Uses `debug.getinfo` to find this file's own directory, so it works +-- both standalone and when require'd from the orchestrator. +local _src = debug.getinfo(1, "S").source:sub(2) +local _dir = _src:match("(.*[/\\])") or "./" +dofile(_dir .. "../duffle_paths.lua") +local duffle = require("duffle") + +-- ════════════════════════════════════════════════════════════════════════════ +-- Type declarations +-- ════════════════════════════════════════════════════════════════════════════ + +--- @class SourceScan +--- @field atoms AtomEntry[] -- MipsAtom_ + MipsAtomComp_ + MipsAtomComp_Proc_ +--- @field raw_atoms AtomEntry[] -- MipsCode code_ { body } (offsets pass only) +--- @field binds BindsEntry[] -- typedef Struct_(Binds_X) { fields } (fields pre-parsed) +--- @field atom_infos AtomInfoEntry[] -- MipsAtom_(name) atom_info(...) (sub-calls pre-parsed) +--- @field macros MacroEntry[] -- #pragma mac_X tape_atom words=N + _Pragma("...") +--- @field line_of fun(pos: integer): integer -- shared LineIndex closure + +--- @class SourceFile +--- @field path string -- absolute path to the source file +--- @field text string -- the full source text +--- @field dir string -- the directory containing the source +--- @field basename string -- filename without extension +--- @field scan table -- pre-scanned SourceScan payload (set by this pass) + +--- @class PassCtx +--- @field sources SourceFile[] +--- @field metadata_path string +--- @field shared table +--- @field out_root string +--- @field project_root string +--- @field upstream table +--- @field flags table +--- @field dry_run boolean +--- @field verbose boolean + +--- @class PassResult +--- @field outputs table[] +--- @field errors table[] +--- @field warnings table[] + +--- @class AtomEntry +--- @field line integer +--- @field name string -- atom name (for components: without ac_ prefix) +--- @field body string -- brace-delimited body (without the braces) +--- @field body_off integer -- char offset of body[1] in source +--- @field kind string -- "atom" | "comp_bare" | "comp_proc" | "raw_atom" +--- @field raw_name string -- un-stripped name (for components: with ac_ prefix) +--- @field ident_pos integer -- position of the MipsAtom_/MipsAtomComp_ ident start +--- @field after_paren integer -- position past the closing paren +--- @field args string|nil -- populated by components pass (backward lookup) +--- @field comment string|nil -- populated by components pass (backward lookup) + +-- ════════════════════════════════════════════════════════════════════════════ +-- Local helpers +-- ════════════════════════════════════════════════════════════════════════════ + +-- C qualifier keywords that may precede a MipsAtom_ / MipsCode declaration. +-- (typedef is NOT a qualifier here — it's a separate construct (`typedef Struct_(Binds_X) { ... };`) +-- and must be read as an ident so the typedef check below can match it.) +local QUALIFIER_KEYWORDS = { + ["static"] = true, ["const"] = true, ["volatile"] = true, ["extern"] = true, + ["register"] = true, ["auto"] = true, ["inline"] = true, + ["internal"] = true, ["LP_"] = true, ["global"] = true, ["gkknown"] = true, +} + +-- Parse the U4 fields from a Binds_X body. Returns (fields, byte_count). +local function scan_binds_fields(body) + local fields = {} + local byte_off = 0 + local body_pos = 1 + while body_pos <= #body do + body_pos = duffle.skip_ws_and_cmt(body, body_pos) + if body_pos > #body then break end + local type_ident, type_end = duffle.read_ident(body, body_pos) + if not type_ident then + body_pos = body_pos + 1 + elseif type_ident == "U4" then + local field_ident, field_end = duffle.read_ident(body, duffle.skip_ws_and_cmt(body, type_end)) + if field_ident then + fields[#fields + 1] = { name = field_ident, offset = byte_off } + byte_off = byte_off + 4 + end + body_pos = field_end or (type_end + 1) + else + body_pos = type_end + 1 + end + end + return fields, byte_off +end + +-- Parse the register list from inside `atom_reads(...)` or `atom_writes(...)`. +local function scan_reg_list(sub_inner) + local regs = {} + local sub_inner_pos = 1 + while sub_inner_pos <= #sub_inner do + sub_inner_pos = duffle.skip_ws_and_cmt(sub_inner, sub_inner_pos) + if sub_inner_pos > #sub_inner then break end + local reg_ident, reg_end = duffle.read_ident(sub_inner, sub_inner_pos) + if reg_ident then + regs[#regs + 1] = duffle.trim(reg_ident) + sub_inner_pos = reg_end + else + sub_inner_pos = sub_inner_pos + 1 + end + if sub_inner_pos > #sub_inner then break end + if sub_inner:sub(sub_inner_pos, sub_inner_pos) == "," then sub_inner_pos = sub_inner_pos + 1 end + end + return regs +end + +-- Parse the sub-calls inside `atom_info(atom_bind(...), atom_reads(...), atom_writes(...))`. +-- Returns (binds, reads, writes). +local function scan_atom_info_subcalls(info_inner) + local binds, reads, writes = nil, nil, nil + local sub_pos = 1 + while sub_pos <= #info_inner do + sub_pos = duffle.skip_ws_and_cmt(info_inner, sub_pos) + if sub_pos > #info_inner then break end + local sub_ident, sub_end = duffle.read_ident(info_inner, sub_pos) + if not sub_ident then + sub_pos = sub_pos + 1 + elseif sub_ident == "atom_bind" then + local sub_open = duffle.skip_ws_and_cmt(info_inner, sub_end) + if info_inner:sub(sub_open, sub_open) == "(" then + local sub_inner, sub_after2 = duffle.read_parens(info_inner, sub_open) + -- scan: atom_bind() + binds = duffle.trim(sub_inner) + sub_pos = sub_after2 + else + sub_pos = sub_open + 1 + end + elseif sub_ident == "atom_reads" or sub_ident == "atom_writes" then + local kind = sub_ident + local sub_open = duffle.skip_ws_and_cmt(info_inner, sub_end) + if info_inner:sub(sub_open, sub_open) == "(" then + local sub_inner, sub_after2 = duffle.read_parens(info_inner, sub_open) + -- scan: atom_reads() OR atom_writes() + local regs = scan_reg_list(sub_inner) + if kind == "atom_reads" then reads = regs else writes = regs end + sub_pos = sub_after2 + else + sub_pos = sub_open + 1 + end + else + sub_pos = sub_end + end + end + return binds, reads, writes +end + +-- Skip C qualifier keywords and return the position past the last one. +local function scan_skip_qualifiers(source, pos) + while true do + pos = duffle.skip_ws_and_cmt(source, pos) + local ident, after = duffle.read_ident(source, pos) + if not ident then return pos end + if QUALIFIER_KEYWORDS[ident] then pos = after else return pos end + end +end + +-- ════════════════════════════════════════════════════════════════════════════ +-- The single source walker +-- ════════════════════════════════════════════════════════════════════════════ + +--- Single-pass source scan. Walks the source ONCE and extracts every construct type the metaprogram passes need. +--- Returns a fat SourceScan table. Each pass filters from this payload instead of re-walking the source. +--- @param source string +--- @return table -- SourceScan { atoms, raw_atoms, binds, atom_infos, macros, line_of } +local function scan_source(source) + local line_of = duffle.LineIndex(source) + local atoms = {} + local raw_atoms = {} + local binds = {} + local atom_infos = {} + local macros = {} + local pos = 1 + local src_len = #source + + while pos <= src_len do + pos = duffle.skip_ws_and_cmt(source, pos) + if pos > src_len then break end + + -- Skip preprocessor directives (#define / #include / #pragma / etc). + -- _Pragma is an operator (not a directive) — it doesn't start with #. + local pp_pos = duffle.skip_preprocessor_line(source, pos) + if pp_pos then pos = pp_pos; goto continue end + + -- Skip C qualifiers (static, const, etc.) that may precede a declaration. + pos = scan_skip_qualifiers(source, pos) + if pos > src_len then break end + + local ident, ident_end = duffle.read_ident(source, pos) + -- scan: + if not ident then pos = pos + 1; goto continue end + + -- ── MipsAtom_ / MipsAtomComp_ / MipsAtomComp_Proc_ ── + if ident == "MipsAtom_" or ident == "MipsAtomComp_" or ident == "MipsAtomComp_Proc_" then + local is_atom = ident == "MipsAtom_" + local is_comp = ident == "MipsAtomComp_" + local is_proc = ident == "MipsAtomComp_Proc_" + local kind = is_atom and "atom" or (is_comp and "comp_bare" or "comp_proc") + local open_paren = duffle.skip_ws_and_cmt(source, ident_end) + if source:sub(open_paren, open_paren) ~= "(" then pos = open_paren + 1; goto continue end + + local inner, after_paren = duffle.read_parens(source, open_paren) + -- scan: () + + if is_proc then + -- MipsAtomComp_Proc_(name, { body }) — body is inside the LAST { } in args. + local last_brace_pos + for search_pos = #inner, 1, -1 do + if inner:sub(search_pos, search_pos) == "{" then last_brace_pos = search_pos; break end + end + if last_brace_pos then + local depth = 1 + local inner_pos = last_brace_pos + 1 + while inner_pos <= #inner and depth > 0 do + local c = inner:byte(inner_pos) + if c == 123 then depth = depth + 1; inner_pos = inner_pos + 1 + elseif c == 125 then depth = depth - 1; if depth == 0 then break end; inner_pos = inner_pos + 1 + elseif c == 40 then local _, a = duffle.read_parens(inner, inner_pos); inner_pos = a + elseif c == 91 then local _, a = duffle.read_brackets(inner, inner_pos); inner_pos = a + elseif c == 34 or c == 39 then inner_pos = duffle.skip_str_or_cmt(inner, inner_pos) + 1 + else inner_pos = inner_pos + 1 end + end + if depth == 0 then + -- scan: (, { }) + local name_match = inner:match("^%s*([%w_]+)") + local raw_name = name_match or "?" + -- Strip "ac_" prefix for component names (components pass convention). + local name = raw_name + if #raw_name > 3 and raw_name:sub(1, 3) == "ac_" then + name = raw_name:sub(4) + end + local body = inner:sub(last_brace_pos + 1, inner_pos - 1) + local body_off = open_paren + 1 + last_brace_pos + atoms[#atoms + 1] = { + line = line_of(pos), name = name, body = body, body_off = body_off + 1, + kind = kind, raw_name = raw_name, + ident_pos = pos, after_paren = after_paren, + args = nil, comment = nil, + } + end + end + pos = after_paren + else + -- MipsAtom_(name) { body } OR MipsAtomComp_(name) { body } + local name_start = 1 + while name_start <= #inner and inner:sub(name_start, name_start):match("[%s]") do name_start = name_start + 1 end + local name_end = name_start + while name_end <= #inner and inner:sub(name_end, name_end):match("[%w_]") do name_end = name_end + 1 end + local raw_name = inner:sub(name_start, name_end - 1) + -- scan: () + if raw_name ~= "" then + local brace = duffle.scan_to_char(source, "{", after_paren) + -- scan: () { + if brace then + local body, after_brace = duffle.read_braces(source, brace) + -- scan: () { } + -- Strip "ac_" prefix for component names (components pass convention). + local disp_name = raw_name + if is_comp and #raw_name > 3 and raw_name:sub(1, 3) == "ac_" then + disp_name = raw_name:sub(4) + end + atoms[#atoms + 1] = { + line = line_of(pos), name = disp_name, body = body, body_off = brace + 1, + kind = kind, raw_name = raw_name, + ident_pos = pos, after_paren = after_paren, + args = nil, comment = nil, + } + pos = after_brace + else + pos = open_paren + 1 + end + else + pos = open_paren + 1 + end + end + + -- For MipsAtom_ entries: check if atom_info(...) follows. + if is_atom then + local lookahead = duffle.skip_ws_and_cmt(source, after_paren) + local look_ident, look_end = duffle.read_ident(source, lookahead) + -- scan: MipsAtom_() + if look_ident == "atom_info" then + local info_open = duffle.skip_ws_and_cmt(source, look_end) + if source:sub(info_open, info_open) == "(" then + local info_inner, info_after = duffle.read_parens(source, info_open) + -- scan: MipsAtom_() atom_info(, , ) + -- Find the atom name from the just-parsed atom entry (last one added). + local last_atom = atoms[#atoms] + local atom_name = last_atom and last_atom.raw_name or "?" + local ai_binds, ai_reads, ai_writes = scan_atom_info_subcalls(info_inner) + atom_infos[#atom_infos + 1] = { + atom_name = atom_name, binds = ai_binds, + reads = ai_reads or {}, writes = ai_writes or {}, + info_line = line_of(lookahead), + } + -- Don't advance pos past info_after — the body { ... } still needs to be skipped + -- by the brace scan below. But if there's no body (forward decl), advance. + local body_brace = duffle.scan_to_char(source, "{", info_after) + if body_brace then + local _, after_body = duffle.read_braces(source, body_brace) + pos = after_body + else + pos = info_after + end + end + end + end + goto continue + end + + -- ── MipsCode code_ { body } (raw atom form — offsets pass only) ── + if ident == "MipsCode" then + local next_pos = duffle.skip_ws_and_cmt(source, ident_end) + local next_ident, next_after = duffle.read_ident(source, next_pos) + -- scan: MipsCode + if next_ident and #next_ident > 5 and next_ident:sub(1, 5) == "code_" then + local atom_name = next_ident:sub(6) + -- scan: MipsCode code_ + local brace_pos = duffle.scan_to_char(source, "{", next_after) + -- scan: MipsCode code_ { + if brace_pos then + local body, after_brace = duffle.read_braces(source, brace_pos) + -- scan: MipsCode code_ { } + raw_atoms[#raw_atoms + 1] = { + line = line_of(pos), name = atom_name, body = body, body_off = brace_pos + 1, + kind = "raw_atom", raw_name = atom_name, + } + pos = after_brace + goto continue + end + end + pos = ident_end + goto continue + end + + -- ── typedef Struct_(Binds_X) { fields } ── + if ident == "typedef" then + local after_typedef = duffle.skip_ws_and_cmt(source, ident_end) + local id2, id2_end = duffle.read_ident(source, after_typedef) + -- scan: typedef + if id2 == "Struct_" then + local open_paren = duffle.skip_ws_and_cmt(source, id2_end) + if source:sub(open_paren, open_paren) == "(" then + local inner, after_paren = duffle.read_parens(source, open_paren) + -- scan: typedef Struct_() + local name = duffle.trim(inner) + local brace = duffle.scan_to_char(source, "{", after_paren) + -- scan: typedef Struct_() { + if brace then + local body, after_brace = duffle.read_braces(source, brace) + -- scan: typedef Struct_() { } + if name:sub(1, 6) == "Binds_" then + local fields, byte_off = scan_binds_fields(body) + binds[#binds + 1] = { line = line_of(pos), name = name, fields = fields, bytes = byte_off } + end + pos = after_brace + goto continue + end + pos = open_paren + 1 + goto continue + end + pos = id2_end or (after_typedef + 1) + goto continue + end + pos = ident_end + goto continue + end + + -- ── _Pragma("mac_X tape_atom words=N") (operator form) ── + if ident == "_Pragma" then + local open_paren = duffle.skip_ws_and_cmt(source, ident_end) + if source:sub(open_paren, open_paren) == "(" then + local str, str_end = duffle.read_parens(source, open_paren) + -- scan: _Pragma() + str = duffle.trim(str) + if str:sub(1, 1) == '"' and str:sub(-1) == '"' then + local inner = str:sub(2, -2) + local space = duffle.find_byte(inner, 32, 1) + if space then + local name = inner:sub(1, space - 1) + local rest = inner:sub(space + 1) + local eq = duffle.find_byte(rest, 61, 1) + if eq then + local key = duffle.trim(rest:sub(1, eq - 1)) + local val = duffle.trim(rest:sub(eq + 1)) + if key == "tape_atom words" or key == "words" then + macros[#macros + 1] = { line = line_of(pos), name = name, words = tonumber(val) or 0 } + end + end + end + end + pos = str_end + goto continue + end + pos = open_paren + 1 + goto continue + end + + -- ── #pragma mac_X tape_atom words=N (directive form) ── + -- (preprocessor skip above handles # lines, but pragma is an ident here + -- only if it appeared without a leading # — which happens when the + -- preprocessor skip didn't fire because the # was on a previous line. + -- The annotation pass handles this via its own skip_preprocessor_line, + -- but scan_source handles it here by checking the ident.) + if ident == "pragma" then + -- This shouldn't normally fire — #pragma lines are skipped by + -- skip_preprocessor_line above. If we get here, it's a _Pragma + -- variant or a non-#-prefixed pragma. Just advance. + pos = ident_end + goto continue + end + + -- ── Unrecognized ident — advance past it ── + pos = ident_end + + ::continue:: + end + + return { + atoms = atoms, + raw_atoms = raw_atoms, + binds = binds, + atom_infos = atom_infos, + macros = macros, + line_of = line_of, + } +end + +-- ════════════════════════════════════════════════════════════════════════════ +-- M — module exports +-- ════════════════════════════════════════════════════════════════════════════ + +--- @class M + +local M = {} + +--- Walk each source once and attach the fat SourceScan payload to `src.scan`. +--- No output files; this is a pure in-memory pre-processing pass. +--- @param ctx PassCtx +--- @return PassResult +function M.run(ctx) + for _, src in ipairs(ctx.sources) do + src.scan = scan_source(src.text) + end + return { outputs = {}, errors = {}, warnings = {} } +end + +return M diff --git a/scripts/passes/static_analysis.lua b/scripts/passes/static_analysis.lua index 3838926..3ca711d 100644 --- a/scripts/passes/static_analysis.lua +++ b/scripts/passes/static_analysis.lua @@ -115,152 +115,21 @@ local OUTPUT_EXTENSION = ".static_analysis.txt" --- @field total_cycles integer -- sum of token cycle costs -- ════════════════════════════════════════════════════════════════════════════ --- Source walkers +-- Source scanning — delegated to duffle.scan_source (ps1_meta.lua pre-scans) -- ════════════════════════════════════════════════════════════════════════════ - --- Parse `MipsAtomComp_Proc_(name, { body })` — the body is inside the LAST `{ ... }` in the args. --- Returns (atom_entry, new_pos) or (nil, fallback_pos). -local function parse_comp_proc_body(inner, line_of, pos, open_paren, after_paren) - -- Find the last `{` in `inner`, then the matching `}`. - local last_brace_pos - for search_pos = #inner, 1, -1 do - if inner:sub(search_pos, search_pos) == "{" then last_brace_pos = search_pos; break end - end - if not last_brace_pos then return nil, open_paren + 1 end - -- Walk forward to find matching `}` honoring balanced ()/[] and strings. - local depth = 1 - local inner_pos = last_brace_pos + 1 - while inner_pos <= #inner and depth > 0 do - local c = inner:byte(inner_pos) - if c == 123 then - depth = depth + 1; inner_pos = inner_pos + 1 - elseif c == 125 then - depth = depth - 1 - if depth == 0 then break end - inner_pos = inner_pos + 1 - elseif c == 40 then - local _, a = duffle.read_parens(inner, inner_pos); inner_pos = a - elseif c == 91 then - local _, a = duffle.read_brackets(inner, inner_pos); inner_pos = a - elseif c == 34 or c == 39 then - inner_pos = duffle.skip_str_or_cmt(inner, inner_pos) + 1 - else - inner_pos = inner_pos + 1 - end - end - if depth ~= 0 then return nil, open_paren + 1 end - -- scan: (, { }) - local name_match = inner:match("^%s*([%w_]+)") - local name = name_match or "?" - local body = inner:sub(last_brace_pos + 1, inner_pos - 1) - local body_off = open_paren + 1 + last_brace_pos - return { - line = line_of(pos), name = name, body = body, body_off = body_off + 1, kind = "comp_proc", - }, after_paren -end - --- Parse `MipsAtom_(name) { body }` or `MipsAtomComp_(name) { body }` — body is the FIRST `{ ... }` after the parens. --- Returns (atom_entry, new_pos) or (nil, fallback_pos). -local function parse_atom_or_comp_bare_body(inner, source_text, line_of, pos, open_paren, after_paren, kind) - -- Extract the name from the first arg. - local name_start = 1 - while name_start <= #inner and inner:sub(name_start, name_start):match("[%s]") do name_start = name_start + 1 end - local name_end = name_start - while name_end <= #inner and inner:sub(name_end, name_end):match("[%w_]") do name_end = name_end + 1 end - local name = inner:sub(name_start, name_end - 1) - -- scan: () - if name == "" then return nil, open_paren + 1 end - local brace = duffle.scan_to_char(source_text, "{", after_paren) - -- scan: () { - if not brace then return nil, open_paren + 1 end - local body, after_brace = duffle.read_braces(source_text, brace) - -- scan: () { } - return { - line = line_of(pos), name = name, body = body, body_off = brace + 1, kind = kind, - }, after_brace -end - ---- Walk source-as-written, return a list of `{line, name, body, ---- body_off, kind}` for every: ---- `MipsAtom_(name) { body };` -> kind = "atom" (baked atom) ---- `MipsAtomComp_(name) { body };` -> kind = "comp_bare" (static-array component) ---- `MipsAtomComp_Proc_(name, { body })` -> kind = "comp_proc" (procedural component) ---- ---- All three forms are recognized because the user explicitly uses ---- both bare components (e.g. `ac_gte_store_f3_post_rtpt`) and ---- procedural components (e.g. `ac_format_f3_color(r, g, b)`) inside ---- atom bodies. The two component forms generate macro equivalents ---- (in gen/duffle.macs.h) that atoms call via `mac_*` -- so the parent ---- atom body is what needs the GTE pipeline-fill + mac_yield checks. ---- The component bodies themselves don't need mac_yield (control ---- transfer is the parent atom's job) but they DO need pre-fill nops ---- before any gte_cmdw_X they contain. ---- ---- Comments / strings inside `name` and `body` are tolerated; `body` ---- is the raw brace inner text (with surrounding whitespace, no ---- leading/trailing `{` `}`). `body_off` is the character offset of ---- `body[1]` in `source_text`, used to compute per-token line numbers ---- later. -local function find_atom_bodies(source_text) - local line_of = duffle.LineIndex(source_text) - local out = {} - local src_len = #source_text - local pos = 1 - while pos <= src_len do - pos = duffle.skip_ws_and_cmt(source_text, pos) - if pos > src_len then break end - - -- Skip preprocessor directives (#define / #include / #pragma / etc). - local pp_pos = duffle.skip_preprocessor_line(source_text, pos) - if pp_pos then pos = pp_pos; goto continue end - - local ident, ident_end = duffle.read_ident(source_text, pos) - -- scan: - if not ident then - pos = pos + 1; goto continue - end - - local is_atom = ident == "MipsAtom_" - local is_comp = ident == "MipsAtomComp_" - local is_proc = ident == "MipsAtomComp_Proc_" - if not is_atom and not is_comp and not is_proc then - pos = ident_end; goto continue - end - - local kind = is_atom and "atom" or (is_comp and "comp_bare" or "comp_proc") - local open_paren = duffle.skip_ws_and_cmt(source_text, ident_end) - if source_text:sub(open_paren, open_paren) ~= "(" then - pos = open_paren + 1; goto continue - end - - local inner, after_paren = duffle.read_parens(source_text, open_paren) - -- scan: () - - local entry, new_pos - if is_proc then - entry, new_pos = parse_comp_proc_body(inner, line_of, pos, open_paren, after_paren) - else - entry, new_pos = parse_atom_or_comp_bare_body(inner, source_text, line_of, pos, open_paren, after_paren, kind) - end - if entry then out[#out + 1] = entry end - pos = new_pos - - ::continue:: - end - return out -end +-- +-- The orchestrator calls duffle.scan_source once per source and stashes the fat SourceScan in src.scan. +-- validate() below reads from src.scan — no source walking in this pass. -- ════════════════════════════════════════════════════════════════════════════ -- Body tokenizer (top-level comma splitter + per-token classification) -- ════════════════════════════════════════════════════════════════════════════ --- Build a map: `body_relative_char_offset` -> `body_relative_line`. ---- Used by the checks to convert per-token offsets in the body to line ---- numbers relative to the start of `body`. The atom's source-line of ---- the body-start is added by the caller. +--- Used by the checks to convert per-token offsets in the body to lina numbers relative to the start of `body`. +--- The atom's source-line of the body-start is added by the caller. --- ---- Simple line-counting: count `\n` chars from offset 1 up to the ---- offset; that count + 1 is the line number (1-based). +--- Simple line-counting: count `\n` chars from offset 1 up to the offset; that count + 1 is the line number (1-based). local function build_body_line_index(body) local index = {} local len = #body @@ -279,19 +148,17 @@ local function build_body_line_index(body) end --- Count of COP2-nop words contributed by a single top-level token. --- `nop` -> 1 --- `nop2` -> 2 (i.e. `nop, nop` baked into one asm word) --- `nop,` / `nop2,` -> same as above; strip trailing comma defensively --- anything else -> 0 +-- `nop` -> 1 +-- `nop2` -> 2 (i.e. `nop, nop` baked into one asm word) +-- `nop,` / `nop2,` -> same as above; strip trailing comma defensively +-- anything else -> 0 -- --- (Branch-delay-slot nops like `branch_*(..., nop)` are tokenized --- separately by split_top_level_commas: the branch arg ends before --- the trailing comma, and `nop` becomes its own token. So no special --- handling is needed here.) +-- (Branch-delay-slot nops like `branch_*(..., nop)` are tokenized separately by split_top_level_commas: +-- The branch arg ends before the trailing comma, and `nop` becomes its own token. +-- So no special handling is needed here.) local function nop_word_count(token) local s = duffle.trim(token) - -- strip trailing comma(s) (defensive against raw text via, but our - -- tokenize_body already strips them; this is a safety net) + -- Strip trailing comma(s) (defensive against raw text via, but our tokenize_body already strips them; this is a safety net) s = s:gsub(",$", "") s = duffle.trim(s) if s == "nop" then return 1 end @@ -300,10 +167,9 @@ local function nop_word_count(token) end --- Tokenize the body inner-text into a flat list of `(token, body_rel_offset)` ---- pairs (nested parens/braces/brackets are honored; comments and strings ---- are skipped). `body_rel_offset` is the char offset within `body` of the ---- start of the token — callers add it to the atom's `body_off` to get ---- an absolute source position for line tracking. +--- pairs (nested parens/braces/brackets are honored; comments and strings are skipped). +--- `body_rel_offset` is the char offset within `body` of the start of the token. +--- Callers add it to the atom's `body_off` to get an absolute source position for line tracking. local function tokenize_body(body) local out = {} local len = #body @@ -316,24 +182,20 @@ local function tokenize_body(body) end if rel > len then break end - -- Find comma/newline/semicolon after this token. Read balanced - -- groups so commas inside parens/braces/brackets aren't treated - -- as separators. Comments / strings are skipped. + -- Find comma/newline/semicolon after this token. + -- Read balanced groups so commas inside parens/braces/brackets aren't treated as separators. + -- Comments / strings are skipped. local scan = rel while scan <= len do local c = body:byte(scan) - if c == 44 then break end -- ',' - if c == 10 then break end -- '\n' - if c == 59 then break end -- ';' - if c == 40 then -- '(' - local _, a = duffle.read_parens(body, scan); scan = a - elseif c == 123 then -- '{' - local _, a = duffle.read_braces(body, scan); scan = a - elseif c == 91 then -- '[' - local _, a = duffle.read_brackets(body, scan); scan = a - elseif c == 34 or c == 39 then -- '"' or '\'' - scan = duffle.skip_str_or_cmt(body, scan) + 1 - else + if c == 44 then break end -- ',' + if c == 10 then break end -- '\n' + if c == 59 then break end -- ';' + if c == 40 then local _, a = duffle.read_parens (body, scan); scan = a -- '(' + elseif c == 123 then local _, a = duffle.read_braces (body, scan); scan = a -- '{' + elseif c == 91 then local _, a = duffle.read_brackets (body, scan); scan = a -- '[' + elseif c == 34 or c == 39 then scan = duffle.skip_str_or_cmt(body, scan) + 1 -- '"' or '\'' + else scan = scan + 1 end end @@ -373,8 +235,8 @@ local function count_preceding_nops(tokens, ti) return have end --- Check a single gte_cmdw_* token for pipeline-fill compliance. Emits a finding if the --- preceding nops are insufficient (error) or the macro isn't in the latency table (warning). +-- Check a single gte_cmdw_* token for pipeline-fill compliance. +-- Emits a finding if the preceding nops are insufficient (error) or the macro isn't in the latency table (warning). -- scan: ... -> validate nop count vs GTE_PIPELINE_LATENCY[X] local function check_one_gte_cmdw(a, tok, tokens, ti, line_in_body, findings) local cmdw_full = tok:match("^(gte_cmdw_[%w_]+)%s*[,%)]") or tok:match("^(gte_cmdw_[%w_]+)%s*$") @@ -411,12 +273,10 @@ local function check_one_gte_cmdw(a, tok, tokens, ti, line_in_body, findings) end end ---- Walk the token list. Whenever we hit a `gte_cmdw_` token, count ---- consecutive nop words immediately preceding it. If count < the ---- minimum declared in `duffle.GTE_PIPELINE_LATENCY[X]`, record a finding. ---- Aliases are resolved against the lookup table directly; if a macro name is not ---- in the table, emit a soft warning (the user might have added a new ---- gte_cmdw_* but not updated duffle.lua). +--- Walk the token list. Whenever we hit a `gte_cmdw_` token, count consecutive nop words immediately preceding it. +--- If count < the minimum declared in `duffle.GTE_PIPELINE_LATENCY[X]`, record a finding. +--- Aliases are resolved against the lookup table directly; if a macro name is not in the table, emit a soft warning +--- (the user might have added a new gte_cmdw_* but not updated duffle.lua). local function check_gte_pipeline_fill(atoms, findings, line_of) for _, a in ipairs(atoms) do local tokens = tokenize_body(a.body) @@ -434,27 +294,20 @@ end -- Check #2: mac_yield uniformity -- ════════════════════════════════════════════════════════════════════════════ ---- Every atom body must contain exactly one `mac_yield()` call and it ---- must be the LAST top-level token in the body (so the tape runtime ---- can pick up cleanly at the next atom's bound registers). +--- Every atom body must contain exactly one `mac_yield()` call and it must be the LAST top-level token in the body +--- (so the tape runtime can pick up cleanly at the next atom's bound registers). --- ---- Empty bodies are not currently flagged — runtime infrastructure ---- atoms like `MipsAtom_(yield) { mac_yield() }` and `MipsAtom_(tape_exit) ---- { jump_reg(rret_addr), nop }` are valid as-is; mac_yield at the end ---- is the contract. +--- Empty bodies are not currently flagged — runtime infrastructure atoms like `MipsAtom_(yield) { mac_yield() }` +--- and `MipsAtom_(tape_exit) { jump_reg(rret_addr), nop }` are valid as-is; mac_yield at the end is the contract. local function check_mac_yield_uniformity(atoms, findings) -- Per-kind semantics: - -- MipsAtom_ (baked atom): exactly 1 mac_yield at the end of - -- the body. Control transfer is the atom's job. + -- MipsAtom_ (baked atom): exactly 1 mac_yield at the end of the body. Control transfer is the atom's job. -- MipsAtomComp_ (bare static-array component): ZERO mac_yield. - -- The component is invoked from inside an atom - -- body; the parent atom does the yield. + -- The component is invoked from inside an atom body; the parent atom does the yield. -- MipsAtomComp_Proc_ (procedural component): ZERO mac_yield. - -- Same reasoning -- it's a function returning - -- a MipsAtom slice, invoked from a parent atom. + -- Same reasoning -- it's a function returning a MipsAtom slice, invoked from a parent atom. -- - -- The GTE pipeline-fill check applies to all 3 kinds (see - -- check_gte_pipeline_fill). Only the mac_yield rule branches on kind. + -- The GTE pipeline-fill check applies to all 3 kinds (see check_gte_pipeline_fill). Only the mac_yield rule branches on kind. for _, a in ipairs(atoms) do local tokens = tokenize_body(a.body) local line_in_body = build_body_line_index(a.body) @@ -497,9 +350,8 @@ local function check_mac_yield_uniformity(atoms, findings) a.name, line_for(last_idx), count), } elseif last_idx < #tokens then - -- 1 call, but not the last token. We DON'T fail if the - -- post-token is just `nop` or `nop2` or a branch with `, nop` - -- delay slot -- it's the standard "yield, then BD nop" idiom. + -- 1 call, but not the last token. We DON'T fail if the post-token is just `nop` or `nop2` or a branch with `, nop` delay slot. + -- It's the standard "yield, then BD nop" idiom. local post_non_nop = false for search_idx = last_idx + 1, #tokens do local t = tokens[search_idx].tok @@ -522,10 +374,10 @@ local function check_mac_yield_uniformity(atoms, findings) end end else - -- Component (comp_bare or comp_proc): ZERO yields. The parent - -- atom does the yield. A yield inside a component would either - -- be dead code (bare) or prematurely terminate the function - -- (proc). Both are bugs. + -- Component (comp_bare or comp_proc): ZERO yields. + -- The parent atom does the yield. + -- A yield inside a component would either be dead code (bare) or prematurely terminate the function (proc). + -- Both are bugs. if count > 0 then findings[#findings + 1] = { atom = a.name, @@ -570,12 +422,9 @@ local function parse_binds_fields(body) return fields, byte_off end ---- Walk source-as-written, return a list of `{line, name, fields, bytes}` ---- for every `typedef Struct_(Binds_X) { ... };` declaration. Only U4 ---- fields are tracked (Binds_* are always word arrays in this codebase -- ---- pointers stored as U4, indices as U4, etc.). Mirrors annotation.lua :: ---- find_binds_structs but is independent (no shared cross-pass state for ---- static-analysis; each pass re-walks source). +--- Walk source-as-written, return a list of `{line, name, fields, bytes}` for every `typedef Struct_(Binds_X) { ... };` declaration. +--- Only U4 fields are tracked (Binds_* are always word arrays in this codebase pointers stored as U4, indices as U4, etc.). +--- Mirrors annotation.lua :: find_binds_structs but is independent (no shared cross-pass state for static-analysis; each pass re-walks source). local function find_binds_structs(source_text) local line_of = duffle.LineIndex(source_text) local out = {} @@ -681,11 +530,8 @@ local function parse_atom_info_subcalls(info_inner) return binds, reads, writes end ---- For every `MipsAtom_(name)` followed by `atom_info(...)`, parse the ---- atom_info's `atom_bind(Binds_X)`, `atom_reads(...)`, and ---- `atom_writes(...)` sub-calls. Returns a list of ---- `{atom_name, binds, reads, writes, info_line}` for use by ---- check_abi_handoff. +--- For every `MipsAtom_(name)` followed by `atom_info(...)`, parse the atom_info's `atom_bind(Binds_X)`, `atom_reads(...)`, and `atom_writes(...)` sub-calls. +--- Returns a list of `{atom_name, binds, reads, writes, info_line}` for use by check_abi_handoff. local function find_atom_info(source_text) local line_of = duffle.LineIndex(source_text) local out = {} @@ -741,18 +587,17 @@ end -- Check #3: ABI handoff discipline -- ════════════════════════════════════════════════════════════════════════════ ---- For every atom with `atom_bind(Binds_X)`, verify the atom body loads ---- every field of `Binds_X` from R_TapePtr (in declaration order) and ---- advances R_TapePtr by S_(Binds_X) at the end. Mismatches are errors. +--- For every atom with `atom_bind(Binds_X)`, verify the atom body reads every field of `Binds_X` from R_TapePtr (in any order) +--- and advances R_TapePtr by S_(Binds_X) at the end. Mismatches are errors. +--- +--- This is the "job boundary sanity check": Binds_X is the atom's input payload (like a C function's argument struct). +--- The body must read each input field and advance the input cursor past the payload. +--- The order of reads doesn't matter — each field is at a different offset in the struct, and the advance at the end is what keeps the tape pointer in sync. +--- --- Rules: ---- 1. Body MUST contain one `load_word(R_*, R_TapePtr, O_(Binds_X, field))` ---- per field of Binds_X. Missing field = error. ---- 2. Body's load_words to R_TapePtr at O_(Binds_X, field) MUST appear ---- in the same order as the fields are declared in Binds_X. ---- Out-of-order load = error. ---- 3. Body MUST contain an `add_ui_self(R_TapePtr, S_(Binds_X))` (or ---- equivalent advance by the struct's byte count). Missing = error. ---- 4. atom_bind(Binds_X) where Binds_X doesn't exist = error. +--- 1. Body MUST contain one `load_word(R_*, R_TapePtr, O_(Binds_X, field))` per field of Binds_X. Missing field = error. +--- 2. Body MUST contain an `add_ui_self(R_TapePtr, S_(Binds_X))` (or equivalent advance by the struct's byte count). Missing = error. +--- 3. atom_bind(Binds_X) where Binds_X doesn't exist = error. local function check_abi_handoff(atoms, atom_infos, binds_index, findings) local info_by_atom = {} for _, info in ipairs(atom_infos) do @@ -774,9 +619,6 @@ local function check_abi_handoff(atoms, atom_infos, binds_index, findings) else local tokens = tokenize_body(a.body) local line_in_body = build_body_line_index(a.body) - local expected_field_seq = {} - for _, f in ipairs(binds.fields) do expected_field_seq[#expected_field_seq + 1] = f.name end - local found_field_seq = {} local found_field_set = {} local found_advance = false local bind_re = "O_%(" .. binds_name .. ",%s*([%w_]+)%s*%)" @@ -787,8 +629,7 @@ local function check_abi_handoff(atoms, atom_infos, binds_index, findings) local field = tok:match(bind_re) -- scan: load_word(R_*, R_TapePtr, O_(, )) if field then - found_field_seq[#found_field_seq + 1] = field - found_field_set[field] = true + found_field_set[field] = true else local body_line = a.line + line_in_body[t.rel] findings[#findings + 1] = { @@ -807,33 +648,17 @@ local function check_abi_handoff(atoms, atom_infos, binds_index, findings) end end - for _, fname in ipairs(expected_field_seq) do - if not found_field_set[fname] then + for _, f in ipairs(binds.fields) do + if not found_field_set[f.name] then findings[#findings + 1] = { atom = a.name, line = a.line, check = "abi_handoff", kind = "error", msg = string.format("%s at line %d binds %s but never loads field `%s` from R_TapePtr (expected O_(%s, %s))", - a.name, a.line, binds_name, fname, binds_name, fname), + a.name, a.line, binds_name, f.name, binds_name, f.name), } end end - if #found_field_seq == #expected_field_seq then - for field_idx = 1, #expected_field_seq do - if found_field_seq[field_idx] ~= expected_field_seq[field_idx] then - findings[#findings + 1] = { - atom = a.name, line = a.line, - check = "abi_handoff", kind = "error", - msg = string.format("%s at line %d loads fields in wrong order: got [%s], expected [%s]", - a.name, a.line, - table.concat(found_field_seq, ", "), - table.concat(expected_field_seq, ", ")), - } - break - end - end - end - if not found_advance then findings[#findings + 1] = { atom = a.name, line = a.line, @@ -851,21 +676,16 @@ end -- Check #4: GPU port-store shape -- ════════════════════════════════════════════════════════════════════════════ ---- For every baked atom body, detect which GP0 primitive it's emitting ---- (first `mac_format__color` call). Sum contributions from ---- `mac_format_X_color` + `mac_gte_store_X_post_*` + `mac_insert_ot_tag_X`. +--- For every baked atom body, detect which GP0 primitive it's emitting +--- (first `mac_format__color` call). Sum contributions from `mac_format_X_color` + `mac_gte_store_X_post_*` + `mac_insert_ot_tag_X`. --- Compare to duffle.GP0_CMD_SIZE[cmd_byte]. Mismatch = error. --- --- Soft behavior (warnings): ---- - Atoms emitting a primitive via raw `store_word(R_PrimCursor, ...)` ---- (no `mac_format_X_color` call) emit a "manual packet assembly" ---- advisory. Cannot auto-validate. ---- - Atoms containing a `mac_(...)` call whose name is not in ---- duffle.GP0_MACRO_CONTRIB emit a "new macro; update duffle.GP0_MACRO_CONTRIB" ---- advisory. +--- - Atoms emitting a primitive via raw `store_word(R_PrimCursor, ...)` (no `mac_format_X_color` call) emit a "manual packet assembly" advisory. +--- Cannot auto-validate. +--- - Atoms containing a `mac_(...)` call whose name is not in duffle.GP0_MACRO_CONTRIB emit a "new macro; update duffle.GP0_MACRO_CONTRIB" advisory. --- ---- Applies only to `kind = "atom"` (baked atoms). Components don't ---- emit full primitives. +--- Applies only to `kind = "atom"` (baked atoms). Components don't emit full primitives. local function check_gpu_portstore_shape(atoms, findings) for _, a in ipairs(atoms) do if a.kind == "atom" then @@ -938,13 +758,10 @@ end --- Compute the cycle cost of one token. The token is a string like --- `add_ui(R_T0, R_T1, 4)` or `nop2` or `gte_cmdw_rtpt`. Returns: ---- cycles - integer cycle cost (from duffle.INSTRUCTION_LATENCY, or ---- duffle.UNKNOWN_INSTRUCTION_CYCLES if not in the table) ---- macro_name - the bare ident (e.g. `add_ui`, `gte_cmdw_rtpt`, ---- `nop2`, `mac_yield`) ---- unknown - true iff the macro wasn't in duffle.INSTRUCTION_LATENCY ---- The function strips trailing `()` from function-call style macros ---- so `mac_yield()` and `mac_yield` resolve identically. +--- cycles - integer cycle cost (from duffle.INSTRUCTION_LATENCY, or duffle.UNKNOWN_INSTRUCTION_CYCLES if not in the table) +--- macro_name - the bare ident (e.g. `add_ui`, `gte_cmdw_rtpt`, `nop2`, `mac_yield`) +--- unknown - true iff the macro wasn't in duffle.INSTRUCTION_LATENCY. +--- The function strips trailing `()` from function-call style macros so `mac_yield()` and `mac_yield` resolve identically. local function token_cycles(tok) -- Extract the leading ident. Tolerate `(...)` args. local ident = tok:match("^([%w_]+)") @@ -956,9 +773,8 @@ local function token_cycles(tok) return cost, ident, false end ---- Find every `atom_label(name)` token in the token list and return a ---- map `label_name -> token_idx`. Labels are 0-cost markers; the path ---- walker uses them as branch targets. +--- Find every `atom_label(name)` token in the token list and return a map `label_name -> token_idx`. Labels are 0-cost markers; +--- the path walker uses them as branch targets. local function find_atom_labels(tokens) local labels = {} for tok_idx, t in ipairs(tokens) do @@ -968,16 +784,13 @@ local function find_atom_labels(tokens) return labels end ---- Find every `branch_*(...)` token in the token list and return a map ---- `token_idx -> label_name|false`. If the branch's args contain an ---- `atom_offset(F, label)` call, the label name is recorded; otherwise ---- the branch's target is unknown (likely a literal offset) and we ---- record `false` as a sentinel. The CFG walker checks KEY PRESENCE ---- (via `is_branch(tok_idx)`) to decide whether a token is a branch; it ---- checks the value to decide whether the taken-path target is known. ---- (We can't use `nil` for the unknown-target case because `targets[tok_idx] = nil` ---- REMOVES the key from the Lua table, which would make `is_branch(tok_idx)` ---- return false for both "not a branch" and "branch with unknown target".) +--- Find every `branch_*(...)` token in the token list and return a map `token_idx -> label_name|false`. +--- If the branch's args contain an `atom_offset(F, label)` call, the label name is recorded; otherwise the branch's target is unknown +--- (likely a literal offset) and we record `false` as a sentinel. +--- The CFG walker checks KEY PRESENCE (via `is_branch(tok_idx)`) to decide whether a token is a branch; it checks the value +--- to decide whether the taken-path target is known. +--- (We can't use `nil` for the unknown-target case because `targets[tok_idx] = nil` REMOVES the key from the Lua table, +--- which would make `is_branch(tok_idx)` return false for both "not a branch" and "branch with unknown target".) local function find_branch_targets(tokens) local targets = {} for tok_idx, t in ipairs(tokens) do @@ -993,11 +806,9 @@ local function find_branch_targets(tokens) end --- Walk all paths through an atom body and return per-path cycle sums. ---- Builds a tiny CFG: each token has a "next" pointer; branches have two ---- (fall-through + taken). The BD-slot nop after a branch is absorbed ---- into the branch's cost (MIPS-accurate: BD slot always runs), and is ---- SKIPPED when continuing down the fall-through path (otherwise we'd ---- double-count it). +--- Builds a tiny CFG: each token has a "next" pointer; branches have two (fall-through + taken). +--- The BD-slot nop after a branch is absorbed into the branch's cost (MIPS-accurate: BD slot always runs), +--- and is SKIPPED when continuing down the fall-through path (otherwise we'd double-count it). --- --- Returns: --- cycles_min - shortest path through the body (sum of token costs) @@ -1025,32 +836,26 @@ local function analyze_atom_paths(atom) end end - -- A token is a terminator if it's `mac_yield` or `mac_yield(...)`. - -- The yield transfers control; we don't count its cost (the next - -- atom's prologue absorbs it). + -- A token is a terminator if it's `mac_yield` or `mac_yield(...)`. + -- The yield transfers control; we don't count its cost (the next atom's prologue absorbs it). local function is_terminator(tok_idx) local tok = tokens[tok_idx].tok return tok == "mac_yield" or tok:match("^mac_yield%s*%(") end - -- CFG successor function. Returns a list of next token indices for - -- the given position. Branch tokens produce 2 successors (fall-through - -- + taken); normal tokens produce 1 (next); terminators produce 0. - -- BD-slot absorption: a branch at tok_idx skips tok_idx+1 (the BD slot) in its - -- fall-through path; the BD slot's cost is added to the branch's - -- own cost instead (so it's counted once). + -- CFG successor function. Returns a list of next token indices for the given position. + -- Branch tokens produce 2 successors (fall-through + taken); normal tokens produce 1 (next); terminators produce 0. + -- BD-slot absorption: a branch at tok_idx skips tok_idx+1 (the BD slot) in its fall-through path; + -- the BD slot's cost is added to the branch's own cost instead (so it's counted once). -- --- A token is a "branch" if its index is a KEY in the `branches` --- map (regardless of whether the value is nil — a branch with --- nil target means "literal offset, taken path is unknown"). --- We check key-presence via `branches[tok_idx] ~= nil` because --- `branches[tok_idx]` returns nil for both "absent" AND "present with --- nil value" — distinguishing them requires the key check. +-- A token is a "branch" if its index is a KEY in the `branches` map +-- (regardless of whether the value is nil — a branch with nil target means "literal offset, taken path is unknown"). +-- We check key-presence via `branches[tok_idx] ~= nil` because `branches[tok_idx]` returns nil for both "absent" AND "present with nil value". +-- Distinguishing them requires the key check. local function is_branch(tok_idx) local v = branches[tok_idx] if v == nil then return false end - -- v is non-nil: either a string (atom_offset target) or false - -- (literal offset, no target). Both indicate a branch. + -- v is non-nil: either a string (atom_offset target) or false (literal offset, no target). Both indicate a branch. return true end local function successors(tok_idx) @@ -1072,10 +877,8 @@ local function analyze_atom_paths(atom) succ[#succ + 1] = label_pos + 1 end end - -- For literal-offset branches (label == false), the taken - -- path would jump to a non-tracked address; conservatively - -- omit. Return (succ, nil) -- the second value is the - -- terminator marker (nil = not a terminator). + -- For literal-offset branches (label == false), the taken path would jump to a non-tracked address; conservatively omit. + -- Return (succ, nil) -- the second value is the terminator marker (nil = not a terminator). return succ, nil end -- Normal token: just the next one @@ -1085,10 +888,8 @@ local function analyze_atom_paths(atom) return {}, nil end - -- DFS through all paths. Track the current cycle sum, a visited set - -- scoped to the current path (to detect loops), and a count of paths. - -- Cap recursion at MAX_PATHS to prevent runaway exploration on - -- pathological bodies. + -- DFS through all paths. Track the current cycle sum, a visited set scoped to the current path (to detect loops), and a count of paths. + -- Cap recursion at MAX_PATHS to prevent runaway exploration on pathological bodies. local MAX_PATHS = 64 local cycles_min = math.huge local cycles_max = -1 @@ -1109,9 +910,8 @@ local function analyze_atom_paths(atom) end -- Add this token's cost. For a branch, ADD the BD-slot cost too - -- (and skip the BD slot in the successor list — already done in - -- `successors` above for fall-through; for taken path the BD - -- slot was at tok_idx+1 which is now skipped entirely). + -- (and skip the BD slot in the successor list — already done in `successors` above for fall-through; + -- for taken path the BD slot was at tok_idx+1 which is now skipped entirely). local cost = costs[tok_idx] if is_branch(tok_idx) and tok_idx + 1 <= n then cost = cost + costs[tok_idx + 1] @@ -1120,12 +920,10 @@ local function analyze_atom_paths(atom) local succ, term = successors(tok_idx) if term then - -- Terminator: record the path's cycle sum. We do NOT add - -- the terminator token to `visited` -- a path ends here, so - -- a different path that ALSO reaches this terminator is a - -- legitimate new path (not a loop). If we marked it - -- visited, subsequent paths that reach the same terminator - -- would be incorrectly flagged as loops. + -- Terminator: record the path's cycle sum. + -- We do NOT add the terminator token to `visited` a path ends here, so a different path that + -- ALSO reaches this terminator is a legitimate new path (not a loop). + -- If we marked it visited, subsequent paths that reach the same terminator would be incorrectly flagged as loops. path_count = path_count + 1 if new_acc < cycles_min then cycles_min = new_acc end if new_acc > cycles_max then cycles_max = new_acc end @@ -1139,8 +937,7 @@ local function analyze_atom_paths(atom) end if n >= 1 then dfs(1, 0, {}) end - -- If no paths were recorded (e.g. atom body is empty), cycles_min/max - -- default to 0 (atom costs nothing). + -- If no paths were recorded (e.g. atom body is empty), cycles_min/max default to 0 (atom costs nothing). if cycles_min == math.huge then cycles_min = 0 end if cycles_max == -1 then cycles_max = 0 end @@ -1163,8 +960,7 @@ local function analyze_atom_paths(atom) end --- Per-source check that emits one finding per unknown macro seen ---- (deduplicated across atoms so the warning section doesn't get ---- spammed with N copies of "macro X not in duffle.INSTRUCTION_LATENCY"). +--- (deduplicated across atoms so the warning section doesn't get spammed with N copies of "macro X not in duffle.INSTRUCTION_LATENCY"). local function check_per_atom_cycle_budget(atoms, findings) local unknown_seen = {} for _, a in ipairs(atoms) do @@ -1198,17 +994,18 @@ end -- ════════════════════════════════════════════════════════════════════════════ local function validate(ctx, src) - local source = src.text - local atoms = find_atom_bodies(source) + local scan = src.scan - -- Build per-source Binds_* index + per-atom atom_info lookup. These - -- are local to the validate() call; no cross-source sharing needed -- - -- every .c / .h file is validated standalone. + -- Read atoms + binds + atom_infos from the pre-scanned SourceScan payload. + -- The scan was done once upstream by duffle.scan_source(); this pass is pure. + local atoms = scan.atoms + local atom_infos = scan.atom_infos + + -- Build per-source Binds_* index. Local to validate() — no cross-source sharing. local binds_index = {} - for _, b in ipairs(find_binds_structs(source)) do + for _, b in ipairs(scan.binds) do binds_index[b.name] = b end - local atom_infos = find_atom_info(source) local findings = {} check_gte_pipeline_fill(atoms, findings) @@ -1232,22 +1029,18 @@ local function validate(ctx, src) local warnings = {} local info = {} for _, f in ipairs(findings) do - -- Per-finding severity is set by the check via `f.kind` - -- ("error" or "warning"). A `gte_pipeline_fill` finding can be - -- either severity (errors for missing nops; warnings for unknown - -- cmdw macros not in the latency table). Bin by `kind`, not by - -- check name. + -- Per-finding severity is set by the check via `f.kind` ("error" or "warning"). + -- A `gte_pipeline_fill` finding can be either severity (errors for missing nops; warnings for unknown cmdw macros not in the latency table). + -- Bin by `kind`, not by check name. if f.kind == "error" then errors [#errors + 1] = { line = f.line, msg = f.msg } else warnings[#warnings + 1] = { line = f.line, msg = f.msg } end end - -- Per-source "scanned:" summary line. Includes the source basename - -- for traceability (the old format was just "scanned: N atom - -- bodies; M findings" which is unidentifiable when the module has - -- multiple sources). Sources with 0 atoms (pure-header files like - -- dsl.h, mips.h, etc.) are SKIPPED — the per-module header already - -- lists them in the "Sources:" section, and emitting a noisy - -- "0 atom bodies" line per header is just clutter. + -- Per-source "scanned:" summary line. + -- Includes the source basename for traceability + -- (the old format was just "scanned: N atom bodies; M findings" which is unidentifiable when the module has multiple sources). + -- Sources with 0 atoms (pure-header files like dsl.h, mips.h, etc.) are SKIPPED. + -- The per-module header already lists them in the "Sources:" section, and emitting a noisy "0 atom bodies" line per header is just clutter. if #atoms > 0 or #findings > 0 then info[#info + 1] = { line = 0, @@ -1291,10 +1084,9 @@ end -- Per-directory output: build/gen/.static_analysis.txt -- ════════════════════════════════════════════════════════════════════════════ ---- Per-directory emit. Aggregates atoms + findings across every source ---- in `dir_sources` and writes a single report to ---- `/.static_analysis.txt`. Called only when ---- at least one atom was found (the caller in M.run handles the skip). +--- Per-directory emit. Aggregates atoms + findings across every source in `dir_sources` +--- and writes a single report to `/.static_analysis.txt`. +--- Called only when at least one atom was found (the caller in M.run handles the skip). local function emit_module_static_analysis_txt(ctx, dir, dir_sources, atoms, findings, errors, warnings, info) -- Module basename = last component of `dir` ("code/duffle" -> "duffle"). local dir_basename = dir:match("([^/\\]+)$") or dir @@ -1377,8 +1169,7 @@ local function emit_module_static_analysis_txt(ctx, dir, dir_sources, atoms, fin -- max = longest path through the body (full fall-through) -- br = number of branch instructions -- paths = number of distinct paths reached - -- Both min and max are best-case (no stalls); BD-slot nops are - -- absorbed into branch costs (MIPS semantics). + -- Both min and max are best-case (no stalls); BD-slot nops are absorbed into branch costs (MIPS semantics). add("") add("── Per-atom cycle counts (path-aware, best case, no stalls) ─") if #atoms == 0 then @@ -1415,12 +1206,9 @@ local function emit_module_static_analysis_txt(ctx, dir, dir_sources, atoms, fin add("") add("── Per-source scan summary ──────────────────────────────") - -- One line per source that contributed atoms. The line includes - -- the source basename + per-source atom count + (if path-aware - -- cycle data is present) the min..max cycle range. Sources with - -- 0 atoms are skipped (they're just header files that declared - -- no MipsAtom_ — they're already listed in the module's - -- "Sources:" section above). + -- One line per source that contributed atoms. + -- The line includes the source basename + per-source atom count + (if path-aware cycle data is present) the min..max cycle range. + -- Sources with 0 atoms are skipped (they're just header files that declared no MipsAtom_ — they're already listed in the module's "Sources:" section above). for _, src in ipairs(dir_sources) do local src_atoms = {} for _, a in ipairs(atoms) do @@ -1457,8 +1245,7 @@ local function emit_module_static_analysis_txt(ctx, dir, dir_sources, atoms, fin add("") add(string.format("Module findings: %d error(s), %d warning(s)", total_errs, total_warns)) - -- Per-source "scanned:" info lines (each line includes the source - -- basename for traceability). + -- Per-source "scanned:" info lines (each line includes the source basename for traceability). if #info > 0 then add("") for _, i_ in ipairs(info) do @@ -1485,14 +1272,11 @@ function M.run(ctx) local errors = {} local warnings = {} - -- Aggregate per-DIRECTORY (per-module). One static_analysis.txt per - -- source-directory, emitted only if the directory contains at least - -- one atom. Empty-source directories (e.g. duffle headers with no - -- atoms) produce no report. + -- Aggregate per-DIRECTORY (per-module). One static_analysis.txt per source-directory, emitted only if the directory contains at least one atom. + -- Empty-source directories (e.g. duffle headers with no atoms) produce no report. -- - -- Group sources by `src.dir`. The first component of `dir` is the - -- module name (e.g. "code/duffle" -> "duffle", "code/gte_hello" -> - -- "gte_hello"). Output path is `/.static_analysis.txt`. + -- Group sources by `src.dir`. The first component of `dir` is the module name (e.g. "code/duffle" -> "duffle", "code/gte_hello" -> "gte_hello"). + -- Output path is `/.static_analysis.txt`. local by_dir = {} for _, src in ipairs(ctx.sources) do by_dir[src.dir] = by_dir[src.dir] or {} @@ -1500,11 +1284,9 @@ function M.run(ctx) end for dir, dir_sources in pairs(by_dir) do - -- Run validate() against every source in this directory; accumulate - -- atoms / findings / errors / warnings. The validate() function - -- does its own per-source analysis (Binds indexing, atom - -- discovery, all 5 checks) and attaches path-aware cycle data to - -- each atom it finds. + -- Run validate() against every source in this directory; accumulate atoms / findings / errors / warnings. + -- The validate() function does its own per-source analysis (Binds indexing, atom discovery, all 5 checks) + -- and attaches path-aware cycle data to each atom it finds. local all_atoms = {} local all_findings = {} local dir_errors = {} @@ -1512,41 +1294,29 @@ function M.run(ctx) local all_info = {} for _, src in ipairs(dir_sources) do local result = validate(ctx, src) - -- Tag each atom with its source so the render step can prefix - -- the atom line with ":" when atoms from multiple - -- sources live in the same module (e.g. lottes_tape.h + - -- atom_dsl.h both declaring atoms). + -- Tag each atom with its source so the render step can prefix the atom line with ":" + -- when atoms from multiple sources live in the same module (e.g. lottes_tape.h + atom_dsl.h both declaring atoms). for _, a in ipairs(result.atoms) do a.source_path = src.path all_atoms[#all_atoms + 1] = a end - for _, f in ipairs(result.findings) do - all_findings[#all_findings + 1] = f - end - for _, e in ipairs(result.errors) do - dir_errors[#dir_errors + 1] = e - end - for _, w in ipairs(result.warnings) do - dir_warnings[#dir_warnings + 1] = w - end - for _, i_ in ipairs(result.info) do - all_info[#all_info + 1] = i_ - end + for _, f in ipairs(result.findings) do all_findings[#all_findings + 1] = f end + for _, e in ipairs(result.errors) do dir_errors [#dir_errors + 1] = e end + for _, w in ipairs(result.warnings) do dir_warnings[#dir_warnings + 1] = w end + for _, i_ in ipairs(result.info) do all_info [#all_info + 1] = i_ end end - -- Skip directories with zero atoms — a directory with only - -- headers / no MipsAtom_ is "nothing to report". + -- Skip directories with zero atoms. A directory with only headers / no MipsAtom_ is "nothing to report". if #all_atoms == 0 then - -- Still aggregate errors/warnings so orchestrator sees them, - -- but don't write a file. - for _, e in ipairs(dir_errors) do errors[#errors + 1] = e end + -- Still aggregate errors/warnings so orchestrator sees them, but don't write a file. + for _, e in ipairs(dir_errors) do errors [#errors + 1] = e end for _, w in ipairs(dir_warnings) do warnings[#warnings + 1] = w end else local out_path = emit_module_static_analysis_txt(ctx, dir, dir_sources, all_atoms, all_findings, dir_errors, dir_warnings, all_info) if out_path then table.insert(outputs, { static_analysis_txt = out_path }) end - for _, e in ipairs(dir_errors) do errors[#errors + 1] = e end + for _, e in ipairs(dir_errors) do errors [#errors + 1] = e end for _, w in ipairs(dir_warnings) do warnings[#warnings + 1] = w end end end diff --git a/scripts/ps1_meta.lua b/scripts/ps1_meta.lua index d79f865..f7befc3 100644 --- a/scripts/ps1_meta.lua +++ b/scripts/ps1_meta.lua @@ -6,13 +6,15 @@ --- --- **Architecture**: --- - **PASSES table** — declarative dep graph (data, not code). ---- - **FLAG_HANDLERS table** — per-flag CLI dispatchers (handler-map ---- pattern; replaces an 8-way if/elseif chain). ---- - **parse_args** → **build_ctx** → **topo_sort** → **dispatch_passes**. +--- - **FLAG_HANDLERS table** — per-flag CLI dispatchers (handler-map pattern; replaces an 8-way if/elseif chain). +--- - **parse_args** → **build_ctx** (just opens + reads source files; no inline scanning) → **topo_sort** → **dispatch_passes**. +--- - The first pass in the dep graph is `scan-source` (see `passes/scan_source.lua`). +--- It calls `duffle.scan_source` once per source to produce the fat `SourceScan` payload, which is attached to each `src.scan`. +--- Every other pass that reads source structure depends on `scan-source` and consumes `src.scan` as a read-only payload. --- --- **Conventions**: tabs (1/level), EmmyLua annotations, no regex, ---- Lua 5.3 compatible. ---- +--- Lua 5.3 compatible. +--- -- ════════════════════════════════════════════════════════════════════════════ -- Module-scope requires + package.path setup -- ════════════════════════════════════════════════════════════════════════════ @@ -104,6 +106,13 @@ local PASS_FLAG_DISPATCH_KEY = "__pass__" -- ════════════════════════════════════════════════════════════════════════════ local PASSES = { + ["scan-source"] = { + module = "passes.scan_source", + kind = "shared", + deps = {}, + desc = "Walk each source once; produce the fat SourceScan payload for downstream passes", + out = {}, + }, ["word-counts"] = { module = "passes.word_count_eval", kind = "shared", @@ -114,14 +123,14 @@ local PASSES = { components = { module = "passes.components", kind = "header-output", - deps = {"word-counts"}, + deps = {"scan-source", "word-counts"}, desc = "Emit mac_X macros from MipsAtomComp_ declarations", out = { { kind = "header", path_template = "/gen/.macs.h" } }, }, annotation = { module = "passes.annotation", kind = "validation", - deps = {"word-counts"}, + deps = {"scan-source", "word-counts"}, desc = "Validate atom DSL usage; emit errors.h + annotations.txt", out = { { kind = "report", path_template = "/.errors.h" }, @@ -131,14 +140,14 @@ local PASSES = { offsets = { module = "passes.offsets", kind = "header-output", - deps = {"word-counts", "components"}, + deps = {"scan-source", "word-counts", "components"}, desc = "Compute branch offsets for atom_label / atom_offset", out = { { kind = "header", path_template = "/gen/.offsets.h" } }, }, ["static-analysis"] = { module = "passes.static_analysis", kind = "validation", - deps = {"word-counts", "components"}, + deps = {"scan-source", "word-counts", "components"}, desc = "[FUTURE] GTE pipeline-fill, mac_yield uniformity, etc.", out = { { kind = "report", path_template = "/.static_analysis.txt" } }, }, @@ -167,11 +176,12 @@ local PASS_FLAG_TO_NAME = { ["--offsets"] = "offsets", ["--static-analysis"] = "static-analysis", ["--report"] = "report", + ["--scan-source"] = "scan-source", ["--all"] = ALL_PASSES_SENTINEL, } local ALL_PASS_NAMES = { - "word-counts", "components", "annotation", + "scan-source", "word-counts", "components", "annotation", "offsets", "static-analysis", "report", } @@ -337,6 +347,9 @@ local function build_ctx(args) dir = dir:sub(1, -2) end + -- src.scan is populated by the "scan-source" pass (the first pass in the + -- dep graph). build_ctx just opens + reads the files; the scan itself + -- happens in the pass module, not inline in the orchestrator. sources[#sources + 1] = { path = path, text = text, @@ -509,41 +522,52 @@ local function render_dep_graph(passes, requested, closed) end add("") - add("[ps1_meta] Pass graph (read top-to-bottom):") + -- Data-driven ASCII graph built from the actual PASSES table. + -- Shows the source -> scan_source -> pass chain. Each pass is + -- shown once; edges are "feeds into" arrows based on deps. + add("[ps1_meta] Pass graph (read top-to-bottom; edges = 'feeds into'):") add("") - add(" metadata.h") - add(" |") - add(" v") - add(" +-----------+ +-----------------+ +-----------------+") - add(" | word- |-->| components |-->| offsets |") - add(" | counts | +-----------------+ +-----------------+") - add(" | (load) | | ^") - add(" +-----------+ | |") - add(" | v |") - add(" | code//gen/.macs.h |") - add(" | (header - co-located for #include) |") - add(" | |") - add(" | +-----------------+ |") - add(" +---------->| annotation |--------------+") - add(" | +-----------------+ |") - add(" | | |") - add(" | v |") - add(" | build/gen/.errors.h |") - add(" | build/gen/.annotations.txt |") - add(" | (report - NOT #included) |") - add(" | |") - add(" | +-----------------+ |") - add(" +---------->| static-analysis |--------------+") - add(" +-----------------+") - add(" |") - add(" v") - add(" +---------------+") - add(" | report |") - add(" +---------------+") - add(" |") - add(" v") - add(" build/gen/annotation_validation.txt") - add(" (project summary)") + + -- Compute which passes feed which other passes (reverse of deps). + local feeds = {} -- feeds[X] = list of passes that X feeds into + for _, name in ipairs(closed) do feeds[name] = {} end + for name, p in pairs(passes) do + for _, dep in ipairs(p.deps) do + if feeds[dep] then feeds[dep][#feeds[dep] + 1] = name end + end + end + + -- Layout: source -> scan_source -> word-counts -> {components, annotation, offsets, static-analysis} -> report + -- Outputs are listed under each pass. + local outputs_for = function(name) + local p = passes[name] + if not p or not p.out or #p.out == 0 then return "" end + local outs = {} + for _, o in ipairs(p.out) do outs[#outs + 1] = o.path_template end + return table.concat(outs, ", ") + end + + add(" +-----------+ +-------------------+ +-----------------+") + add(" | source |-->| scan_source |--->| word-counts |") + add(" | files | | (scan_source.lua) | | (load) |") + add(" +-----------+ +-------------------+ +-----------------+") + add(" (single walk) |") + add(" |") + add(" +-------------------+-------------------+-----------+") + add(" v v v v") + add(" +--------------+ +--------------+ +--------------+ +---------------+") + add(" | components | | annotation | | offsets | |static-analysis|") + add(" +--------------+ +--------------+ +--------------+ +---------------+") + add(" |/gen/ | |build/gen/ | |/gen/ | |build/gen/ |") + add(" |.macs.h | |.errors | |.offsets| |.static |") + add(" | (header) | | .h | | .h | | _analysis |") + add(" +------+-------+ | +annot.txt | | (header) | | .txt |") + add(" | +------+-------+ +--------------+ +------+--------+") + add(" v v v") + add(" +------+----------------+ +------+-------+ |") + add(" |offsets|static-analysis| |report| |<--------------------+") + add(" | | | +------+-------+") + add(" +-------+---------------+") return table.concat(lines, "\n") .. "\n" end diff --git a/scripts/update_deps.ps1 b/scripts/update_deps.ps1 index 036ffa1..91dc9bf 100644 --- a/scripts/update_deps.ps1 +++ b/scripts/update_deps.ps1 @@ -10,19 +10,6 @@ $ErrorActionPreference = 'Stop' $misc = join-path $PSScriptRoot 'helpers/misc.ps1' . $misc -# TODO(Ed): Review usage of these deps -# I originally cloned them when starting to get to the C runtime usage of the course -# However, based on the heavy reliance of the PSX.Dev extension I might fallback; also -# The gdb server doesn't need the full repo and were only using the src/mips -# which has a standalone repo (nuggets) -# armips may not be used at all but I'm not sure... -# -# PCSX-Redux: built via MSBuild (VS2022) — automated in the build section below. -# Requires: VS2022 with C++ desktop workload + PlatformToolset=v143 retarget. -# The .vcxproj files request v145; we pass /p:PlatformToolset=v143 to MSBuild. -# NuGet packages are restored automatically on first build. -# Output: toolchain\pcsx-redux\vsprojects\x64\Debug\pcsx-redux.exe - $url_armips = 'https://github.com/Kingcom/armips.git' $url_pcsx_redux = 'https://github.com/grumpycoders/pcsx-redux.git' $url_psyq_iwyu = 'https://github.com/johnbaumann/psyq_include_what_you_use.git'