From 9f7a4a00ceae03f364d39a838958665addefb002 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sat, 11 Jul 2026 19:53:12 -0400 Subject: [PATCH] final pass on metaprogram --- scripts/duffle.lua | 48 +-- scripts/duffle_paths.lua | 44 +-- scripts/passes/annotation.lua | 338 +++++++++++------- scripts/passes/components.lua | 64 ++-- scripts/passes/offsets.lua | 20 +- scripts/passes/report.lua | 47 ++- scripts/passes/scan_source.lua | 553 ++++++++++++++++------------- scripts/passes/static_analysis.lua | 181 +++++----- scripts/passes/word_count_eval.lua | 57 +-- 9 files changed, 741 insertions(+), 611 deletions(-) diff --git a/scripts/duffle.lua b/scripts/duffle.lua index a7dcd15..d4e9896 100644 --- a/scripts/duffle.lua +++ b/scripts/duffle.lua @@ -12,19 +12,13 @@ --- - **Process-bootstrap helper** (`setup_package_path`replaces the 8-line `arg[0]`-resolution boilerplate duplicated across 7 entry scripts) --- --- **Conventions**: tabs (1/level), EmmyLua annotations, no regex. ---- Lua 5.3 compatible; no ``/``, no `continue`, no ---- 5.4 string.dump improvements. LuaJIT 5.1+extensions model is the primary target. ---- ---- **No `:match` / `:gmatch` regex use anywhere**; ---- all delimiter-splitting is hand-rolled or via LPeg (the regex-free PEG library). local M = {} --- Optional native extension: lfs (LuaFileSystem). When present, ensure_dir uses --- lfs.attributes + lfs.mkdir instead of spawning `cmd.exe mkdir` — saves ~55ms per --- unique directory on Windows. Built by `update_deps.ps1` to `toolchain/lfs/lfs.dll` --- and wired into package.cpath by `scripts/duffle_paths.lua`. -local lfs = pcall(require, "lfs") and require("lfs") or nil +-- Required native extension: lfs (LuaFileSystem). Built by `update_deps.ps1` to +-- `toolchain/lfs/lfs.dll` and wired into package.cpath by `scripts/duffle_paths.lua`. +-- If lfs is missing, `require` throws — fail loud per the build-tool convention. +local lfs = require("lfs") -- ════════════════════════════════════════════════════════════════════════════ -- Cross-file type aliases @@ -315,7 +309,7 @@ end -- Normalizes forward slashes to backslashes on Windows. -- Used for byte-identical emit: the // Source: comment line uses the absolute path. -- --- The CWD is memoized (one `io.popen("cd")` per process — ~50ms on Windows). +-- The CWD is memoized on first call (one lfs.currentdir() per process — ~0ms). -- Without the cache, calling this per-source in the components pass added ~1.5s to a 30-source build. -- @param path string -- @return string @@ -329,16 +323,8 @@ function M.to_absolute_path(path) _absolute_path_cache[path] = result return result end - -- Native: lfs.currentdir() is ~0ms vs io.popen("cd") at ~50ms per call. - local cwd - if lfs then - cwd = lfs.currentdir() - else - local p = io.popen("cd") - if not p then _absolute_path_cache[path] = path; return path end - cwd = p:read("*l") - p:close() - end + -- lfs.currentdir() is ~0ms vs io.popen("cd") at ~50ms per call on Windows. + local cwd = lfs.currentdir() if not cwd then _absolute_path_cache[path] = path; return path end cwd = cwd:gsub("/", "\\") local tail = (path:gsub("/", "\\")) @@ -356,15 +342,9 @@ local _ensured_dirs = {} function M.ensure_dir(path) if _ensured_dirs[path] then return end _ensured_dirs[path] = true - if lfs then - -- Native: ~0ms when dir exists (the common case). lfs.mkdir on a new dir is ~2ms (no shell spawn). - -- Falls through silently if lfs.mkdir fails (e.g. permission denied); the subsequent write_file will surface the error. - if lfs.attributes(path, "mode") ~= "directory" then lfs.mkdir(path) end - else - -- Fallback: shell mkdir. Slow (~55ms per call on Windows due to cmd.exe spawn) but works without lfs. - local is_win = package.config:sub(1, 1) == "\\" - os.execute(is_win and ('if not exist "' .. path .. '" mkdir "' .. path .. '"') or ('mkdir -p "' .. path .. '" 2>/dev/null')) - end + -- lfs.attributes + lfs.mkdir: ~0ms when dir exists, ~2ms when creating. No shell spawn. + -- Falls through silently if lfs.mkdir fails (e.g. permission denied); the subsequent write_file will surface the error. + if lfs.attributes(path, "mode") ~= "directory" then lfs.mkdir(path) end end -- Test helper: clear the cache (used by tests + between process runs). @@ -585,8 +565,9 @@ end -- Moved here from passes/static_analysis.lua so all passes can share the memoized -- per-body tokenization. The memoization key is the body string (immutable per pass). -local _tokenize_body_cache = {} -local _body_line_index_cache = {} +local _tokenize_body_cache = {} +local _tokenize_body_simple_cache = {} +local _body_line_index_cache = {} --- Tokenize the body inner-text into a flat list of `{tok, rel}` pairs. --- `tok` is the trimmed token string; `rel` is the byte offset within `body`. @@ -633,12 +614,15 @@ end --- Tokenize the body into a flat list of trimmed string tokens (preserves comments). --- Uses `split_top_level_commas` (which appends trailing comments to the previous token) --- so the components pass can emit `/* Words: ... */` comments in the .macs.h output. +--- Memoized on body string (R7 lift; mirror of M.tokenize_body's memoization). --- @param body string --- @return string[] function M.tokenize_body_simple(body) + if _tokenize_body_simple_cache[body] ~= nil then return _tokenize_body_simple_cache[body] end local tokens = M.split_top_level_commas(body) local out = {} for i = 1, #tokens do out[i] = M.trim(tokens[i]) end + _tokenize_body_simple_cache[body] = out return out end diff --git a/scripts/duffle_paths.lua b/scripts/duffle_paths.lua index a3c4060..ab74b94 100644 --- a/scripts/duffle_paths.lua +++ b/scripts/duffle_paths.lua @@ -30,47 +30,25 @@ local CACHE_KEY = "__duffle_repo_root__" --- Replaces the prior `io.popen("git rev-parse --show-toplevel")` approach, which cost ~100-180ms per --- LuaJIT process on Windows due to git's CLI startup. The path-derive approach costs <1ms. --- ---- If this script's path can't be parsed (shouldn't happen — dofile/debug.getinfo always populates source), ---- fall back to a defensive walk: starting from this script's directory, walk UP until we find a parent that ---- contains a `scripts/` directory. The first match is the repo root. +--- If `debug.getinfo` can't parse this script's path (shouldn't happen — dofile always populates source), +--- return nil and let `M.setup()` fail loud. --- @return string|nil local function find_repo_root() if package.loaded[CACHE_KEY] then return package.loaded[CACHE_KEY] end local source = debug.getinfo(1, "S").source -- Strip the leading `@` (Lua's dofile marker) and the trailing `/duffle_paths.lua` filename. - -- What remains is the directory containing this script, i.e. `/scripts/` (with trailing slash or not). + -- What remains is the directory containing this script, i.e. `/scripts/`. local scripts_dir = source and source:match("^@?(.*)[/\\]duffle_paths%.lua$") - if scripts_dir then - -- The repo root is the parent of `scripts/`. Strip the trailing `scripts/` (with or without trailing slash). - local root = scripts_dir:gsub("scripts[\\/]?$", "") - root = root:gsub("\\", "/") - if root == "" then root = "./" end - if not root:match("/$") then root = root .. "/" end - package.loaded[CACHE_KEY] = root - return root - end + if not scripts_dir then return nil end - -- Defensive fallback: walk UP from this script's directory until we find a parent that contains `scripts/`. - -- In practice this branch never fires — debug.getinfo always returns a source for dofile()'d chunks. - local lfs = pcall(require, "lfs") and require("lfs") or nil - if lfs then - local dir = source and source:match("^@?(.*[/\\])") or "./" - dir = dir:gsub("\\", "/") - while dir and dir ~= "" do - local candidate_scripts = dir .. "scripts" - if lfs.attributes(candidate_scripts, "mode") == "directory" then - dir = dir:gsub("/$", "") - package.loaded[CACHE_KEY] = dir .. "/" - return dir .. "/" - end - local parent = dir:match("^(.*)/[^/]+/$") - if not parent then break end - dir = parent .. "/" - end - end - - return nil + -- The repo root is the parent of `scripts/`. Strip the trailing `scripts/` (with or without trailing slash). + local root = scripts_dir:gsub("scripts[\\/]?$", "") + root = root:gsub("\\", "/") + if root == "" then root = "./" end + if not root:match("/$") then root = root .. "/" end + package.loaded[CACHE_KEY] = root + return root end --- Set `package.path` (for `require("duffle")` + `require("passes.X")`) and diff --git a/scripts/passes/annotation.lua b/scripts/passes/annotation.lua index 7a99bac..4b8d7d2 100644 --- a/scripts/passes/annotation.lua +++ b/scripts/passes/annotation.lua @@ -3,16 +3,13 @@ --- Validates `MipsAtom_(name) atom_info(atom_bind(Binds_X), atom_reads(...), atom_writes(...)) { ... }` declarations in source files. --- Also reads: `Binds_*` struct declarations (`typedef Struct_(Binds_X) { ... };`) --- ---- Source scanning: done ONCE upstream by `duffle.scan_source()` (ps1_meta.lua pre-scans each ---- source and stashes the result in `src.scan`). This pass is pure: read from the scan, run ---- checks, emit findings. No source re-walking. +--- Source scanning: done ONCE upstream by `duffle.scan_source()` (ps1_meta.lua pre-scans each source and stashes the result in `src.scan`). --- --- Writes: --- - `/.errors.h` — one per module, with `#error` directives on findings (the C compile will surface the error) --- - The annotations.txt report is rendered by `passes/report.lua` from the per-module results stashed in `ctx.flags._annot_results` --- ---- **Conventions**: tabs (1/level), EmmyLua annotations, no regex, ---- Lua 5.3 compatible +--- **Conventions**: tabs (1/level), EmmyLua annotations, no regex, Lua 5.3 compatible -- Bootstrap: same as entry scripts. See `ps1_meta.lua` for the rationale. -- Bootstrap: load `scripts/duffle_paths.lua` (sets package.path + package.cpath). @@ -21,13 +18,12 @@ -- Bootstrap: load `duffle_paths.lua` via `debug.getinfo(1, "S").source` (works both standalone + when require'd). -- duffle_paths.lua sets package.path then returns `require("duffle")` at the bottom, so the dofile value IS the duffle module. local _bootstrap_dir = debug.getinfo(1, "S").source:match("^@?(.*[/\\])") or "./" -local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua") -local write_file = duffle.write_file -local ensure_dir = duffle.ensure_dir +local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua") +local write_file = duffle.write_file +local ensure_dir = duffle.ensure_dir -- Domain tables (single source of truth in duffle.lua). local WAVE_CONTEXT_REGS = duffle.WAVE_CONTEXT_REGS -local TAPE_ATOM_MACROS = duffle.TAPE_ATOM_MACROS local function is_wave_context_reg(n) return WAVE_CONTEXT_REGS[n] ~= nil end @@ -68,11 +64,22 @@ local function is_wave_context_reg(n) return WAVE_CONTEXT_REGS[n] ~= nil end --- @field binds string|nil -- Binds_X name if any --- @field reads string[] -- R_* names (read targets) --- @field writes string[] -- R_* names (write targets) +--- @field errors string[]|nil -- parse-time errors from scan_source (atom_info body malformed) --- @class Finding --- @field line integer -- source line (or 0 for pass-level) --- @field msg string -- finding message +--- @class Findings +--- @field errors Finding[] +--- @field warnings Finding[] +--- @field info Finding[] + +--- @class PipeCtx +--- @field atom_index table -- name -> AtomAnnotation (only kind=="atom") +--- @field binds_index table -- name -> BindsStruct +--- @field annot_counts table -- name -> annotation count (for unique_annotation check) + --- @class AnnotatedResult --- @field atoms AtomEntry[] --- @field annots AtomAnnotation[] @@ -82,6 +89,154 @@ local function is_wave_context_reg(n) return WAVE_CONTEXT_REGS[n] ~= nil end --- @field warnings Finding[] --- @field info Finding[] +-- ════════════════════════════════════════════════════════════════════════════ +-- Per-check functions (the CHECK_RULES table's payload) +-- ════════════════════════════════════════════════════════════════════════════ +-- +-- Each check has a uniform `append_to_findings` shape (errors[] / warnings[] / info[]). +-- The dispatcher in `validate()` decides which findings list each check writes to — by convention, +-- "existence" checks (declaration must exist, struct must exist) write errors[]; "shape" checks +-- (writes/reads must be wave-context) write warnings[]. The `macro_word_drift` check writes +-- both errors[] (missing/mismatch) and info[] (match). + +--- Check: every annotated atom must have a matching MipsAtom_(name) declaration. +--- @param a AtomAnnotation +--- @param pipe_ctx PipeCtx +--- @param findings Findings +local function check_atom_decl_exists(a, pipe_ctx, findings) + if not pipe_ctx.atom_index[a.name] then + findings.errors[#findings.errors + 1] = { + line = a.line, + msg = string.format("annotation for '%s' has no matching MipsAtom_(%s) { ... }", a.name, a.name), + } + end +end + +--- Check: every atom may have AT MOST ONE annotation. +--- Post-loop: needs full-corpus `annot_counts` from pipe_ctx. +--- @param pipe_ctx PipeCtx +--- @param findings Findings +local function check_unique_annotation(pipe_ctx, findings) + for name, n in pairs(pipe_ctx.annot_counts) do + if n > 1 then + findings.errors[#findings.errors + 1] = { + line = pipe_ctx.atom_index[name] and pipe_ctx.atom_index[name].line or 0, + msg = string.format("MipsAtom_(%s) has %d annotations (expected at most 1)", name, n), + } + end + end +end + +--- Check: BIND atoms must reference a real Binds_* struct. +--- Demoted from error to warning (2026-07-10): the same condition is now caught by passes/static_analysis.lua's +--- check_abi_handoff() as an error. Emitting a warning here keeps the annotation pass from being stop-on-error +--- for the common test-fixture case, while still surfacing the issue in the report. +--- The static-analysis report remains the source of truth for build-stopping errors. +--- @param a AtomAnnotation +--- @param pipe_ctx PipeCtx +--- @param findings Findings +local function check_binds_struct_exists(a, pipe_ctx, findings) + if not a.binds then return end + if pipe_ctx.binds_index[a.binds] then return end + findings.warnings[#findings.warnings + 1] = { + line = a.line, + msg = string.format("'%s' binds '%s' but no Struct_(%s) { ... } " + .. "declaration found (also flagged as an error by check_abi_handoff in the static-analysis pass)" + , a.name, a.binds, a.binds), + } +end + +--- Check: Binds_* struct fields must correspond to known wave-context registers. +--- Also checks that all `atom_writes(...)` entries are wave-context registers. +--- @param a AtomAnnotation +--- @param pipe_ctx PipeCtx +--- @param findings Findings +local function check_binds_field_wave_context(a, pipe_ctx, findings) + if not (a.binds and pipe_ctx.binds_index[a.binds]) then return end + local bs = pipe_ctx.binds_index[a.binds] + + for _, f in ipairs(bs.fields) do + local candidate = "R_" .. f.name + if not is_wave_context_reg(candidate) then + findings.warnings[#findings.warnings + 1] = { + line = bs.line, + msg = string.format("%s field '%s' doesn't match a known wave-context register (candidate '%s')", a.binds, f.name, candidate), + } + end + end + + for _, w in ipairs(a.writes) do + if not is_wave_context_reg(w) then + findings.warnings[#findings.warnings + 1] = { + line = a.line, + msg = string.format("%s writes '%s' which is not a known wave-context register", a.name, w), + } + end + end +end + +--- Check: atom_reads(...) entries should be wave-context registers (or R_TapePtr for rbind). +--- @param a AtomAnnotation +--- @param pipe_ctx PipeCtx +--- @param findings Findings +local function check_reads_wave_context(a, pipe_ctx, findings) + for _, r in ipairs(a.reads) do + if not is_wave_context_reg(r) and r ~= "R_TapePtr" then + findings.warnings[#findings.warnings + 1] = { + line = a.line, + msg = string.format("atom '%s' reads '%s' which is not a known wave-context register", a.name, r), + } + end + end +end + +--- Check: TAPE_WORDS(mac_X, N) ↔ WORD_COUNT(mac_X, N) drift. +--- Three outcomes: missing (error), mismatch (error), match (info). +--- @param m MacroEntry +--- @param wc table -- the shared word-count table (from ctx.shared.word_counts) +--- @param findings Findings +local function check_macro_word_drift(m, wc, findings) + local declared = wc[m.name] + if not declared then + findings.errors[#findings.errors + 1] = { + line = m.line, + msg = string.format("TAPE_WORDS(%s, %d) but '%s' is not in metadata.h", m.name, m.words, m.name), + } + return + end + if declared ~= m.words then + findings.errors[#findings.errors + 1] = { + line = m.line, + msg = string.format("DRIFT: TAPE_WORDS(%s, %d) but metadata.h declares WORD_COUNT(%s, %d)", m.name, m.words, m.name, declared), + } + return + end + findings.info[#findings.info + 1] = { + line = m.line, + msg = string.format("OK: %s = %d words", m.name, m.words), + } +end + +-- ════════════════════════════════════════════════════════════════════════════ +-- CHECK_RULES — data-driven check dispatch (the plex pattern) +-- ════════════════════════════════════════════════════════════════════════════ +-- +-- Each rule entry picks one of three "shapes" of dispatch: +-- per_annot(annot, pipe_ctx, findings) — runs once per AtomAnnotation +-- post(pipe_ctx, findings) — runs once after all per_annot calls complete (full-corpus aggregation) +-- per_macro(macro, wc, findings) — runs once per TAPE_WORDS / _Pragma macro declaration +-- +-- Adding a new check = 1 row here + 1 function above. The `validate()` dispatch loop never needs editing. + +local CHECK_RULES = { + { name = "atom_decl_exists", per_annot = check_atom_decl_exists }, + { name = "binds_struct_exists", per_annot = check_binds_struct_exists }, + { name = "binds_field_wave_context", per_annot = check_binds_field_wave_context }, + { name = "reads_wave_context", per_annot = check_reads_wave_context }, + { name = "unique_annotation", post = check_unique_annotation }, + { name = "macro_word_drift", per_macro = check_macro_word_drift }, +} + -- ════════════════════════════════════════════════════════════════════════════ -- Validation -- ════════════════════════════════════════════════════════════════════════════ @@ -115,138 +270,65 @@ local function validate(ctx, src) binds = info.binds, reads = info.reads or {}, writes = info.writes or {}, - errors = {}, + errors = info.errors, } end - -- Index atoms by name for lookup. - local atom_index = {} - for _, a in ipairs(atoms) do atom_index[a.name] = a end - - -- Index binds by name for lookup. - local binds_index = {} - for _, b in ipairs(scan.binds) do binds_index[b.name] = b end - - local errors = {} - local warnings = {} - local info = {} - - -- 1. Every annotated atom must exist as a real MipsAtom_ declaration. - for _, a in ipairs(annots) do - if not atom_index[a.name] then - errors[#errors + 1] = { - line = a.line, - msg = string.format("annotation for '%s' has no matching MipsAtom_(%s) { ... }", a.name, a.name), - } + -- Build pipe_ctx (Fleury: expose structure). Pre-compute everything the per-check functions need. + -- Single source of truth for atom / binds / annotation-count lookups. + local pipe_ctx = { + atom_index = {}, + binds_index = {}, + annot_counts = {}, + } + for _, a in ipairs(atoms) do pipe_ctx.atom_index [a.name] = a end + for _, b in ipairs(scan.binds) do pipe_ctx.binds_index[b.name] = b end + for _, a in ipairs(annots) do + if a.name then + pipe_ctx.annot_counts[a.name] = (pipe_ctx.annot_counts[a.name] or 0) + 1 end + end + + -- Findings live in a single struct with three lists (errors / warnings / info). + -- Each check writes to the list appropriate for its severity. + local findings = { errors = {}, warnings = {}, info = {} } + + -- Propagate parse-time errors from scan_source's atom_info parsing. + -- These are errors found in the atom_info(...) body itself (e.g., malformed args). + -- They are pre-existing in the scan payload — we just lift them into our findings list. + for _, a in ipairs(annots) do if a.errors then for _, msg in ipairs(a.errors) do - errors[#errors + 1] = {line = a.line, msg = string.format("'%s': %s", a.name, msg)} - end - end - end - - -- 2. Every atom may have AT MOST ONE annotation (no duplicates). - -- (Atoms with ZERO annotations are valid in the new minimal shape.) - local count_per_atom = {} - for _, a in ipairs(annots) do - if a.name then - count_per_atom[a.name] = (count_per_atom[a.name] or 0) + 1 - end - end - for name, n in pairs(count_per_atom) do - if n > 1 then - errors[#errors + 1] = { - line = atom_index[name] and atom_index[name].line or 0, - msg = string.format("MipsAtom_(%s) has %d annotations (expected at most 1)", name, n), - } - end - end - - -- 3. (Phase validity check DROPPED. Phases were removed from the annotation DSL.) - - -- 4. BIND atoms must reference a real Binds_* struct. - for _, a in ipairs(annots) do - if a.binds then - if not binds_index[a.binds] then - -- Demoted from error to warning (2026-07-10): the same condition is now caught by passes/static_analysis.lua's - -- check_abi_handoff() as an error. Emitting a warning here keeps the annotation pass from being stop-on-error - -- for the common test-fixture case, while still surfacing the issue in the report. - -- The static-analysis report remains the source of truth for build-stopping errors. - warnings[#warnings + 1] = { + findings.errors[#findings.errors + 1] = { line = a.line, - msg = string.format("'%s' binds '%s' but no Struct_(%s) { ... } declaration found (also flagged as an error by check_abi_handoff in the static-analysis pass)", a.name, a.binds, a.binds), + msg = string.format("'%s': %s", a.name, msg), } end end end - -- 5. BIND writes must be wave-context registers that match Binds_ fields. + -- THE per-annotation pipeline. ONE loop. CHECK_RULES dispatches per_annot rules. for _, a in ipairs(annots) do - if a.binds and binds_index[a.binds] then - local bs = binds_index[a.binds] - - for _, f in ipairs(bs.fields) do - local candidate = "R_" .. f.name - if not is_wave_context_reg(candidate) then - warnings[#warnings + 1] = { - line = bs.line, - msg = string.format("%s field '%s' doesn't match a known wave-context register (candidate '%s')", a.binds, f.name, candidate), - } - end - end - - for _, w in ipairs(a.writes) do - if not is_wave_context_reg(w) then - warnings[#warnings + 1] = { - line = a.line, - msg = string.format("%s writes '%s' which is not a known wave-context register", a.name, w), - } - end - end + for _, rule in ipairs(CHECK_RULES) do + if rule.per_annot then rule.per_annot(a, pipe_ctx, findings) end end end - -- 6. INFO reads should be wave-context registers (or R_TapePtr for rbind). - for _, a in ipairs(annots) do - for _, r in ipairs(a.reads) do - if not is_wave_context_reg(r) and r ~= "R_TapePtr" then - warnings[#warnings + 1] = { - line = a.line, - msg = string.format("atom '%s' reads '%s' which is not a known wave-context register", a.name, r), - } - end - end + -- Post-loop rules (one-shot checks that need full-corpus aggregation in pipe_ctx). + for _, rule in ipairs(CHECK_RULES) do + if rule.post then rule.post(pipe_ctx, findings) end end - -- 7. TAPE_WORDS(mac_X, N) ↔ WORD_COUNT(mac_X, N) drift. - -- Three outcomes: missing (error), mismatch (error), match (info). - local function check_macro_drift(m, declared) - if not declared then - errors[#errors + 1] = { - line = m.line, - msg = string.format("TAPE_WORDS(%s, %d) but '%s' is not in metadata.h", m.name, m.words, m.name), - } - return - end - if declared ~= m.words then - errors[#errors + 1] = { - line = m.line, - msg = string.format("DRIFT: TAPE_WORDS(%s, %d) but metadata.h declares WORD_COUNT(%s, %d)", m.name, m.words, m.name, declared), - } - return - end - info[#info + 1] = { - line = m.line, - msg = string.format("OK: %s = %d words", m.name, m.words), - } - end + -- Per-macro rules (TAPE_WORDS vs WORD_COUNT drift). + local wc = ctx.shared.word_counts for _, m in ipairs(scan.macros) do - check_macro_drift(m, ctx.shared.word_counts[m.name]) + for _, rule in ipairs(CHECK_RULES) do + if rule.per_macro then rule.per_macro(m, wc, findings) end + end end - -- 8. Information summary. - info[#info + 1] = { + -- Information summary (always emitted). + findings.info[#findings.info + 1] = { line = 0, msg = string.format("scanned: %d atom(s), %d annotation(s), %d macro-word-decl(s), %d binds struct(s)", #atoms, #annots, #scan.macros, #scan.binds), @@ -257,9 +339,9 @@ local function validate(ctx, src) annots = annots, macros = scan.macros, binds = scan.binds, - errors = errors, - warnings = warnings, - info = info, + errors = findings.errors, + warnings = findings.warnings, + info = findings.info, } end @@ -300,7 +382,7 @@ end --- Stash aggregated per-module results for the report pass to consume. local function emit_module_annotations_stub(ctx, dir, dir_basename, atoms_count) - ctx.flags = ctx.flags or {} + ctx.flags = ctx.flags or {} ctx.flags._annot_results = ctx.flags._annot_results or {} ctx.flags._annot_results[#ctx.flags._annot_results + 1] = { dir = dir, @@ -339,20 +421,20 @@ function M.run(ctx) local dir_errors = {} local dir_warnings = {} -- Per-source validate() results, cached for the report pass (it reads from this instead of re-validating each source). - ctx.flags = ctx.flags or {} + ctx.flags = ctx.flags or {} ctx.flags._annot_source_results = ctx.flags._annot_source_results or {} for _, src in ipairs(dir_sources) do - local result = validate(ctx, src) + local result = validate(ctx, src) result.source = src.path -- tag for downstream rendering ctx.flags._annot_source_results[src.path] = result -- stash so report.lua reads from cache instead of re-running validate() dir_atoms = dir_atoms + #result.atoms for _, e in ipairs(result.errors) do dir_errors[#dir_errors + 1] = { line = e.line, msg = e.msg, source = src.path } - errors[#errors + 1] = { line = e.line, msg = e.msg } + errors [#errors + 1] = { line = e.line, msg = e.msg } end for _, w in ipairs(result.warnings) do dir_warnings[#dir_warnings + 1] = { line = w.line, msg = w.msg } - warnings[#warnings + 1] = { line = w.line, msg = w.msg } + warnings [#warnings + 1] = { line = w.line, msg = w.msg } end end diff --git a/scripts/passes/components.lua b/scripts/passes/components.lua index a12123a..708551d 100644 --- a/scripts/passes/components.lua +++ b/scripts/passes/components.lua @@ -30,8 +30,8 @@ -- both standalone and when require'd from the orchestrator. -- Bootstrap: load `duffle_paths.lua` via `debug.getinfo(1, "S").source` (works both standalone + when require'd). -- duffle_paths.lua sets package.path then returns `require("duffle")` at the bottom, so the dofile value IS the duffle module. -local _bootstrap_dir = debug.getinfo(1, "S").source:match("^@?(.*[/\\])") or "./" -local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua") +local _bootstrap_dir = debug.getinfo(1, "S").source:match("^@?(.*[/\\])") or "./" +local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua") local word_count_eval = require("word_count_eval") -- ════════════════════════════════════════════════════════════════════════════ @@ -107,13 +107,14 @@ local M = {} -- @param before_pos integer -- @return integer|nil local function find_last_name_open_paren(source, name, before_pos) - local search = source:sub(1, before_pos) local name_open = name .. "(" local last_idx = nil local scan_pos = 1 while true do - local found = search:find(name_open, scan_pos, true) -- plain (no regex) - if not found then break end + -- Pass `before_pos + 1` so string.find only returns positions < before_pos + 1 + -- (string.find's 4th arg `plain` is true; we use the 3rd arg `init` for the upper bound). + local found = source:find(name_open, scan_pos, true) + if not found or found >= before_pos then break end last_idx = found scan_pos = found + #name_open end @@ -126,15 +127,15 @@ end --- Convention: function form is --- `FI_ MipsAtom ac_X(args) MipsAtomComp_Proc_(ac_X, { body })` --- We find the LAST occurrence of `"ac_X("` before `before_pos` and extract the args from inside the parens. ---- We then verify the preceding context ends with `MipsAtom` (the function-decl keyword ---- with possible qualifiers between). +--- We then verify the preceding context ends with `MipsAtom` +--- (the function-decl keyword with possible qualifiers between). --- --- @param source string --- @param name string --- @param before_pos integer --- @return string|nil local function find_function_args_for(source, name, before_pos) - local last_idx = find_last_name_open_paren(source, name, before_pos) + local last_idx = find_last_name_open_paren(source, name, before_pos) if not last_idx then return nil end -- Verify the preceding context ends with "MipsAtom" (with possible qualifiers between). @@ -224,7 +225,7 @@ end -- ends at `close_end_pos`. Returns (block_text, new_scan_pos) where `new_scan_pos` -- is where to continue scanning for more comments, or nil if no block comment was found. local function capture_block_comment(source, close_end_pos) - local open_at = find_block_comment_open(source, close_end_pos) + local open_at = find_block_comment_open(source, close_end_pos) if not open_at then return nil end local block_start = extend_left_over_indent(source, open_at) return source:sub(block_start, close_end_pos), block_start @@ -255,18 +256,18 @@ local function preceding_comment_block(source, pos) local pieces = {} while true do local non_ws = skip_ws_backward(source, scan_pos) - if non_ws == 0 then break end + if non_ws == 0 then break end local is_block_close = non_ws >= 2 and source:sub(non_ws - 1, non_ws) == "*/" local is_line_end = source:sub(non_ws, non_ws) == "\n" or source:sub(non_ws, non_ws) == "\r" if is_block_close then - local block_text, new_scan_pos = capture_block_comment(source, non_ws) + local block_text, new_scan_pos = capture_block_comment(source, non_ws) if not block_text then break end table.insert(pieces, 1, block_text) scan_pos = new_scan_pos elseif is_line_end then - local line_text, new_scan_pos = capture_line_comment(source, non_ws) + local line_text, new_scan_pos = capture_line_comment(source, non_ws) if not line_text then break end table.insert(pieces, 1, line_text) scan_pos = new_scan_pos @@ -282,8 +283,8 @@ end -- Argument-name extraction -- ════════════════════════════════════════════════════════════════════════════ --- Walk `trimmed` backward from `pos` over trailing whitespace / asterisks / brackets, returning the position of the first --- non-trailer character (i.e. the end of the identifier). +-- Walk `trimmed` backward from `pos` over trailing whitespace / asterisks / brackets, +-- returning the position of the first non-trailer character (i.e. the end of the identifier). -- @param trimmed string -- @param pos integer -- @return integer @@ -322,9 +323,6 @@ end --- `"U4 off, U4 code, U1 r, U1 g, U1 b"` -> `{"off", "code", "r", "g", "b"}` --- `"U4 *ptr"` -> `{"ptr"}` --- `""` -> nil ---- ---- No regex — uses `duffle.is_alnum` + plain string ops. ---- --- @param args_str string|nil --- @return string[]|nil local function extract_arg_names(args_str) @@ -334,9 +332,9 @@ local function extract_arg_names(args_str) for _, tok in ipairs(tokens) do local trimmed = duffle.trim(tok) if trimmed ~= "" then - local ident_end = trim_trailer_back(trimmed, #trimmed) + local ident_end = trim_trailer_back(trimmed, #trimmed) local ident_start = trim_ident_back(trimmed, ident_end) + 1 - local name = trimmed:sub(ident_start, ident_end) + local name = trimmed:sub(ident_start, ident_end) if name ~= "" then names[#names + 1] = name end end end @@ -379,7 +377,6 @@ end -- ════════════════════════════════════════════════════════════════════════════ -- Convert `//` line comments to `/* */` block comments in a token. --- -- C macros use `\` line-continuations; a `//` comment before `\` would consume the continuation, -- breaking the macro. We convert `//` to `/* */` so the multi-line macro structure is preserved. -- @@ -403,8 +400,8 @@ local function convert_line_comments_to_block(s) while eol <= len and result:byte(eol) ~= BYTE_NEWLINE do eol = eol + 1 end - local before = result:sub(1, pos - 1) - local comment = result:sub(pos + 2, eol - 1) -- skip the `//` + local before = result:sub(1, pos - 1) + local comment = result:sub(pos + 2, eol - 1) -- skip the `//` local after if eol <= len and result:byte(eol) == BYTE_NEWLINE then after = " */" .. result:sub(eol) -- keep the newline @@ -422,8 +419,9 @@ end -- Word-count computation (memoized recursive lookup) -- ════════════════════════════════════════════════════════════════════════════ --- Strip the `mac_` prefix from a component-call ident so we can look it up against the components-by-name table. Returns the ident unchanged --- if it doesn't start with the prefix (so a non-component ident like `mask_upper` falls through to the wc-table branch). +-- Strip the `mac_` prefix from a component-call ident so we can look it up against the components-by-name table. +-- Returns the ident unchanged if it doesn't start with the prefix +-- (so a non-component ident like `mask_upper` falls through to the wc-table branch). -- @param ident string|nil -- @return string|nil local function strip_mac_prefix(ident) @@ -476,8 +474,7 @@ end --- Compute word counts for every component in `components` in a single pass. --- The name-lookup table + memoization cache are built ONCE (per source) instead of per-component, --- so the cache survives across siblings and a component's recursive `mac_Y(...)` references hit memoized values ---- instead of re-walking the body. Previously each call rebuilt both tables (O(N) tables per call → O(N^2)). ---- +--- instead of re-walking the body. --- Cycle detection (A -> B -> A) is preserved via the in-progress `-1` sentinel in `cache`. --- --- @param components Component[] @@ -530,7 +527,7 @@ end --- @return string local function signature_from_args(args_str) local arg_names = extract_arg_names(args_str) - if arg_names and #arg_names > 0 then + if arg_names and #arg_names > 0 then return table.concat(arg_names, ", ") end return "..." @@ -540,7 +537,7 @@ end --- The last 2 chars are always that pair. local function strip_trailing_continuation(lines) local last = lines[#lines] - if last:sub(-2) == " \\" then + if last:sub(-2) == " \\" then lines[#lines] = last:sub(1, -3) end end @@ -627,14 +624,13 @@ end -- @return string -- the output directory -- @return string -- the full output path local function compute_macs_h_path(src) - local out_dir = src.dir .. "/" .. GEN_SUBDIR - local out_path = out_dir .. "/" .. duffle.basename_no_ext(src.dir) .. ".macs.h" + local out_dir = src.dir .. "/" .. GEN_SUBDIR + local out_path = out_dir .. "/" .. duffle.basename_no_ext(src.dir) .. ".macs.h" return out_dir, out_path end ---- Emit a per-source `.macs.h` header with the `mac_X` macros + `WORD_COUNT` entries. Writes in BINARY mode so LF line endings are ---- preserved (the git blob is LF; Windows text-mode would emit CRLF and break the byte-identical diff). ---- +--- Emit a per-source `.macs.h` header with the `mac_X` macros + `WORD_COUNT` entries. +--- Writes in BINARY mode so LF line endings are preserved (the git blob is LF; Windows text-mode would emit CRLF and break the byte-identical diff). --- Honors `ctx.dry_run`: prints the intended path but does not write the file. --- --- @param ctx PassCtx @@ -644,7 +640,6 @@ end --- @return string|nil -- path to the written file (nil if no components) local function emit_component_macros_h(ctx, src, components, counts) if #components == 0 then return nil end - local out_dir, out_path = compute_macs_h_path(src) local lines = header_boilerplate(src) @@ -655,7 +650,6 @@ local function emit_component_macros_h(ctx, src, components, counts) end local content = table.concat(lines, "\n") .. "\n" - if ctx.dry_run then print(string.format(" -> %s (dry-run)", out_path)) return out_path diff --git a/scripts/passes/offsets.lua b/scripts/passes/offsets.lua index a29e017..43495d4 100644 --- a/scripts/passes/offsets.lua +++ b/scripts/passes/offsets.lua @@ -132,6 +132,14 @@ local function record_offset_marker(branches, args, at_pos) end end +-- MARKER_TO_HANDLER — data-driven marker dispatch (the plex pattern). +-- Maps the marker ident to its recorder function. Each handler takes (out_table, args, at_pos). +-- Adding a new marker type = 1 row + 1 recorder function. +local MARKER_TO_HANDLER = { + [LABEL_MARKER] = record_label_marker, + [OFFSET_MARKER] = record_offset_marker, +} + --- Scan a single token for atom_label/atom_offset markers, walking through balanced groups transparently (so nested calls are found). --- @param token string --- @param at_pos integer -- the branch-free word position of this token in the body @@ -146,13 +154,13 @@ local function scan_for_atom_markers(token, at_pos, labels, branches) local ch = token:sub(pos, pos) if duffle.is_alpha(ch) then local ident, after = duffle.read_ident(token, pos) - if ident == LABEL_MARKER then + local handler = MARKER_TO_HANDLER[ident] + if handler then local args, after_paren = extract_ident_args(token, after) - record_label_marker(labels, args, at_pos) - pos = after_paren or after - elseif ident == OFFSET_MARKER then - local args, after_paren = extract_ident_args(token, after) - record_offset_marker(branches, args, at_pos) + -- Marker found — dispatch to its recorder. markers share labels and branches as + -- out-tables; the recorder picks which one(s) to write to based on its semantics. + -- (record_label_marker writes to labels; record_offset_marker writes to branches.) + handler(ident == LABEL_MARKER and labels or branches, args, at_pos) pos = after_paren or after else pos = after diff --git a/scripts/passes/report.lua b/scripts/passes/report.lua index ff6db84..c1b1aaa 100644 --- a/scripts/passes/report.lua +++ b/scripts/passes/report.lua @@ -261,6 +261,28 @@ local function render_module_warnings_section(add, results, total_warnings) add("") end +-- ════════════════════════════════════════════════════════════════════════════ +-- SECTION_RENDERERS — data-driven section dispatch (the plex pattern) +-- ════════════════════════════════════════════════════════════════════════════ +-- +-- Each entry maps a section to its (header, render_fn). The render_fn signature: +-- render_fn(add, results, totals) +-- add -- the `add(line)` closure from the surrounding report renderer +-- results -- AnnotationResult[] (per-source results) +-- totals -- {atoms, annots, binds, macros, errors, warnings} counts +-- +-- Sections that need to render "(none)" vs iterate use totals.errors / totals.warnings; +-- other sections ignore the totals arg. +-- Adding a new section = 1 row here + 1 render__section function. +local SECTION_RENDERERS = { + { header = SECTION_HEADER_ATOMS, render = render_module_atoms_section }, + { header = SECTION_HEADER_ANNOTS, render = render_module_annots_section }, + { header = SECTION_HEADER_BINDS, render = render_module_binds_section }, + { header = SECTION_HEADER_MACROS, render = render_module_macros_section }, + { header = SECTION_HEADER_ERRORS, render = function(add, results, totals) return render_module_errors_section(add, results, totals.errors) end }, + { header = SECTION_HEADER_WARNINGS, render = function(add, results, totals) return render_module_warnings_section(add, results, totals.warnings) end }, +} + --- Render the per-MODULE annotation report (one `.annotations.txt`). --- @param dir string -- module directory path --- @param sources SourceFile[] -- sources in this module @@ -282,12 +304,22 @@ local function render_module_report(dir, sources, results) total_atoms, total_annots, total_binds, total_macros)) add("") - render_module_atoms_section(add, results) - render_module_annots_section(add, results) - render_module_binds_section(add, results) - render_module_macros_section(add, results) - render_module_errors_section(add, results, total_errors) - render_module_warnings_section(add, results, total_warnings) + -- Bundle the totals so the section renderers don't need separate parameter lists. + -- Errors/warnings sections need their total count to decide "(none)" vs iterate. + -- Sections without totals (atoms/annots/binds/macros) ignore this arg. + local totals = { + atoms = total_atoms, annots = total_annots, binds = total_binds, + macros = total_macros, errors = total_errors, warnings = total_warnings, + } + + -- THE per-section dispatch. ONE loop over SECTION_RENDERERS. Each renderer writes its + -- header + content via the `add` closure (pre-bound above). + -- Adding a new section = 1 row here + 1 render__section function. + for _, section in ipairs(SECTION_RENDERERS) do + add(section.header) + section.render(add, results, totals) + add("") + end return table.concat(lines, "\n") .. "\n" end @@ -347,7 +379,8 @@ end -- ════════════════════════════════════════════════════════════════════════════ -- (internal) Pull per-source validate() results from the annotation pass's stash. --- The annotation pass runs first in the dep chain and caches results in `ctx.flags._annot_source_results`; we read from there instead of re-validating each source. +-- The annotation pass runs first in the dep chain and caches results in `ctx.flags._annot_source_results`; +-- we read from there instead of re-validating each source. -- Returns the list of module results + the flat list of all results (for the project-wide summary). -- @param ctx PassCtx -- @param dir_sources SourceFile[] diff --git a/scripts/passes/scan_source.lua b/scripts/passes/scan_source.lua index 8f2afc6..0e0eebd 100644 --- a/scripts/passes/scan_source.lua +++ b/scripts/passes/scan_source.lua @@ -1,6 +1,6 @@ --- passes/scan_source.lua — Source pre-scan pass (the "mega entity" pass). --- ---- Single source-walk pass that produces the fat `SourceScan` payload consumed by all downstream passes. Walks each `ctx.sources` entry once, +--- Single source-walk pass that produces the fat `SourceScan` payload consumed by all downstream passes. Walks each `ctx.sources` entry once, --- extracting every construct type the metaprograms need: --- --- MipsAtom_ (kind = "atom", with optional atom_info inner) @@ -14,8 +14,7 @@ --- This is the first pass in the dep graph (no deps). --- Every other pass that reads source structure depends on this one — see `ps1_meta.lua :: PASSES`. --- ---- **Conventions**: tabs (1/level), EmmyLua annotations, no regex, ---- Lua 5.3 compatible. +--- **Conventions**: tabs (1/level), EmmyLua annotations, no regex, Lua 5.3 compatible -- Bootstrap: same as entry scripts. See `ps1_meta.lua` for the rationale. -- Bootstrap: load `scripts/duffle_paths.lua` (sets package.path + package.cpath). @@ -35,7 +34,7 @@ local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua") --- @field raw_atoms AtomEntry[] -- MipsCode code_ { body } (offsets pass only) --- @field binds BindsEntry[] -- typedef Struct_(Binds_X) { fields } (fields pre-parsed) --- @field atom_infos AtomInfoEntry[] -- MipsAtom_(name) atom_info(...) (sub-calls pre-parsed) ---- @field macros MacroEntry[] -- #pragma mac_X tape_atom words=N + _Pragma("...") +--- @field macros MacroEntry[] -- #pragma mac_X tape_atom words=N + _Pragma("...") --- @field line_of fun(pos: integer): integer -- shared LineIndex closure --- @class SourceFile @@ -74,7 +73,7 @@ local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua") --- @field comment string|nil -- populated by components pass (backward lookup) -- ════════════════════════════════════════════════════════════════════════════ --- Local helpers +-- Local helpers (shared by per-form parsers) -- ════════════════════════════════════════════════════════════════════════════ -- C qualifier keywords that may precede a MipsAtom_ / MipsCode declaration. @@ -86,6 +85,22 @@ local QUALIFIER_KEYWORDS = { ["internal"] = true, ["LP_"] = true, ["global"] = true, ["gkknown"] = true, } +-- "ac_" prefix length on component names (e.g., `MipsAtomComp_(ac_X, ...)`). The components pass strips +-- this prefix to derive the macro name (e.g., `mac_X`). Single source of truth — was duplicated in two +-- branches of the pre-refactor scan_source. +local AC_PREFIX = "ac_" +local AC_PREFIX_LEN = 3 + +-- Strip the "ac_" prefix from a component name. Returns the input unchanged if it doesn't start with the prefix. +-- @param raw_name string +-- @return string +local function strip_ac_prefix(raw_name) + if #raw_name > AC_PREFIX_LEN and raw_name:sub(1, AC_PREFIX_LEN) == AC_PREFIX then + return raw_name:sub(AC_PREFIX_LEN + 1) + end + return raw_name +end + -- Parse the U4 fields from a Binds_X body. Returns (fields, byte_count). local function scan_binds_fields(body) local fields = {} @@ -147,13 +162,13 @@ local function scan_atom_info_subcalls(info_inner) if info_inner:sub(sub_open, sub_open) == "(" then local sub_inner, sub_after2 = duffle.read_parens(info_inner, sub_open) -- scan: atom_bind() - binds = duffle.trim(sub_inner) + binds = duffle.trim(sub_inner) sub_pos = sub_after2 else sub_pos = sub_open + 1 end elseif sub_ident == "atom_reads" or sub_ident == "atom_writes" then - local kind = sub_ident + local kind = sub_ident local sub_open = duffle.skip_ws_and_cmt(info_inner, sub_end) if info_inner:sub(sub_open, sub_open) == "(" then local sub_inner, sub_after2 = duffle.read_parens(info_inner, sub_open) @@ -181,6 +196,262 @@ local function scan_skip_qualifiers(source, pos) end end +-- ════════════════════════════════════════════════════════════════════════════ +-- Per-form parsers (the DECL_PARSERS table's payload) +-- ════════════════════════════════════════════════════════════════════════════ +-- +-- Each parser has the uniform signature: +-- parser(source, pos, ident_end, line_of, out) -> new_pos +-- where: +-- source -- the full source text +-- pos -- position of the construct's leading ident (e.g., `M` of `MipsAtom_`) +-- ident_end -- position past the leading ident (where the `(` should be) +-- line_of -- closure over LineIndex(source) for 1-based line lookups +-- out -- the SourceScan out table (mutated in place: out.atoms / out.raw_atoms / out.binds / out.atom_infos / out.macros) +-- returns -- new position after the construct +-- +-- All parsers read source-as-written via the duffle primitives (skip_ws_and_cmt / read_parens / read_braces / read_balanced). +-- No regex per the no_regex constraint; no hand-rolled depth tracking (the MipsAtomComp_Proc_ brace matcher +-- now uses duffle.read_braces instead of bespoke byte-dispatch). +-- +-- Adding a new construct = 1 row in DECL_PARSERS + 1 parser function. The scan_source() loop never needs editing. + +--- Parse: `MipsAtom_() [atom_info(, , )] { }` +--- @param source string +--- @param pos integer +--- @param ident_end integer +--- @param line_of fun(pos: integer): integer +--- @param out SourceScan +--- @return integer +local function parse_mips_atom(source, pos, ident_end, line_of, out) + local open_paren = duffle.skip_ws_and_cmt(source, ident_end) + if source:sub(open_paren, open_paren) ~= "(" then return open_paren + 1 end + local inner, after_paren = duffle.read_parens(source, open_paren) + + local raw_name = duffle.read_ident(inner, 1) + + -- Lookahead for atom_info(...) between `)` and `{`. Captures sub-calls; updates brace search start. + local brace_search_pos = after_paren + local lookahead = duffle.skip_ws_and_cmt(source, after_paren) + local look_ident, look_end = duffle.read_ident(source, lookahead) + if look_ident == "atom_info" then + local info_open = duffle.skip_ws_and_cmt(source, look_end) + if source:sub(info_open, info_open) == "(" then + local info_inner, info_after = duffle.read_parens(source, info_open) + local ai_binds, ai_reads, ai_writes = scan_atom_info_subcalls(info_inner) + out.atom_infos[#out.atom_infos + 1] = { + atom_name = raw_name or "?", binds = ai_binds, + reads = ai_reads or {}, writes = ai_writes or {}, + info_line = line_of(lookahead), + } + brace_search_pos = info_after + end + end + + local brace = duffle.scan_to_char(source, "{", brace_search_pos) + if not brace then return open_paren + 1 end + + local body, after_brace = duffle.read_braces(source, brace) + if raw_name and raw_name ~= "" then + out.atoms[#out.atoms + 1] = { + line = line_of(pos), name = raw_name, body = body, body_off = brace + 1, + kind = "atom", raw_name = raw_name, + ident_pos = pos, after_paren = after_paren, + } + end + + return after_brace +end + +--- Parse: `MipsAtomComp_() { }` +--- @param source string +--- @param pos integer +--- @param ident_end integer +--- @param line_of fun(pos: integer): integer +--- @param out SourceScan +--- @return integer +local function parse_mips_atom_comp(source, pos, ident_end, line_of, out) + local open_paren = duffle.skip_ws_and_cmt(source, ident_end) + if source:sub(open_paren, open_paren) ~= "(" then return open_paren + 1 end + local inner, after_paren = duffle.read_parens(source, open_paren) + + local raw_name = duffle.read_ident(inner, 1) + if not raw_name then return open_paren + 1 end + + local brace = duffle.scan_to_char(source, "{", after_paren) + if not brace then return open_paren + 1 end + + local body, after_brace = duffle.read_braces(source, brace) + + out.atoms[#out.atoms + 1] = { + line = line_of(pos), name = strip_ac_prefix(raw_name), body = body, body_off = brace + 1, + kind = "comp_bare", raw_name = raw_name, + ident_pos = pos, after_paren = after_paren, + } + + return after_brace +end + +--- Parse: `MipsAtomComp_Proc_(, { })` — body is inside the LAST `{` in args. +--- @param source string +--- @param pos integer +--- @param ident_end integer +--- @param line_of fun(pos: integer): integer +--- @param out SourceScan +--- @return integer +local function parse_mips_atom_comp_proc(source, pos, ident_end, line_of, out) + local open_paren = duffle.skip_ws_and_cmt(source, ident_end) + if source:sub(open_paren, open_paren) ~= "(" then return open_paren + 1 end + local inner, after_paren = duffle.read_parens(source, open_paren) + + -- Find the LAST `{` in inner (the body brace, not any potential embedded braces in expressions). + local last_brace_pos = nil + for search_pos = #inner, 1, -1 do + if inner:sub(search_pos, search_pos) == "{" then last_brace_pos = search_pos; break end + end + if not last_brace_pos then return after_paren end + + -- Use duffle.read_braces to find the matching close brace. + -- Replaces the pre-refactor hand-rolled depth tracker (~25 LOC of `if c == 123 then depth = depth + 1 ...`). + -- If close_pos is past the end of inner, the brace didn't match (malformed input); skip. + local body, close_pos = duffle.read_braces(inner, last_brace_pos) + if close_pos > #inner + 1 then return after_paren end + + local raw_name = inner:match("^%s*([%w_]+)") or "?" + -- Position of body[1] in source = open_paren + 1 (start of inner) + last_brace_pos + 1 (past '{'). + local body_off = open_paren + 2 + last_brace_pos + + out.atoms[#out.atoms + 1] = { + line = line_of(pos), name = strip_ac_prefix(raw_name), body = body, body_off = body_off, + kind = "comp_proc", raw_name = raw_name, + ident_pos = pos, after_paren = after_paren, + } + + return after_paren +end + +--- Parse: `MipsCode code_ { }` (raw atom form — offsets pass only). +--- @param source string +--- @param pos integer +--- @param ident_end integer +--- @param line_of fun(pos: integer): integer +--- @param out SourceScan +--- @return integer +local function parse_mips_code(source, pos, ident_end, line_of, out) + local next_pos = duffle.skip_ws_and_cmt(source, ident_end) + local next_ident, next_after = duffle.read_ident(source, next_pos) + if not next_ident or #next_ident <= 5 or next_ident:sub(1, 5) ~= "code_" then + return ident_end + end + + local atom_name = next_ident:sub(6) + local brace_pos = duffle.scan_to_char(source, "{", next_after) + if not brace_pos then return ident_end end + + local body, after_brace = duffle.read_braces(source, brace_pos) + out.raw_atoms[#out.raw_atoms + 1] = { + line = line_of(pos), name = atom_name, body = body, body_off = brace_pos + 1, + kind = "raw_atom", raw_name = atom_name, + } + + return after_brace +end + +--- Parse: `typedef Struct_() { }` — only emits when name starts with `Binds_`. +--- @param source string +--- @param pos integer +--- @param ident_end integer +--- @param line_of fun(pos: integer): integer +--- @param out SourceScan +--- @return integer +local function parse_typedef_binds(source, pos, ident_end, line_of, out) + local after_typedef = duffle.skip_ws_and_cmt(source, ident_end) + local id2, id2_end = duffle.read_ident(source, after_typedef) + if id2 ~= "Struct_" then return ident_end end + + local open_paren = duffle.skip_ws_and_cmt(source, id2_end) + if source:sub(open_paren, open_paren) ~= "(" then return id2_end end + + local inner, after_paren = duffle.read_parens(source, open_paren) + local name = duffle.trim(inner) + + local brace = duffle.scan_to_char(source, "{", after_paren) + if not brace then return open_paren + 1 end + + local body, after_brace = duffle.read_braces(source, brace) + if name:sub(1, 6) == "Binds_" then + local fields, byte_off = scan_binds_fields(body) + out.binds[#out.binds + 1] = { line = line_of(pos), name = name, fields = fields, bytes = byte_off } + end + + return after_brace +end + +--- Parse: `_Pragma("mac_X tape_atom words=N")` (operator form). +--- @param source string +--- @param pos integer +--- @param ident_end integer +--- @param line_of fun(pos: integer): integer +--- @param out SourceScan +--- @return integer +local function parse_pragma_macro(source, pos, ident_end, line_of, out) + local open_paren = duffle.skip_ws_and_cmt(source, ident_end) + if source:sub(open_paren, open_paren) ~= "(" then return open_paren + 1 end + + local str, str_end = duffle.read_parens(source, open_paren) + str = duffle.trim(str) + if str:sub(1, 1) ~= '"' or str:sub(-1) ~= '"' then return str_end end + + local inner = str:sub(2, -2) + local space = duffle.find_byte(inner, 32, 1) + if not space then return str_end end + + local name = inner:sub(1, space - 1) + local rest = inner:sub(space + 1) + local eq = duffle.find_byte(rest, 61, 1) + if not eq then return str_end end + + local key = duffle.trim(rest:sub(1, eq - 1)) + local val = duffle.trim(rest:sub(eq + 1)) + if key == "tape_atom words" or key == "words" then + out.macros[#out.macros + 1] = { line = line_of(pos), name = name, words = tonumber(val) or 0 } + end + + return str_end +end + +--- Parse: `pragma` ident (no-op — directive form `#pragma` is handled by `skip_preprocessor_line` upstream). +--- If we reach this parser it means the directive skip didn't fire, which can happen for non-#-prefixed pragma. +--- Just advance past the ident. +--- @param source string +--- @param pos integer +--- @param ident_end integer +--- @param line_of fun(pos: integer): integer +--- @param out SourceScan +--- @return integer +local function parse_pragma_dummy(source, pos, ident_end, line_of, out) + return ident_end +end + +-- ════════════════════════════════════════════════════════════════════════════ +-- DECL_PARSERS — data-driven construct dispatch (the plex pattern) +-- ════════════════════════════════════════════════════════════════════════════ +-- +-- Each entry maps a leading ident to its parser function. The main scan_source() loop is one line of dispatch: +-- local parser = DECL_PARSERS[ident]; if parser then pos = parser(...) end +-- +-- Adding a new construct = 1 row here + 1 parser function above. + +local DECL_PARSERS = { + MipsAtom_ = parse_mips_atom, + MipsAtomComp_ = parse_mips_atom_comp, + MipsAtomComp_Proc_ = parse_mips_atom_comp_proc, + MipsCode = parse_mips_code, + typedef = parse_typedef_binds, + _Pragma = parse_pragma_macro, + pragma = parse_pragma_dummy, +} + -- ════════════════════════════════════════════════════════════════════════════ -- The single source walker -- ════════════════════════════════════════════════════════════════════════════ @@ -191,13 +462,16 @@ end --- @return table -- SourceScan { atoms, raw_atoms, binds, atom_infos, macros, line_of } local function scan_source(source) local line_of = duffle.LineIndex(source) - local atoms = {} - local raw_atoms = {} - local binds = {} - local atom_infos = {} - local macros = {} - local pos = 1 - local src_len = #source + local out = { + atoms = {}, + raw_atoms = {}, + binds = {}, + atom_infos = {}, + macros = {}, + line_of = line_of, + } + local pos = 1 + local src_len = #source while pos <= src_len do pos = duffle.skip_ws_and_cmt(source, pos) @@ -206,250 +480,29 @@ local function scan_source(source) -- Skip preprocessor directives (#define / #include / #pragma / etc). -- _Pragma is an operator (not a directive) — it doesn't start with #. local pp_pos = duffle.skip_preprocessor_line(source, pos) - if pp_pos then pos = pp_pos; goto continue end - - -- Skip C qualifiers (static, const, etc.) that may precede a declaration. - pos = scan_skip_qualifiers(source, pos) - if pos > src_len then break end - - local ident, ident_end = duffle.read_ident(source, pos) - -- scan: - if not ident then pos = pos + 1; goto continue end - - -- ── MipsAtom_ / MipsAtomComp_ / MipsAtomComp_Proc_ ── - if ident == "MipsAtom_" or ident == "MipsAtomComp_" or ident == "MipsAtomComp_Proc_" then - local is_atom = ident == "MipsAtom_" - local is_comp = ident == "MipsAtomComp_" - local is_proc = ident == "MipsAtomComp_Proc_" - local kind = is_atom and "atom" or (is_comp and "comp_bare" or "comp_proc") - local open_paren = duffle.skip_ws_and_cmt(source, ident_end) - if source:sub(open_paren, open_paren) ~= "(" then pos = open_paren + 1; goto continue end - - local inner, after_paren = duffle.read_parens(source, open_paren) - -- scan: () - - if is_proc then - -- MipsAtomComp_Proc_(name, { body }) — body is inside the LAST { } in args. - local last_brace_pos - for search_pos = #inner, 1, -1 do - if inner:sub(search_pos, search_pos) == "{" then last_brace_pos = search_pos; break end - end - if last_brace_pos then - local depth = 1 - local inner_pos = last_brace_pos + 1 - while inner_pos <= #inner and depth > 0 do - local c = inner:byte(inner_pos) - if c == 123 then depth = depth + 1; inner_pos = inner_pos + 1 - elseif c == 125 then depth = depth - 1; if depth == 0 then break end; inner_pos = inner_pos + 1 - elseif c == 40 then local _, a = duffle.read_parens(inner, inner_pos); inner_pos = a - elseif c == 91 then local _, a = duffle.read_brackets(inner, inner_pos); inner_pos = a - elseif c == 34 or c == 39 then inner_pos = duffle.skip_str_or_cmt(inner, inner_pos) + 1 - else inner_pos = inner_pos + 1 end - end - if depth == 0 then - -- scan: (, { }) - local name_match = inner:match("^%s*([%w_]+)") - local raw_name = name_match or "?" - -- Strip "ac_" prefix for component names (components pass convention). - local name = raw_name - if #raw_name > 3 and raw_name:sub(1, 3) == "ac_" then - name = raw_name:sub(4) - end - local body = inner:sub(last_brace_pos + 1, inner_pos - 1) - local body_off = open_paren + 1 + last_brace_pos - atoms[#atoms + 1] = { - line = line_of(pos), name = name, body = body, body_off = body_off + 1, - kind = kind, raw_name = raw_name, - ident_pos = pos, after_paren = after_paren, - args = nil, comment = nil, - } - end - end - pos = after_paren - else - -- MipsAtom_(name) { body } OR MipsAtomComp_(name) { body } - local name_start = 1 - while name_start <= #inner and inner:sub(name_start, name_start):match("[%s]") do name_start = name_start + 1 end - local name_end = name_start - while name_end <= #inner and inner:sub(name_end, name_end):match("[%w_]") do name_end = name_end + 1 end - local raw_name = inner:sub(name_start, name_end - 1) - -- scan: () - if raw_name ~= "" then - local brace = duffle.scan_to_char(source, "{", after_paren) - -- scan: () { - if brace then - local body, after_brace = duffle.read_braces(source, brace) - -- scan: () { } - -- Strip "ac_" prefix for component names (components pass convention). - local disp_name = raw_name - if is_comp and #raw_name > 3 and raw_name:sub(1, 3) == "ac_" then - disp_name = raw_name:sub(4) - end - atoms[#atoms + 1] = { - line = line_of(pos), name = disp_name, body = body, body_off = brace + 1, - kind = kind, raw_name = raw_name, - ident_pos = pos, after_paren = after_paren, - args = nil, comment = nil, - } - pos = after_brace + if pp_pos then + pos = pp_pos + else + -- Skip C qualifiers (static, const, etc.) that may precede a declaration. + pos = scan_skip_qualifiers(source, pos) + if pos <= src_len then + local ident, ident_end = duffle.read_ident(source, pos) + if ident then + local parser = DECL_PARSERS[ident] + if parser then + pos = parser(source, pos, ident_end, line_of, out) else - pos = open_paren + 1 + -- Unrecognized ident — advance past it. + pos = ident_end end else - pos = open_paren + 1 + pos = pos + 1 end end - - -- For MipsAtom_ entries: check if atom_info(...) follows. - if is_atom then - local lookahead = duffle.skip_ws_and_cmt(source, after_paren) - local look_ident, look_end = duffle.read_ident(source, lookahead) - -- scan: MipsAtom_() - if look_ident == "atom_info" then - local info_open = duffle.skip_ws_and_cmt(source, look_end) - if source:sub(info_open, info_open) == "(" then - local info_inner, info_after = duffle.read_parens(source, info_open) - -- scan: MipsAtom_() atom_info(, , ) - -- Find the atom name from the just-parsed atom entry (last one added). - local last_atom = atoms[#atoms] - local atom_name = last_atom and last_atom.raw_name or "?" - local ai_binds, ai_reads, ai_writes = scan_atom_info_subcalls(info_inner) - atom_infos[#atom_infos + 1] = { - atom_name = atom_name, binds = ai_binds, - reads = ai_reads or {}, writes = ai_writes or {}, - info_line = line_of(lookahead), - } - -- Don't advance pos past info_after — the body { ... } still needs to be skipped - -- by the brace scan below. But if there's no body (forward decl), advance. - local body_brace = duffle.scan_to_char(source, "{", info_after) - if body_brace then - local _, after_body = duffle.read_braces(source, body_brace) - pos = after_body - else - pos = info_after - end - end - end - end - goto continue end - - -- ── MipsCode code_ { body } (raw atom form — offsets pass only) ── - if ident == "MipsCode" then - local next_pos = duffle.skip_ws_and_cmt(source, ident_end) - local next_ident, next_after = duffle.read_ident(source, next_pos) - -- scan: MipsCode - if next_ident and #next_ident > 5 and next_ident:sub(1, 5) == "code_" then - local atom_name = next_ident:sub(6) - -- scan: MipsCode code_ - local brace_pos = duffle.scan_to_char(source, "{", next_after) - -- scan: MipsCode code_ { - if brace_pos then - local body, after_brace = duffle.read_braces(source, brace_pos) - -- scan: MipsCode code_ { } - raw_atoms[#raw_atoms + 1] = { - line = line_of(pos), name = atom_name, body = body, body_off = brace_pos + 1, - kind = "raw_atom", raw_name = atom_name, - } - pos = after_brace - goto continue - end - end - pos = ident_end - goto continue - end - - -- ── typedef Struct_(Binds_X) { fields } ── - if ident == "typedef" then - local after_typedef = duffle.skip_ws_and_cmt(source, ident_end) - local id2, id2_end = duffle.read_ident(source, after_typedef) - -- scan: typedef - if id2 == "Struct_" then - local open_paren = duffle.skip_ws_and_cmt(source, id2_end) - if source:sub(open_paren, open_paren) == "(" then - local inner, after_paren = duffle.read_parens(source, open_paren) - -- scan: typedef Struct_() - local name = duffle.trim(inner) - local brace = duffle.scan_to_char(source, "{", after_paren) - -- scan: typedef Struct_() { - if brace then - local body, after_brace = duffle.read_braces(source, brace) - -- scan: typedef Struct_() { } - if name:sub(1, 6) == "Binds_" then - local fields, byte_off = scan_binds_fields(body) - binds[#binds + 1] = { line = line_of(pos), name = name, fields = fields, bytes = byte_off } - end - pos = after_brace - goto continue - end - pos = open_paren + 1 - goto continue - end - pos = id2_end or (after_typedef + 1) - goto continue - end - pos = ident_end - goto continue - end - - -- ── _Pragma("mac_X tape_atom words=N") (operator form) ── - if ident == "_Pragma" then - local open_paren = duffle.skip_ws_and_cmt(source, ident_end) - if source:sub(open_paren, open_paren) == "(" then - local str, str_end = duffle.read_parens(source, open_paren) - -- scan: _Pragma() - str = duffle.trim(str) - if str:sub(1, 1) == '"' and str:sub(-1) == '"' then - local inner = str:sub(2, -2) - local space = duffle.find_byte(inner, 32, 1) - if space then - local name = inner:sub(1, space - 1) - local rest = inner:sub(space + 1) - local eq = duffle.find_byte(rest, 61, 1) - if eq then - local key = duffle.trim(rest:sub(1, eq - 1)) - local val = duffle.trim(rest:sub(eq + 1)) - if key == "tape_atom words" or key == "words" then - macros[#macros + 1] = { line = line_of(pos), name = name, words = tonumber(val) or 0 } - end - end - end - end - pos = str_end - goto continue - end - pos = open_paren + 1 - goto continue - end - - -- ── #pragma mac_X tape_atom words=N (directive form) ── - -- (preprocessor skip above handles # lines, but pragma is an ident here - -- only if it appeared without a leading # — which happens when the - -- preprocessor skip didn't fire because the # was on a previous line. - -- The annotation pass handles this via its own skip_preprocessor_line, - -- but scan_source handles it here by checking the ident.) - if ident == "pragma" then - -- This shouldn't normally fire — #pragma lines are skipped by - -- skip_preprocessor_line above. If we get here, it's a _Pragma - -- variant or a non-#-prefixed pragma. Just advance. - pos = ident_end - goto continue - end - - -- ── Unrecognized ident — advance past it ── - pos = ident_end - - ::continue:: end - return { - atoms = atoms, - raw_atoms = raw_atoms, - binds = binds, - atom_infos = atom_infos, - macros = macros, - line_of = line_of, - } + return out end -- ════════════════════════════════════════════════════════════════════════════ diff --git a/scripts/passes/static_analysis.lua b/scripts/passes/static_analysis.lua index d829e19..cb09b2c 100644 --- a/scripts/passes/static_analysis.lua +++ b/scripts/passes/static_analysis.lua @@ -14,7 +14,7 @@ --- `["static-analysis"] = { module = "passes.static_analysis", kind = "validation", deps = {"word-counts", "components"}, --- out = { { kind = "report", path_template = "/.static_analysis.txt" } } }` --- ---- **Conventions**: tabs (1/level), EmmyLua annotations, no regex, Lua 5.3 compatible. See `lua.md` in the ps1-ai styleguides. +--- **Conventions**: tabs (1/level), EmmyLua annotations, no regex, Lua 5.3 compatible. -- ════════════════════════════════════════════════════════════════════════════ -- Module-scope requires + package.path setup @@ -114,25 +114,6 @@ local OUTPUT_EXTENSION = ".static_analysis.txt" --- @field findings Finding[] -- findings for this atom --- @field total_cycles integer -- sum of token cycle costs --- ════════════════════════════════════════════════════════════════════════════ --- Source scanning — delegated to duffle.scan_source (ps1_meta.lua pre-scans) --- ════════════════════════════════════════════════════════════════════════════ --- --- The orchestrator calls duffle.scan_source once per source and stashes the fat SourceScan in src.scan. --- validate() below reads from src.scan — no source walking in this pass. - --- ════════════════════════════════════════════════════════════════════════════ --- Body tokenizer (top-level comma splitter + per-token classification) --- ════════════════════════════════════════════════════════════════════════════ - ---- Build a map: `body_relative_char_offset` -> `body_relative_line`. ---- Used by the checks to convert per-token offsets in the body to lina numbers relative to the start of `body`. ---- The atom's source-line of the body-start is added by the caller. ---- --- NOTE: `tokenize_body` and `build_body_line_index` moved to `duffle.lua` as shared --- memoized utilities (`duffle.tokenize_body`, `duffle.build_body_line_index`). --- The local copies were deleted; all callers now use the duffle versions. - -- ════════════════════════════════════════════════════════════════════════════ -- classify_tokens — per-token classification (the plex's pre-computed data layer) -- ════════════════════════════════════════════════════════════════════════════ @@ -156,16 +137,30 @@ local OUTPUT_EXTENSION = ".static_analysis.txt" -- Checks that need "how many nops before token i" use `tok_class.nop_prefix` instead of walking backwards. --- @class TokClass ---- @field ident string -- leading identifier ---- @field nop_words integer -- 0/1/2 ---- @field nop_prefix integer -- consecutive nop words before this token ---- @field is_yield boolean ---- @field is_atom_label boolean ---- @field label_name string|nil -- for atom_label(name) ---- @field is_branch boolean ---- @field branch_label string|false|nil -- for branch_*(..., atom_offset(F, label)) ---- @field is_load_word boolean ---- @field is_store_word boolean +--- @field ident string -- leading identifier +--- @field nop_words integer -- 0/1/2 +--- @field nop_prefix integer -- consecutive nop words before this token +--- @field is_yield boolean +--- @field is_atom_label boolean +--- @field label_name string|nil -- for atom_label(name) +--- @field is_branch boolean +--- @field branch_label string|false|nil -- for branch_*(..., atom_offset(F, label)) +--- @field is_load_word boolean +--- @field is_store_word boolean +--- @field mac_format_shape string|nil -- "f3" / "g4" etc. for mac_format_X_color; nil otherwise +--- @field is_gte_store boolean -- ident matches `mac_gte_store_` +--- @field is_ot_tag boolean -- ident matches `mac_insert_ot_tag_` +--- @field writes_r_prim_cursor boolean -- store_word targeting R_PrimCursor +--- @field reads_r_tape_ptr boolean -- any token referencing R_TapePtr +--- @field o_arg1 string|nil -- first arg of O_(, ) captures; nil for non-O_ tokens +--- @field o_arg2 string|nil -- second arg of O_(, ) captures +--- @field s_arg1 string|nil -- arg of S_() captures; nil for non-S_ tokens + +-- Patterns for O_(, ) and S_() captures. UNANCHORED — the substring can appear +-- anywhere in the token (e.g., `load_word(R_T0, R_TapePtr, O_(Binds_X, field))` matches at position ~24). +-- The binds_name match is deferred to check_abi_handoff (which compares tc.o_arg1 == atom.info.binds). +local O_PATTERN = "O_%(([%w_]+),%s*([%w_]+)%s*%)" +local S_PATTERN = "S_%(([%w_]+)%s*%)" local function classify_tokens(tokens) local n = #tokens @@ -186,6 +181,16 @@ local function classify_tokens(tokens) local is_load_word = ident == "load_word" local is_store_word = ident == "store_word" + -- Per-check pre-computes (R3 lift). Each pre-compute eliminates one per-token regex/string-find + -- call from check_abi_handoff / check_gpu_portstore_shape. + local mac_format_shape = nil + local is_gte_store = false + local is_ot_tag = false + local writes_r_prim_cursor = false + local reads_r_tape_ptr = false + local o_arg1, o_arg2 = nil, nil + local s_arg1 = nil + if ident == "atom_label" then is_atom_label = true label_name = tok:match("^atom_label%s*%(%s*([%w_]+)%s*%)") @@ -194,17 +199,40 @@ local function classify_tokens(tokens) branch_label = tok:match("atom_offset%s*%([^,]+,%s*([%w_]+)%s*%)") or false end + -- mac_format_X_color / mac_gte_store_ / mac_insert_ot_tag_ (used by check_gpu_portstore_shape). + local shape = ident:match("^mac_format_([%w_]+)_color$") + if shape then mac_format_shape = shape end + if ident:match("^mac_gte_store_[%w_]+$") then is_gte_store = true end + if ident:match("^mac_insert_ot_tag_[%w_]+$") then is_ot_tag = true end + + -- O_(, ) / S_() captures (used by check_abi_handoff). + -- Cheap pattern match — anchored, fails fast on non-matching tokens. + o_arg1, o_arg2 = tok:match(O_PATTERN) + if not o_arg1 then s_arg1 = tok:match(S_PATTERN) end + + -- R_TapePtr + R_PrimCursor references (used by check_abi_handoff / check_gpu_portstore_shape). + if tok:find("R_TapePtr", 1, true) then reads_r_tape_ptr = true end + if is_store_word and tok:find("R_PrimCursor", 1, true) then writes_r_prim_cursor = true end + tc[tok_idx] = { - ident = ident, - nop_words = nop_words, - nop_prefix = nop_run, - is_yield = is_yield, - is_atom_label = is_atom_label, - label_name = label_name, - is_branch = is_branch, - branch_label = branch_label, - is_load_word = is_load_word, - is_store_word = is_store_word, + ident = ident, + nop_words = nop_words, + nop_prefix = nop_run, + is_yield = is_yield, + is_atom_label = is_atom_label, + label_name = label_name, + is_branch = is_branch, + branch_label = branch_label, + is_load_word = is_load_word, + is_store_word = is_store_word, + mac_format_shape = mac_format_shape, + is_gte_store = is_gte_store, + is_ot_tag = is_ot_tag, + writes_r_prim_cursor = writes_r_prim_cursor, + reads_r_tape_ptr = reads_r_tape_ptr, + o_arg1 = o_arg1, + o_arg2 = o_arg2, + s_arg1 = s_arg1, } -- Advance the nop run for the NEXT token. if nop_words > 0 then @@ -406,29 +434,28 @@ local function check_abi_handoff(atom, pipe_ctx, findings) local tc = atom.paths.tok_class local found_field_set = {} local found_advance = false - local bind_re = "O_%(" .. binds_name .. ",%s*([%w_]+)%s*%)" - for tok_idx, t in ipairs(tokens) do - local tok = t.tok - if tc[tok_idx].is_load_word then - if tok:find("R_TapePtr", 1, true) and tok:find("O_(" .. binds_name .. ",", 1, true) then - local field = tok:match(bind_re) - -- scan: load_word(R_*, R_TapePtr, O_(, )) - if field then - found_field_set[field] = true - else - local body_line = atom.line + line_in_body[t.rel] - findings[#findings + 1] = { - atom = atom.name, line = body_line, - check = "abi_handoff", kind = "error", - msg = string.format("%s at line %d has load_word(R_TapePtr, O_(%s, )); expected O_(%s, )", - atom.name, body_line, binds_name, binds_name), - } - end + + -- Reads from tc_entry fields pre-computed by classify_tokens (R3 lift). Eliminates 3 per-token + -- string-find/match calls (R_TapePtr + O_(binds_name,...) + bind_re) → 3 O(1) field reads. + for tok_idx = 1, #tokens do + local tc_entry = tc[tok_idx] + -- scan: load_word(R_*, R_TapePtr, O_(, )) + if tc_entry.is_load_word and tc_entry.reads_r_tape_ptr and tc_entry.o_arg1 == binds_name then + local field = tc_entry.o_arg2 + if field then + found_field_set[field] = true + else + local body_line = atom.line + line_in_body[tokens[tok_idx].rel] + findings[#findings + 1] = { + atom = atom.name, line = body_line, + check = "abi_handoff", kind = "error", + msg = string.format("%s at line %d has load_word(R_TapePtr, O_(%s, )); expected O_(%s, )", + atom.name, body_line, binds_name, binds_name), + } end end - if tok:find("R_TapePtr", 1, true) - and tok:find("S_(" .. binds_name .. ")", 1, true) then - -- scan: add_ui_self(R_TapePtr, S_()) + -- scan: add_ui_self(R_TapePtr, S_()) + if tc_entry.reads_r_tape_ptr and tc_entry.s_arg1 == binds_name then found_advance = true end end @@ -468,7 +495,6 @@ end --- - Atoms containing a `mac_(...)` call whose name is not in duffle.GP0_MACRO_CONTRIB emit a "new macro; update duffle.GP0_MACRO_CONTRIB" advisory. --- --- Applies only to `kind = "atom"` (baked atoms). Components don't emit full primitives. ---- Stage 2: signature uniformized to `(atom, pipe_ctx, findings)` — pipe_ctx is ignored here. local function check_gpu_portstore_shape(atom, pipe_ctx, findings) if atom.kind ~= "atom" then return end local tokens = atom.paths.tokens @@ -479,31 +505,30 @@ local function check_gpu_portstore_shape(atom, pipe_ctx, findings) local contrib = 0 local saw_format = false local saw_prim_write = false - for tok_idx, t in ipairs(tokens) do - local tok = t.tok - local ident = tc[tok_idx].ident - -- Match `mac_format__color(...)` and strip `_color` - -- to get the bare shape suffix (f3 / g4 / etc). - local shape = ident:match("^mac_format_([%w_]+)_color$") + + -- Reads from tc_entry fields pre-computed by classify_tokens (R3 lift). Eliminates 4 per-token + -- string matches (mac_format_X_color + mac_gte_store_ + mac_insert_ot_tag_ + R_PrimCursor) + for tok_idx = 1, #tokens do + local tc_entry = tc[tok_idx] + local shape = tc_entry.mac_format_shape if shape and duffle.GP0_CMD_BY_SHAPE[shape] then if not cmd_byte then cmd_byte = duffle.GP0_CMD_BY_SHAPE[shape] - cmd_line = atom.line + line_in_body[t.rel] + cmd_line = atom.line + line_in_body[tokens[tok_idx].rel] end saw_format = true - local contrib_key = "mac_format_" .. shape .. "_color" - local n = duffle.GP0_MACRO_CONTRIB[contrib_key] + local n = duffle.GP0_MACRO_CONTRIB["mac_format_" .. shape .. "_color"] if n then contrib = contrib + n end end - if ident:match("^mac_gte_store_[%w_]+$") then - local n = duffle.GP0_MACRO_CONTRIB[ident] + if tc_entry.is_gte_store then + local n = duffle.GP0_MACRO_CONTRIB[tc_entry.ident] if n then contrib = contrib + n end end - if ident:match("^mac_insert_ot_tag_[%w_]+$") then - local n = duffle.GP0_MACRO_CONTRIB[ident] + if tc_entry.is_ot_tag then + local n = duffle.GP0_MACRO_CONTRIB[tc_entry.ident] if n then contrib = contrib + n end end - if tc[tok_idx].is_store_word and tok:find("R_PrimCursor", 1, true) then + if tc_entry.writes_r_prim_cursor then saw_prim_write = true end end @@ -536,10 +561,6 @@ end -- Check #5: per-atom cycle budget (uses analyze_atom_paths's unknown_macros) -- ════════════════════════════════════════════════════════════════════════════ --- NOTE: `token_cycles`, `find_atom_labels`, `find_branch_targets` were removed --- when `classify_tokens` (the pre-computed per-token classification) replaced them. --- The classification lives on `atom.paths.tok_class`; analyze_atom_paths reads it. - --- Walk all paths through an atom body and return per-path cycle sums. --- Builds a tiny CFG: each token has a "next" pointer; branches have two (fall-through + taken). --- The BD-slot nop after a branch is absorbed into the branch's cost (MIPS-accurate: BD slot always runs), diff --git a/scripts/passes/word_count_eval.lua b/scripts/passes/word_count_eval.lua index bd758f8..7308b54 100644 --- a/scripts/passes/word_count_eval.lua +++ b/scripts/passes/word_count_eval.lua @@ -28,17 +28,13 @@ local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua") -- Constants -- ════════════════════════════════════════════════════════════════════════════ --- Windows separator chars — used to convert `dir` output (which uses `\`) into POSIX paths (which our scripts expect). +-- Windows separator char — used by `fname:match` to recognize `.macs.h` files. local PATH_SEP_BACKSLASH = "\\" -local PATH_SEP_FORWARD = "/" --- Fallback glob command (subprocess). Used when `lfs` (LuaFileSystem) is not available. --- Scoped to `code\` to avoid walking `.git/`, `toolchain/`, `build/`, etc. -local DIR_GLOB_CMD = 'dir /b /s "%s\\code\\%s" 2>nul' - --- Try to load lfs (LuaFileSystem). If available, scan_dir uses native directory enumeration (~2ms) --- instead of spawning `dir /b /s` as a subprocess (~56ms). Built by update_deps.ps1 into toolchain/lfs/lfs.dll. -local lfs = pcall(require, "lfs") and require("lfs") or nil +-- Required native extension: lfs (LuaFileSystem). Built by `update_deps.ps1` to +-- `toolchain/lfs/lfs.dll` and wired into package.cpath by `scripts/duffle_paths.lua`. +-- If lfs is missing, `require` throws — fail loud per the build-tool convention. +local lfs = require("lfs") -- ════════════════════════════════════════════════════════════════════════════ -- Type declarations @@ -106,26 +102,20 @@ end -- └────────────────────────────────────────────────────────────────────┘ --- Recursively scan a directory for files matching a glob suffix. ---- No regex per the no_regex constraint — uses plain byte matching via `dir /b /s` on Windows. --- --- The `.macs.h` files produced by the components pass always live at `//gen/`. ---- We can shortcut the `dir /b /s` walk by listing modules first (one `dir /b /ad`), then walking each `/gen/` ---- (one `dir /b` per module, no recursion). ---- For projects with 2 modules and 0 .macs.h files, this drops the cost from ~52ms ---- (full recursive walk of the entire project tree) to ~5ms. +--- Native walk via lfs.attributes + lfs.dir: ~2ms vs ~56ms for the prior `dir /b /s` subprocess. --- --- @param dir string -- directory to scan (absolute or relative) --- @param suffix string -- file pattern, e.g. "*.macs.h" --- @return string[] --- Cache the scan_dir result per (dir, suffix) in package.loaded. --- Each `io.popen` call on Windows is ~50-100ms of subprocess overhead, so caching the result saves a fixed cost on every build. +-- Cache the scan_dir result per (dir, suffix) in package.loaded. -- The cache persists for the lifetime of the Lua process (cleared when ps1_meta.lua exits). -- If a build removes/creates .macs.h files mid-process, the caller can invalidate by calling `M._invalidate_scan_cache()`. local SCAN_CACHE_KEY = "__word_count_eval_scan_cache__" --- Scan `code/` for files matching `suffix` (e.g. `*.macs.h`). ---- Uses `lfs` (LuaFileSystem) when available — native directory enumeration at ~2ms. ---- Falls back to `dir /b /s` subprocess (~56ms) when `lfs` is not compiled. +--- Native directory enumeration via lfs (~2ms). Zero subprocess spawns. --- --- @param dir string -- project root directory --- @param suffix string -- file pattern, e.g. "*.macs.h" @@ -137,33 +127,20 @@ function M.scan_dir(dir, suffix) if cache and cache[key] then return cache[key] end local results = {} - - if lfs then - -- Native walk: list code//gen/ for matching files. Zero subprocess spawns. - local code_dir = dir .. "/code" - if lfs.attributes(code_dir, "mode") == "directory" then - for mod_name in lfs.dir(code_dir) do - if mod_name ~= "." and mod_name ~= ".." then - local gen_path = code_dir .. "/" .. mod_name .. "/gen" - if lfs.attributes(gen_path, "mode") == "directory" then - for fname in lfs.dir(gen_path) do - if fname:match("%.macs%.h$") then - results[#results + 1] = gen_path .. "/" .. fname - end + local code_dir = dir .. "/code" + if lfs.attributes(code_dir, "mode") == "directory" then + for mod_name in lfs.dir(code_dir) do + if mod_name ~= "." and mod_name ~= ".." then + local gen_path = code_dir .. "/" .. mod_name .. "/gen" + if lfs.attributes(gen_path, "mode") == "directory" then + for fname in lfs.dir(gen_path) do + if fname:match("%.macs%.h$") then + results[#results + 1] = gen_path .. "/" .. fname end end end end end - else - -- Fallback: single `dir /b /s` subprocess scoped to code\. - local pipe = io.popen(DIR_GLOB_CMD:format(dir, suffix)) - if pipe then - for raw_line in pipe:lines() do - results[#results + 1] = raw_line:gsub(PATH_SEP_BACKSLASH, PATH_SEP_FORWARD) - end - pipe:close() - end end -- Cache the result (including empty results).