ed/pikuma_ps1
1
0
Fork 0
mirror of https://github.com/Ed94/pikuma_ps1.git synced 2026-07-12 12:21:26 -07:00
Code Issues Packages Projects Releases Wiki Activity

final pass on metaprogram

Browse Source
This commit is contained in:
Edward R. Gonzalez
2026-07-11 19:53:12 -04:00
parent 277af1c901
commit 9f7a4a00ce
9 changed files with 741 additions and 611 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/duffle.lua
+16 -32
View File
@@ -12,19 +12,13 @@
--- - **Process-bootstrap helper** (`setup_package_path`replaces the 8-line `arg[0]`-resolution boilerplate duplicated across 7 entry scripts)
---
--- **Conventions**: tabs (1/level), EmmyLua annotations, no regex.
--- Lua 5.3 compatible; no `<close>`/`<toclose>`, no `continue`, no
--- 5.4 string.dump improvements. LuaJIT 5.1+extensions model is the primary target.
---
--- **No `:match` / `:gmatch` regex use anywhere**;
--- all delimiter-splitting is hand-rolled or via LPeg (the regex-free PEG library).
local M = {}
-- Optional native extension: lfs (LuaFileSystem). When present, ensure_dir uses
-- lfs.attributes + lfs.mkdir instead of spawning `cmd.exe mkdir` — saves ~55ms per
-- unique directory on Windows. Built by `update_deps.ps1` to `toolchain/lfs/lfs.dll`
-- and wired into package.cpath by `scripts/duffle_paths.lua`.
local lfs = pcall(require, "lfs") and require("lfs") or nil
-- Required native extension: lfs (LuaFileSystem). Built by `update_deps.ps1` to
-- `toolchain/lfs/lfs.dll` and wired into package.cpath by `scripts/duffle_paths.lua`.
-- If lfs is missing, `require` throws — fail loud per the build-tool convention.
local lfs = require("lfs")
-- ════════════════════════════════════════════════════════════════════════════
-- Cross-file type aliases
@@ -315,7 +309,7 @@ end
-- Normalizes forward slashes to backslashes on Windows.
-- Used for byte-identical emit: the // Source: comment line uses the absolute path.
--
-- The CWD is memoized (one `io.popen("cd")` per process — ~50ms on Windows).
-- The CWD is memoized on first call (one lfs.currentdir() per process — ~0ms).
-- Without the cache, calling this per-source in the components pass added ~1.5s to a 30-source build.
-- @param path string
-- @return string
@@ -329,16 +323,8 @@ function M.to_absolute_path(path)
_absolute_path_cache[path] = result
return result
end
-- Native: lfs.currentdir() is ~0ms vs io.popen("cd") at ~50ms per call.
local cwd
if lfs then
cwd = lfs.currentdir()
else
local p = io.popen("cd")
if not p then _absolute_path_cache[path] = path; return path end
cwd = p:read("*l")
p:close()
end
-- lfs.currentdir() is ~0ms vs io.popen("cd") at ~50ms per call on Windows.
local cwd = lfs.currentdir()
if not cwd then _absolute_path_cache[path] = path; return path end
cwd = cwd:gsub("/", "\\")
local tail = (path:gsub("/", "\\"))
@@ -356,15 +342,9 @@ local _ensured_dirs = {}
function M.ensure_dir(path)
if _ensured_dirs[path] then return end
_ensured_dirs[path] = true
if lfs then
-- Native: ~0ms when dir exists (the common case). lfs.mkdir on a new dir is ~2ms (no shell spawn).
-- Falls through silently if lfs.mkdir fails (e.g. permission denied); the subsequent write_file will surface the error.
if lfs.attributes(path, "mode") ~= "directory" then lfs.mkdir(path) end
else
-- Fallback: shell mkdir. Slow (~55ms per call on Windows due to cmd.exe spawn) but works without lfs.
local is_win = package.config:sub(1, 1) == "\\"
os.execute(is_win and ('if not exist "' .. path .. '" mkdir "' .. path .. '"') or ('mkdir -p "' .. path .. '" 2>/dev/null'))
end
-- lfs.attributes + lfs.mkdir: ~0ms when dir exists, ~2ms when creating. No shell spawn.
-- Falls through silently if lfs.mkdir fails (e.g. permission denied); the subsequent write_file will surface the error.
if lfs.attributes(path, "mode") ~= "directory" then lfs.mkdir(path) end
end
-- Test helper: clear the cache (used by tests + between process runs).
@@ -585,8 +565,9 @@ end
-- Moved here from passes/static_analysis.lua so all passes can share the memoized
-- per-body tokenization. The memoization key is the body string (immutable per pass).
local _tokenize_body_cache = {}
local _body_line_index_cache = {}
local _tokenize_body_cache = {}
local _tokenize_body_simple_cache = {}
local _body_line_index_cache = {}
--- Tokenize the body inner-text into a flat list of `{tok, rel}` pairs.
--- `tok` is the trimmed token string; `rel` is the byte offset within `body`.
@@ -633,12 +614,15 @@ end
--- Tokenize the body into a flat list of trimmed string tokens (preserves comments).
--- Uses `split_top_level_commas` (which appends trailing comments to the previous token)
--- so the components pass can emit `/* Words: ... */` comments in the .macs.h output.
--- Memoized on body string (R7 lift; mirror of M.tokenize_body's memoization).
--- @param body string
--- @return string[]
function M.tokenize_body_simple(body)
if _tokenize_body_simple_cache[body] ~= nil then return _tokenize_body_simple_cache[body] end
local tokens = M.split_top_level_commas(body)
local out = {}
for i = 1, #tokens do out[i] = M.trim(tokens[i]) end
_tokenize_body_simple_cache[body] = out
return out
end
scripts/duffle_paths.lua
+11 -33
View File
@@ -30,47 +30,25 @@ local CACHE_KEY = "__duffle_repo_root__"
--- Replaces the prior `io.popen("git rev-parse --show-toplevel")` approach, which cost ~100-180ms per
--- LuaJIT process on Windows due to git's CLI startup. The path-derive approach costs <1ms.
---
--- If this script's path can't be parsed (shouldn't happen — dofile/debug.getinfo always populates source),
--- fall back to a defensive walk: starting from this script's directory, walk UP until we find a parent that
--- contains a `scripts/` directory. The first match is the repo root.
--- If `debug.getinfo` can't parse this script's path (shouldn't happen — dofile always populates source),
--- return nil and let `M.setup()` fail loud.
--- @return string|nil
local function find_repo_root()
if package.loaded[CACHE_KEY] then return package.loaded[CACHE_KEY] end
local source = debug.getinfo(1, "S").source
-- Strip the leading `@` (Lua's dofile marker) and the trailing `/duffle_paths.lua` filename.
-- What remains is the directory containing this script, i.e. `<repo>/scripts/` (with trailing slash or not).
-- What remains is the directory containing this script, i.e. `<repo>/scripts/`.
local scripts_dir = source and source:match("^@?(.*)[/\\]duffle_paths%.lua$")
if scripts_dir then
-- The repo root is the parent of `scripts/`. Strip the trailing `scripts/` (with or without trailing slash).
local root = scripts_dir:gsub("scripts[\\/]?$", "")
root = root:gsub("\\", "/")
if root == "" then root = "./" end
if not root:match("/$") then root = root .. "/" end
package.loaded[CACHE_KEY] = root
return root
end
if not scripts_dir then return nil end
-- Defensive fallback: walk UP from this script's directory until we find a parent that contains `scripts/`.
-- In practice this branch never fires — debug.getinfo always returns a source for dofile()'d chunks.
local lfs = pcall(require, "lfs") and require("lfs") or nil
if lfs then
local dir = source and source:match("^@?(.*[/\\])") or "./"
dir = dir:gsub("\\", "/")
while dir and dir ~= "" do
local candidate_scripts = dir .. "scripts"
if lfs.attributes(candidate_scripts, "mode") == "directory" then
dir = dir:gsub("/$", "")
package.loaded[CACHE_KEY] = dir .. "/"
return dir .. "/"
end
local parent = dir:match("^(.*)/[^/]+/$")
if not parent then break end
dir = parent .. "/"
end
end
return nil
-- The repo root is the parent of `scripts/`. Strip the trailing `scripts/` (with or without trailing slash).
local root = scripts_dir:gsub("scripts[\\/]?$", "")
root = root:gsub("\\", "/")
if root == "" then root = "./" end
if not root:match("/$") then root = root .. "/" end
package.loaded[CACHE_KEY] = root
return root
end
--- Set `package.path` (for `require("duffle")` + `require("passes.X")`) and
scripts/passes/annotation.lua
+210 -128
View File
@@ -3,16 +3,13 @@
--- Validates `MipsAtom_(name) atom_info(atom_bind(Binds_X), atom_reads(...), atom_writes(...)) { ... }` declarations in source files.
--- Also reads: `Binds_*` struct declarations (`typedef Struct_(Binds_X) { ... };`)
---
--- Source scanning: done ONCE upstream by `duffle.scan_source()` (ps1_meta.lua pre-scans each
--- source and stashes the result in `src.scan`). This pass is pure: read from the scan, run
--- checks, emit findings. No source re-walking.
--- Source scanning: done ONCE upstream by `duffle.scan_source()` (ps1_meta.lua pre-scans each source and stashes the result in `src.scan`).
---
--- Writes:
--- - `<ctx.out_root>/<dir_basename>.errors.h` — one per module, with `#error` directives on findings (the C compile will surface the error)
--- - The annotations.txt report is rendered by `passes/report.lua` from the per-module results stashed in `ctx.flags._annot_results`
---
--- **Conventions**: tabs (1/level), EmmyLua annotations, no regex,
--- Lua 5.3 compatible
--- **Conventions**: tabs (1/level), EmmyLua annotations, no regex, Lua 5.3 compatible
-- Bootstrap: same as entry scripts. See `ps1_meta.lua` for the rationale.
-- Bootstrap: load `scripts/duffle_paths.lua` (sets package.path + package.cpath).
@@ -21,13 +18,12 @@
-- Bootstrap: load `duffle_paths.lua` via `debug.getinfo(1, "S").source` (works both standalone + when require'd).
-- duffle_paths.lua sets package.path then returns `require("duffle")` at the bottom, so the dofile value IS the duffle module.
local _bootstrap_dir = debug.getinfo(1, "S").source:match("^@?(.*[/\\])") or "./"
local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua")
local write_file = duffle.write_file
local ensure_dir = duffle.ensure_dir
local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua")
local write_file = duffle.write_file
local ensure_dir = duffle.ensure_dir
-- Domain tables (single source of truth in duffle.lua).
local WAVE_CONTEXT_REGS = duffle.WAVE_CONTEXT_REGS
local TAPE_ATOM_MACROS = duffle.TAPE_ATOM_MACROS
local function is_wave_context_reg(n) return WAVE_CONTEXT_REGS[n] ~= nil end
@@ -68,11 +64,22 @@ local function is_wave_context_reg(n) return WAVE_CONTEXT_REGS[n] ~= nil end
--- @field binds string|nil -- Binds_X name if any
--- @field reads string[] -- R_* names (read targets)
--- @field writes string[] -- R_* names (write targets)
--- @field errors string[]|nil -- parse-time errors from scan_source (atom_info body malformed)
--- @class Finding
--- @field line integer -- source line (or 0 for pass-level)
--- @field msg string -- finding message
--- @class Findings
--- @field errors Finding[]
--- @field warnings Finding[]
--- @field info Finding[]
--- @class PipeCtx
--- @field atom_index table<string, AtomAnnotation> -- name -> AtomAnnotation (only kind=="atom")
--- @field binds_index table<string, BindsStruct> -- name -> BindsStruct
--- @field annot_counts table<string, integer> -- name -> annotation count (for unique_annotation check)
--- @class AnnotatedResult
--- @field atoms AtomEntry[]
--- @field annots AtomAnnotation[]
@@ -82,6 +89,154 @@ local function is_wave_context_reg(n) return WAVE_CONTEXT_REGS[n] ~= nil end
--- @field warnings Finding[]
--- @field info Finding[]
-- ════════════════════════════════════════════════════════════════════════════
-- Per-check functions (the CHECK_RULES table's payload)
-- ════════════════════════════════════════════════════════════════════════════
--
-- Each check has a uniform `append_to_findings` shape (errors[] / warnings[] / info[]).
-- The dispatcher in `validate()` decides which findings list each check writes to — by convention,
-- "existence" checks (declaration must exist, struct must exist) write errors[]; "shape" checks
-- (writes/reads must be wave-context) write warnings[]. The `macro_word_drift` check writes
-- both errors[] (missing/mismatch) and info[] (match).
--- Check: every annotated atom must have a matching MipsAtom_(name) declaration.
--- @param a AtomAnnotation
--- @param pipe_ctx PipeCtx
--- @param findings Findings
local function check_atom_decl_exists(a, pipe_ctx, findings)
if not pipe_ctx.atom_index[a.name] then
findings.errors[#findings.errors + 1] = {
line = a.line,
msg = string.format("annotation for '%s' has no matching MipsAtom_(%s) { ... }", a.name, a.name),
}
end
end
--- Check: every atom may have AT MOST ONE annotation.
--- Post-loop: needs full-corpus `annot_counts` from pipe_ctx.
--- @param pipe_ctx PipeCtx
--- @param findings Findings
local function check_unique_annotation(pipe_ctx, findings)
for name, n in pairs(pipe_ctx.annot_counts) do
if n > 1 then
findings.errors[#findings.errors + 1] = {
line = pipe_ctx.atom_index[name] and pipe_ctx.atom_index[name].line or 0,
msg = string.format("MipsAtom_(%s) has %d annotations (expected at most 1)", name, n),
}
end
end
end
--- Check: BIND atoms must reference a real Binds_* struct.
--- Demoted from error to warning (2026-07-10): the same condition is now caught by passes/static_analysis.lua's
--- check_abi_handoff() as an error. Emitting a warning here keeps the annotation pass from being stop-on-error
--- for the common test-fixture case, while still surfacing the issue in the report.
--- The static-analysis report remains the source of truth for build-stopping errors.
--- @param a AtomAnnotation
--- @param pipe_ctx PipeCtx
--- @param findings Findings
local function check_binds_struct_exists(a, pipe_ctx, findings)
if not a.binds then return end
if pipe_ctx.binds_index[a.binds] then return end
findings.warnings[#findings.warnings + 1] = {
line = a.line,
msg = string.format("'%s' binds '%s' but no Struct_(%s) { ... } "
.. "declaration found (also flagged as an error by check_abi_handoff in the static-analysis pass)"
, a.name, a.binds, a.binds),
}
end
--- Check: Binds_* struct fields must correspond to known wave-context registers.
--- Also checks that all `atom_writes(...)` entries are wave-context registers.
--- @param a AtomAnnotation
--- @param pipe_ctx PipeCtx
--- @param findings Findings
local function check_binds_field_wave_context(a, pipe_ctx, findings)
if not (a.binds and pipe_ctx.binds_index[a.binds]) then return end
local bs = pipe_ctx.binds_index[a.binds]
for _, f in ipairs(bs.fields) do
local candidate = "R_" .. f.name
if not is_wave_context_reg(candidate) then
findings.warnings[#findings.warnings + 1] = {
line = bs.line,
msg = string.format("%s field '%s' doesn't match a known wave-context register (candidate '%s')", a.binds, f.name, candidate),
}
end
end
for _, w in ipairs(a.writes) do
if not is_wave_context_reg(w) then
findings.warnings[#findings.warnings + 1] = {
line = a.line,
msg = string.format("%s writes '%s' which is not a known wave-context register", a.name, w),
}
end
end
end
--- Check: atom_reads(...) entries should be wave-context registers (or R_TapePtr for rbind).
--- @param a AtomAnnotation
--- @param pipe_ctx PipeCtx
--- @param findings Findings
local function check_reads_wave_context(a, pipe_ctx, findings)
for _, r in ipairs(a.reads) do
if not is_wave_context_reg(r) and r ~= "R_TapePtr" then
findings.warnings[#findings.warnings + 1] = {
line = a.line,
msg = string.format("atom '%s' reads '%s' which is not a known wave-context register", a.name, r),
}
end
end
end
--- Check: TAPE_WORDS(mac_X, N) ↔ WORD_COUNT(mac_X, N) drift.
--- Three outcomes: missing (error), mismatch (error), match (info).
--- @param m MacroEntry
--- @param wc table<string, integer> -- the shared word-count table (from ctx.shared.word_counts)
--- @param findings Findings
local function check_macro_word_drift(m, wc, findings)
local declared = wc[m.name]
if not declared then
findings.errors[#findings.errors + 1] = {
line = m.line,
msg = string.format("TAPE_WORDS(%s, %d) but '%s' is not in metadata.h", m.name, m.words, m.name),
}
return
end
if declared ~= m.words then
findings.errors[#findings.errors + 1] = {
line = m.line,
msg = string.format("DRIFT: TAPE_WORDS(%s, %d) but metadata.h declares WORD_COUNT(%s, %d)", m.name, m.words, m.name, declared),
}
return
end
findings.info[#findings.info + 1] = {
line = m.line,
msg = string.format("OK: %s = %d words", m.name, m.words),
}
end
-- ════════════════════════════════════════════════════════════════════════════
-- CHECK_RULES — data-driven check dispatch (the plex pattern)
-- ════════════════════════════════════════════════════════════════════════════
--
-- Each rule entry picks one of three "shapes" of dispatch:
-- per_annot(annot, pipe_ctx, findings) — runs once per AtomAnnotation
-- post(pipe_ctx, findings) — runs once after all per_annot calls complete (full-corpus aggregation)
-- per_macro(macro, wc, findings) — runs once per TAPE_WORDS / _Pragma macro declaration
--
-- Adding a new check = 1 row here + 1 function above. The `validate()` dispatch loop never needs editing.
local CHECK_RULES = {
{ name = "atom_decl_exists", per_annot = check_atom_decl_exists },
{ name = "binds_struct_exists", per_annot = check_binds_struct_exists },
{ name = "binds_field_wave_context", per_annot = check_binds_field_wave_context },
{ name = "reads_wave_context", per_annot = check_reads_wave_context },
{ name = "unique_annotation", post = check_unique_annotation },
{ name = "macro_word_drift", per_macro = check_macro_word_drift },
}
-- ════════════════════════════════════════════════════════════════════════════
-- Validation
-- ════════════════════════════════════════════════════════════════════════════
@@ -115,138 +270,65 @@ local function validate(ctx, src)
binds = info.binds,
reads = info.reads or {},
writes = info.writes or {},
errors = {},
errors = info.errors,
}
end
-- Index atoms by name for lookup.
local atom_index = {}
for _, a in ipairs(atoms) do atom_index[a.name] = a end
-- Index binds by name for lookup.
local binds_index = {}
for _, b in ipairs(scan.binds) do binds_index[b.name] = b end
local errors = {}
local warnings = {}
local info = {}
-- 1. Every annotated atom must exist as a real MipsAtom_ declaration.
for _, a in ipairs(annots) do
if not atom_index[a.name] then
errors[#errors + 1] = {
line = a.line,
msg = string.format("annotation for '%s' has no matching MipsAtom_(%s) { ... }", a.name, a.name),
}
-- Build pipe_ctx (Fleury: expose structure). Pre-compute everything the per-check functions need.
-- Single source of truth for atom / binds / annotation-count lookups.
local pipe_ctx = {
atom_index = {},
binds_index = {},
annot_counts = {},
}
for _, a in ipairs(atoms) do pipe_ctx.atom_index [a.name] = a end
for _, b in ipairs(scan.binds) do pipe_ctx.binds_index[b.name] = b end
for _, a in ipairs(annots) do
if a.name then
pipe_ctx.annot_counts[a.name] = (pipe_ctx.annot_counts[a.name] or 0) + 1
end
end
-- Findings live in a single struct with three lists (errors / warnings / info).
-- Each check writes to the list appropriate for its severity.
local findings = { errors = {}, warnings = {}, info = {} }
-- Propagate parse-time errors from scan_source's atom_info parsing.
-- These are errors found in the atom_info(...) body itself (e.g., malformed args).
-- They are pre-existing in the scan payload — we just lift them into our findings list.
for _, a in ipairs(annots) do
if a.errors then
for _, msg in ipairs(a.errors) do
errors[#errors + 1] = {line = a.line, msg = string.format("'%s': %s", a.name, msg)}
end
end
end
-- 2. Every atom may have AT MOST ONE annotation (no duplicates).
-- (Atoms with ZERO annotations are valid in the new minimal shape.)
local count_per_atom = {}
for _, a in ipairs(annots) do
if a.name then
count_per_atom[a.name] = (count_per_atom[a.name] or 0) + 1
end
end
for name, n in pairs(count_per_atom) do
if n > 1 then
errors[#errors + 1] = {
line = atom_index[name] and atom_index[name].line or 0,
msg = string.format("MipsAtom_(%s) has %d annotations (expected at most 1)", name, n),
}
end
end
-- 3. (Phase validity check DROPPED. Phases were removed from the annotation DSL.)
-- 4. BIND atoms must reference a real Binds_* struct.
for _, a in ipairs(annots) do
if a.binds then
if not binds_index[a.binds] then
-- Demoted from error to warning (2026-07-10): the same condition is now caught by passes/static_analysis.lua's
-- check_abi_handoff() as an error. Emitting a warning here keeps the annotation pass from being stop-on-error
-- for the common test-fixture case, while still surfacing the issue in the report.
-- The static-analysis report remains the source of truth for build-stopping errors.
warnings[#warnings + 1] = {
findings.errors[#findings.errors + 1] = {
line = a.line,
msg = string.format("'%s' binds '%s' but no Struct_(%s) { ... } declaration found (also flagged as an error by check_abi_handoff in the static-analysis pass)", a.name, a.binds, a.binds),
msg = string.format("'%s': %s", a.name, msg),
}
end
end
end
-- 5. BIND writes must be wave-context registers that match Binds_ fields.
-- THE per-annotation pipeline. ONE loop. CHECK_RULES dispatches per_annot rules.
for _, a in ipairs(annots) do
if a.binds and binds_index[a.binds] then
local bs = binds_index[a.binds]
for _, f in ipairs(bs.fields) do
local candidate = "R_" .. f.name
if not is_wave_context_reg(candidate) then
warnings[#warnings + 1] = {
line = bs.line,
msg = string.format("%s field '%s' doesn't match a known wave-context register (candidate '%s')", a.binds, f.name, candidate),
}
end
end
for _, w in ipairs(a.writes) do
if not is_wave_context_reg(w) then
warnings[#warnings + 1] = {
line = a.line,
msg = string.format("%s writes '%s' which is not a known wave-context register", a.name, w),
}
end
end
for _, rule in ipairs(CHECK_RULES) do
if rule.per_annot then rule.per_annot(a, pipe_ctx, findings) end
end
end
-- 6. INFO reads should be wave-context registers (or R_TapePtr for rbind).
for _, a in ipairs(annots) do
for _, r in ipairs(a.reads) do
if not is_wave_context_reg(r) and r ~= "R_TapePtr" then
warnings[#warnings + 1] = {
line = a.line,
msg = string.format("atom '%s' reads '%s' which is not a known wave-context register", a.name, r),
}
end
end
-- Post-loop rules (one-shot checks that need full-corpus aggregation in pipe_ctx).
for _, rule in ipairs(CHECK_RULES) do
if rule.post then rule.post(pipe_ctx, findings) end
end
-- 7. TAPE_WORDS(mac_X, N) ↔ WORD_COUNT(mac_X, N) drift.
-- Three outcomes: missing (error), mismatch (error), match (info).
local function check_macro_drift(m, declared)
if not declared then
errors[#errors + 1] = {
line = m.line,
msg = string.format("TAPE_WORDS(%s, %d) but '%s' is not in metadata.h", m.name, m.words, m.name),
}
return
end
if declared ~= m.words then
errors[#errors + 1] = {
line = m.line,
msg = string.format("DRIFT: TAPE_WORDS(%s, %d) but metadata.h declares WORD_COUNT(%s, %d)", m.name, m.words, m.name, declared),
}
return
end
info[#info + 1] = {
line = m.line,
msg = string.format("OK: %s = %d words", m.name, m.words),
}
end
-- Per-macro rules (TAPE_WORDS vs WORD_COUNT drift).
local wc = ctx.shared.word_counts
for _, m in ipairs(scan.macros) do
check_macro_drift(m, ctx.shared.word_counts[m.name])
for _, rule in ipairs(CHECK_RULES) do
if rule.per_macro then rule.per_macro(m, wc, findings) end
end
end
-- 8. Information summary.
info[#info + 1] = {
-- Information summary (always emitted).
findings.info[#findings.info + 1] = {
line = 0,
msg = string.format("scanned: %d atom(s), %d annotation(s), %d macro-word-decl(s), %d binds struct(s)",
#atoms, #annots, #scan.macros, #scan.binds),
@@ -257,9 +339,9 @@ local function validate(ctx, src)
annots = annots,
macros = scan.macros,
binds = scan.binds,
errors = errors,
warnings = warnings,
info = info,
errors = findings.errors,
warnings = findings.warnings,
info = findings.info,
}
end
@@ -300,7 +382,7 @@ end
--- Stash aggregated per-module results for the report pass to consume.
local function emit_module_annotations_stub(ctx, dir, dir_basename, atoms_count)
ctx.flags = ctx.flags or {}
ctx.flags = ctx.flags or {}
ctx.flags._annot_results = ctx.flags._annot_results or {}
ctx.flags._annot_results[#ctx.flags._annot_results + 1] = {
dir = dir,
@@ -339,20 +421,20 @@ function M.run(ctx)
local dir_errors = {}
local dir_warnings = {}
-- Per-source validate() results, cached for the report pass (it reads from this instead of re-validating each source).
ctx.flags = ctx.flags or {}
ctx.flags = ctx.flags or {}
ctx.flags._annot_source_results = ctx.flags._annot_source_results or {}
for _, src in ipairs(dir_sources) do
local result = validate(ctx, src)
local result = validate(ctx, src)
result.source = src.path -- tag for downstream rendering
ctx.flags._annot_source_results[src.path] = result -- stash so report.lua reads from cache instead of re-running validate()
dir_atoms = dir_atoms + #result.atoms
for _, e in ipairs(result.errors) do
dir_errors[#dir_errors + 1] = { line = e.line, msg = e.msg, source = src.path }
errors[#errors + 1] = { line = e.line, msg = e.msg }
errors [#errors + 1] = { line = e.line, msg = e.msg }
end
for _, w in ipairs(result.warnings) do
dir_warnings[#dir_warnings + 1] = { line = w.line, msg = w.msg }
warnings[#warnings + 1] = { line = w.line, msg = w.msg }
warnings [#warnings + 1] = { line = w.line, msg = w.msg }
end
end
scripts/passes/components.lua
+29 -35
View File
@@ -30,8 +30,8 @@
-- both standalone and when require'd from the orchestrator.
-- Bootstrap: load `duffle_paths.lua` via `debug.getinfo(1, "S").source` (works both standalone + when require'd).
-- duffle_paths.lua sets package.path then returns `require("duffle")` at the bottom, so the dofile value IS the duffle module.
local _bootstrap_dir = debug.getinfo(1, "S").source:match("^@?(.*[/\\])") or "./"
local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua")
local _bootstrap_dir = debug.getinfo(1, "S").source:match("^@?(.*[/\\])") or "./"
local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua")
local word_count_eval = require("word_count_eval")
-- ════════════════════════════════════════════════════════════════════════════
@@ -107,13 +107,14 @@ local M = {}
-- @param before_pos integer
-- @return integer|nil
local function find_last_name_open_paren(source, name, before_pos)
local search = source:sub(1, before_pos)
local name_open = name .. "("
local last_idx = nil
local scan_pos = 1
while true do
local found = search:find(name_open, scan_pos, true) -- plain (no regex)
if not found then break end
-- Pass `before_pos + 1` so string.find only returns positions < before_pos + 1
-- (string.find's 4th arg `plain` is true; we use the 3rd arg `init` for the upper bound).
local found = source:find(name_open, scan_pos, true)
if not found or found >= before_pos then break end
last_idx = found
scan_pos = found + #name_open
end
@@ -126,15 +127,15 @@ end
--- Convention: function form is
--- `FI_ MipsAtom ac_X(args) MipsAtomComp_Proc_(ac_X, { body })`
--- We find the LAST occurrence of `"ac_X("` before `before_pos` and extract the args from inside the parens.
--- We then verify the preceding context ends with `MipsAtom` (the function-decl keyword
--- with possible qualifiers between).
--- We then verify the preceding context ends with `MipsAtom`
--- (the function-decl keyword with possible qualifiers between).
---
--- @param source string
--- @param name string
--- @param before_pos integer
--- @return string|nil
local function find_function_args_for(source, name, before_pos)
local last_idx = find_last_name_open_paren(source, name, before_pos)
local last_idx = find_last_name_open_paren(source, name, before_pos)
if not last_idx then return nil end
-- Verify the preceding context ends with "MipsAtom" (with possible qualifiers between).
@@ -224,7 +225,7 @@ end
-- ends at `close_end_pos`. Returns (block_text, new_scan_pos) where `new_scan_pos`
-- is where to continue scanning for more comments, or nil if no block comment was found.
local function capture_block_comment(source, close_end_pos)
local open_at = find_block_comment_open(source, close_end_pos)
local open_at = find_block_comment_open(source, close_end_pos)
if not open_at then return nil end
local block_start = extend_left_over_indent(source, open_at)
return source:sub(block_start, close_end_pos), block_start
@@ -255,18 +256,18 @@ local function preceding_comment_block(source, pos)
local pieces = {}
while true do
local non_ws = skip_ws_backward(source, scan_pos)
if non_ws == 0 then break end
if non_ws == 0 then break end
local is_block_close = non_ws >= 2 and source:sub(non_ws - 1, non_ws) == "*/"
local is_line_end = source:sub(non_ws, non_ws) == "\n" or source:sub(non_ws, non_ws) == "\r"
if is_block_close then
local block_text, new_scan_pos = capture_block_comment(source, non_ws)
local block_text, new_scan_pos = capture_block_comment(source, non_ws)
if not block_text then break end
table.insert(pieces, 1, block_text)
scan_pos = new_scan_pos
elseif is_line_end then
local line_text, new_scan_pos = capture_line_comment(source, non_ws)
local line_text, new_scan_pos = capture_line_comment(source, non_ws)
if not line_text then break end
table.insert(pieces, 1, line_text)
scan_pos = new_scan_pos
@@ -282,8 +283,8 @@ end
-- Argument-name extraction
-- ════════════════════════════════════════════════════════════════════════════
-- Walk `trimmed` backward from `pos` over trailing whitespace / asterisks / brackets, returning the position of the first
-- non-trailer character (i.e. the end of the identifier).
-- Walk `trimmed` backward from `pos` over trailing whitespace / asterisks / brackets,
-- returning the position of the first non-trailer character (i.e. the end of the identifier).
-- @param trimmed string
-- @param pos integer
-- @return integer
@@ -322,9 +323,6 @@ end
--- `"U4 off, U4 code, U1 r, U1 g, U1 b"` -> `{"off", "code", "r", "g", "b"}`
--- `"U4 *ptr"` -> `{"ptr"}`
--- `""` -> nil
---
--- No regex — uses `duffle.is_alnum` + plain string ops.
---
--- @param args_str string|nil
--- @return string[]|nil
local function extract_arg_names(args_str)
@@ -334,9 +332,9 @@ local function extract_arg_names(args_str)
for _, tok in ipairs(tokens) do
local trimmed = duffle.trim(tok)
if trimmed ~= "" then
local ident_end = trim_trailer_back(trimmed, #trimmed)
local ident_end = trim_trailer_back(trimmed, #trimmed)
local ident_start = trim_ident_back(trimmed, ident_end) + 1
local name = trimmed:sub(ident_start, ident_end)
local name = trimmed:sub(ident_start, ident_end)
if name ~= "" then names[#names + 1] = name end
end
end
@@ -379,7 +377,6 @@ end
-- ════════════════════════════════════════════════════════════════════════════
-- Convert `//` line comments to `/* */` block comments in a token.
--
-- C macros use `\` line-continuations; a `//` comment before `\` would consume the continuation,
-- breaking the macro. We convert `//` to `/* */` so the multi-line macro structure is preserved.
--
@@ -403,8 +400,8 @@ local function convert_line_comments_to_block(s)
while eol <= len and result:byte(eol) ~= BYTE_NEWLINE do
eol = eol + 1
end
local before = result:sub(1, pos - 1)
local comment = result:sub(pos + 2, eol - 1) -- skip the `//`
local before = result:sub(1, pos - 1)
local comment = result:sub(pos + 2, eol - 1) -- skip the `//`
local after
if eol <= len and result:byte(eol) == BYTE_NEWLINE then
after = " */" .. result:sub(eol) -- keep the newline
@@ -422,8 +419,9 @@ end
-- Word-count computation (memoized recursive lookup)
-- ════════════════════════════════════════════════════════════════════════════
-- Strip the `mac_` prefix from a component-call ident so we can look it up against the components-by-name table. Returns the ident unchanged
-- if it doesn't start with the prefix (so a non-component ident like `mask_upper` falls through to the wc-table branch).
-- Strip the `mac_` prefix from a component-call ident so we can look it up against the components-by-name table.
-- Returns the ident unchanged if it doesn't start with the prefix
-- (so a non-component ident like `mask_upper` falls through to the wc-table branch).
-- @param ident string|nil
-- @return string|nil
local function strip_mac_prefix(ident)
@@ -476,8 +474,7 @@ end
--- Compute word counts for every component in `components` in a single pass.
--- The name-lookup table + memoization cache are built ONCE (per source) instead of per-component,
--- so the cache survives across siblings and a component's recursive `mac_Y(...)` references hit memoized values
--- instead of re-walking the body. Previously each call rebuilt both tables (O(N) tables per call → O(N^2)).
---
--- instead of re-walking the body.
--- Cycle detection (A -> B -> A) is preserved via the in-progress `-1` sentinel in `cache`.
---
--- @param components Component[]
@@ -530,7 +527,7 @@ end
--- @return string
local function signature_from_args(args_str)
local arg_names = extract_arg_names(args_str)
if arg_names and #arg_names > 0 then
if arg_names and #arg_names > 0 then
return table.concat(arg_names, ", ")
end
return "..."
@@ -540,7 +537,7 @@ end
--- The last 2 chars are always that pair.
local function strip_trailing_continuation(lines)
local last = lines[#lines]
if last:sub(-2) == " \\" then
if last:sub(-2) == " \\" then
lines[#lines] = last:sub(1, -3)
end
end
@@ -627,14 +624,13 @@ end
-- @return string -- the output directory
-- @return string -- the full output path
local function compute_macs_h_path(src)
local out_dir = src.dir .. "/" .. GEN_SUBDIR
local out_path = out_dir .. "/" .. duffle.basename_no_ext(src.dir) .. ".macs.h"
local out_dir = src.dir .. "/" .. GEN_SUBDIR
local out_path = out_dir .. "/" .. duffle.basename_no_ext(src.dir) .. ".macs.h"
return out_dir, out_path
end
--- Emit a per-source `.macs.h` header with the `mac_X` macros + `WORD_COUNT` entries. Writes in BINARY mode so LF line endings are
--- preserved (the git blob is LF; Windows text-mode would emit CRLF and break the byte-identical diff).
---
--- Emit a per-source `.macs.h` header with the `mac_X` macros + `WORD_COUNT` entries.
--- Writes in BINARY mode so LF line endings are preserved (the git blob is LF; Windows text-mode would emit CRLF and break the byte-identical diff).
--- Honors `ctx.dry_run`: prints the intended path but does not write the file.
---
--- @param ctx PassCtx
@@ -644,7 +640,6 @@ end
--- @return string|nil -- path to the written file (nil if no components)
local function emit_component_macros_h(ctx, src, components, counts)
if #components == 0 then return nil end
local out_dir, out_path = compute_macs_h_path(src)
local lines = header_boilerplate(src)
@@ -655,7 +650,6 @@ local function emit_component_macros_h(ctx, src, components, counts)
end
local content = table.concat(lines, "\n") .. "\n"
if ctx.dry_run then
print(string.format(" -> %s (dry-run)", out_path))
return out_path
scripts/passes/offsets.lua
+14 -6
View File
@@ -132,6 +132,14 @@ local function record_offset_marker(branches, args, at_pos)
end
end
-- MARKER_TO_HANDLER — data-driven marker dispatch (the plex pattern).
-- Maps the marker ident to its recorder function. Each handler takes (out_table, args, at_pos).
-- Adding a new marker type = 1 row + 1 recorder function.
local MARKER_TO_HANDLER = {
[LABEL_MARKER] = record_label_marker,
[OFFSET_MARKER] = record_offset_marker,
}
--- Scan a single token for atom_label/atom_offset markers, walking through balanced groups transparently (so nested calls are found).
--- @param token string
--- @param at_pos integer -- the branch-free word position of this token in the body
@@ -146,13 +154,13 @@ local function scan_for_atom_markers(token, at_pos, labels, branches)
local ch = token:sub(pos, pos)
if duffle.is_alpha(ch) then
local ident, after = duffle.read_ident(token, pos)
if ident == LABEL_MARKER then
local handler = MARKER_TO_HANDLER[ident]
if handler then
local args, after_paren = extract_ident_args(token, after)
record_label_marker(labels, args, at_pos)
pos = after_paren or after
elseif ident == OFFSET_MARKER then
local args, after_paren = extract_ident_args(token, after)
record_offset_marker(branches, args, at_pos)
-- Marker found — dispatch to its recorder. markers share labels and branches as
-- out-tables; the recorder picks which one(s) to write to based on its semantics.
-- (record_label_marker writes to labels; record_offset_marker writes to branches.)
handler(ident == LABEL_MARKER and labels or branches, args, at_pos)
pos = after_paren or after
else
pos = after
scripts/passes/report.lua
+40 -7
View File
@@ -261,6 +261,28 @@ local function render_module_warnings_section(add, results, total_warnings)
add("")
end
-- ════════════════════════════════════════════════════════════════════════════
-- SECTION_RENDERERS — data-driven section dispatch (the plex pattern)
-- ════════════════════════════════════════════════════════════════════════════
--
-- Each entry maps a section to its (header, render_fn). The render_fn signature:
-- render_fn(add, results, totals)
-- add -- the `add(line)` closure from the surrounding report renderer
-- results -- AnnotationResult[] (per-source results)
-- totals -- {atoms, annots, binds, macros, errors, warnings} counts
--
-- Sections that need to render "(none)" vs iterate use totals.errors / totals.warnings;
-- other sections ignore the totals arg.
-- Adding a new section = 1 row here + 1 render_<thing>_section function.
local SECTION_RENDERERS = {
{ header = SECTION_HEADER_ATOMS, render = render_module_atoms_section },
{ header = SECTION_HEADER_ANNOTS, render = render_module_annots_section },
{ header = SECTION_HEADER_BINDS, render = render_module_binds_section },
{ header = SECTION_HEADER_MACROS, render = render_module_macros_section },
{ header = SECTION_HEADER_ERRORS, render = function(add, results, totals) return render_module_errors_section(add, results, totals.errors) end },
{ header = SECTION_HEADER_WARNINGS, render = function(add, results, totals) return render_module_warnings_section(add, results, totals.warnings) end },
}
--- Render the per-MODULE annotation report (one `<dir_basename>.annotations.txt`).
--- @param dir string -- module directory path
--- @param sources SourceFile[] -- sources in this module
@@ -282,12 +304,22 @@ local function render_module_report(dir, sources, results)
total_atoms, total_annots, total_binds, total_macros))
add("")
render_module_atoms_section(add, results)
render_module_annots_section(add, results)
render_module_binds_section(add, results)
render_module_macros_section(add, results)
render_module_errors_section(add, results, total_errors)
render_module_warnings_section(add, results, total_warnings)
-- Bundle the totals so the section renderers don't need separate parameter lists.
-- Errors/warnings sections need their total count to decide "(none)" vs iterate.
-- Sections without totals (atoms/annots/binds/macros) ignore this arg.
local totals = {
atoms = total_atoms, annots = total_annots, binds = total_binds,
macros = total_macros, errors = total_errors, warnings = total_warnings,
}
-- THE per-section dispatch. ONE loop over SECTION_RENDERERS. Each renderer writes its
-- header + content via the `add` closure (pre-bound above).
-- Adding a new section = 1 row here + 1 render_<thing>_section function.
for _, section in ipairs(SECTION_RENDERERS) do
add(section.header)
section.render(add, results, totals)
add("")
end
return table.concat(lines, "\n") .. "\n"
end
@@ -347,7 +379,8 @@ end
-- ════════════════════════════════════════════════════════════════════════════
-- (internal) Pull per-source validate() results from the annotation pass's stash.
-- The annotation pass runs first in the dep chain and caches results in `ctx.flags._annot_source_results`; we read from there instead of re-validating each source.
-- The annotation pass runs first in the dep chain and caches results in `ctx.flags._annot_source_results`;
-- we read from there instead of re-validating each source.
-- Returns the list of module results + the flat list of all results (for the project-wide summary).
-- @param ctx PassCtx
-- @param dir_sources SourceFile[]
scripts/passes/scan_source.lua
+303 -250
View File
@@ -1,6 +1,6 @@
--- passes/scan_source.lua — Source pre-scan pass (the "mega entity" pass).
---
--- Single source-walk pass that produces the fat `SourceScan` payload consumed by all downstream passes. Walks each `ctx.sources` entry once,
--- Single source-walk pass that produces the fat `SourceScan` payload consumed by all downstream passes. Walks each `ctx.sources` entry once,
--- extracting every construct type the metaprograms need:
---
--- MipsAtom_ (kind = "atom", with optional atom_info inner)
@@ -14,8 +14,7 @@
--- This is the first pass in the dep graph (no deps).
--- Every other pass that reads source structure depends on this one — see `ps1_meta.lua :: PASSES`.
---
--- **Conventions**: tabs (1/level), EmmyLua annotations, no regex,
--- Lua 5.3 compatible.
--- **Conventions**: tabs (1/level), EmmyLua annotations, no regex, Lua 5.3 compatible
-- Bootstrap: same as entry scripts. See `ps1_meta.lua` for the rationale.
-- Bootstrap: load `scripts/duffle_paths.lua` (sets package.path + package.cpath).
@@ -35,7 +34,7 @@ local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua")
--- @field raw_atoms AtomEntry[] -- MipsCode code_<name> { body } (offsets pass only)
--- @field binds BindsEntry[] -- typedef Struct_(Binds_X) { fields } (fields pre-parsed)
--- @field atom_infos AtomInfoEntry[] -- MipsAtom_(name) atom_info(...) (sub-calls pre-parsed)
--- @field macros MacroEntry[] -- #pragma mac_X tape_atom words=N + _Pragma("...")
--- @field macros MacroEntry[] -- #pragma mac_X tape_atom words=N + _Pragma("...")
--- @field line_of fun(pos: integer): integer -- shared LineIndex closure
--- @class SourceFile
@@ -74,7 +73,7 @@ local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua")
--- @field comment string|nil -- populated by components pass (backward lookup)
-- ════════════════════════════════════════════════════════════════════════════
-- Local helpers
-- Local helpers (shared by per-form parsers)
-- ════════════════════════════════════════════════════════════════════════════
-- C qualifier keywords that may precede a MipsAtom_ / MipsCode declaration.
@@ -86,6 +85,22 @@ local QUALIFIER_KEYWORDS = {
["internal"] = true, ["LP_"] = true, ["global"] = true, ["gkknown"] = true,
}
-- "ac_" prefix length on component names (e.g., `MipsAtomComp_(ac_X, ...)`). The components pass strips
-- this prefix to derive the macro name (e.g., `mac_X`). Single source of truth — was duplicated in two
-- branches of the pre-refactor scan_source.
local AC_PREFIX = "ac_"
local AC_PREFIX_LEN = 3
-- Strip the "ac_" prefix from a component name. Returns the input unchanged if it doesn't start with the prefix.
-- @param raw_name string
-- @return string
local function strip_ac_prefix(raw_name)
if #raw_name > AC_PREFIX_LEN and raw_name:sub(1, AC_PREFIX_LEN) == AC_PREFIX then
return raw_name:sub(AC_PREFIX_LEN + 1)
end
return raw_name
end
-- Parse the U4 fields from a Binds_X body. Returns (fields, byte_count).
local function scan_binds_fields(body)
local fields = {}
@@ -147,13 +162,13 @@ local function scan_atom_info_subcalls(info_inner)
if info_inner:sub(sub_open, sub_open) == "(" then
local sub_inner, sub_after2 = duffle.read_parens(info_inner, sub_open)
-- scan: atom_bind(<Binds_X>)
binds = duffle.trim(sub_inner)
binds = duffle.trim(sub_inner)
sub_pos = sub_after2
else
sub_pos = sub_open + 1
end
elseif sub_ident == "atom_reads" or sub_ident == "atom_writes" then
local kind = sub_ident
local kind = sub_ident
local sub_open = duffle.skip_ws_and_cmt(info_inner, sub_end)
if info_inner:sub(sub_open, sub_open) == "(" then
local sub_inner, sub_after2 = duffle.read_parens(info_inner, sub_open)
@@ -181,6 +196,262 @@ local function scan_skip_qualifiers(source, pos)
end
end
-- ════════════════════════════════════════════════════════════════════════════
-- Per-form parsers (the DECL_PARSERS table's payload)
-- ════════════════════════════════════════════════════════════════════════════
--
-- Each parser has the uniform signature:
-- parser(source, pos, ident_end, line_of, out) -> new_pos
-- where:
-- source -- the full source text
-- pos -- position of the construct's leading ident (e.g., `M` of `MipsAtom_`)
-- ident_end -- position past the leading ident (where the `(` should be)
-- line_of -- closure over LineIndex(source) for 1-based line lookups
-- out -- the SourceScan out table (mutated in place: out.atoms / out.raw_atoms / out.binds / out.atom_infos / out.macros)
-- returns -- new position after the construct
--
-- All parsers read source-as-written via the duffle primitives (skip_ws_and_cmt / read_parens / read_braces / read_balanced).
-- No regex per the no_regex constraint; no hand-rolled depth tracking (the MipsAtomComp_Proc_ brace matcher
-- now uses duffle.read_braces instead of bespoke byte-dispatch).
--
-- Adding a new construct = 1 row in DECL_PARSERS + 1 parser function. The scan_source() loop never needs editing.
--- Parse: `MipsAtom_(<name>) [atom_info(<binds>, <reads>, <writes>)] { <body> }`
--- @param source string
--- @param pos integer
--- @param ident_end integer
--- @param line_of fun(pos: integer): integer
--- @param out SourceScan
--- @return integer
local function parse_mips_atom(source, pos, ident_end, line_of, out)
local open_paren = duffle.skip_ws_and_cmt(source, ident_end)
if source:sub(open_paren, open_paren) ~= "(" then return open_paren + 1 end
local inner, after_paren = duffle.read_parens(source, open_paren)
local raw_name = duffle.read_ident(inner, 1)
-- Lookahead for atom_info(...) between `)` and `{`. Captures sub-calls; updates brace search start.
local brace_search_pos = after_paren
local lookahead = duffle.skip_ws_and_cmt(source, after_paren)
local look_ident, look_end = duffle.read_ident(source, lookahead)
if look_ident == "atom_info" then
local info_open = duffle.skip_ws_and_cmt(source, look_end)
if source:sub(info_open, info_open) == "(" then
local info_inner, info_after = duffle.read_parens(source, info_open)
local ai_binds, ai_reads, ai_writes = scan_atom_info_subcalls(info_inner)
out.atom_infos[#out.atom_infos + 1] = {
atom_name = raw_name or "?", binds = ai_binds,
reads = ai_reads or {}, writes = ai_writes or {},
info_line = line_of(lookahead),
}
brace_search_pos = info_after
end
end
local brace = duffle.scan_to_char(source, "{", brace_search_pos)
if not brace then return open_paren + 1 end
local body, after_brace = duffle.read_braces(source, brace)
if raw_name and raw_name ~= "" then
out.atoms[#out.atoms + 1] = {
line = line_of(pos), name = raw_name, body = body, body_off = brace + 1,
kind = "atom", raw_name = raw_name,
ident_pos = pos, after_paren = after_paren,
}
end
return after_brace
end
--- Parse: `MipsAtomComp_(<name>) { <body> }`
--- @param source string
--- @param pos integer
--- @param ident_end integer
--- @param line_of fun(pos: integer): integer
--- @param out SourceScan
--- @return integer
local function parse_mips_atom_comp(source, pos, ident_end, line_of, out)
local open_paren = duffle.skip_ws_and_cmt(source, ident_end)
if source:sub(open_paren, open_paren) ~= "(" then return open_paren + 1 end
local inner, after_paren = duffle.read_parens(source, open_paren)
local raw_name = duffle.read_ident(inner, 1)
if not raw_name then return open_paren + 1 end
local brace = duffle.scan_to_char(source, "{", after_paren)
if not brace then return open_paren + 1 end
local body, after_brace = duffle.read_braces(source, brace)
out.atoms[#out.atoms + 1] = {
line = line_of(pos), name = strip_ac_prefix(raw_name), body = body, body_off = brace + 1,
kind = "comp_bare", raw_name = raw_name,
ident_pos = pos, after_paren = after_paren,
}
return after_brace
end
--- Parse: `MipsAtomComp_Proc_(<name>, { <body> })` — body is inside the LAST `{` in args.
--- @param source string
--- @param pos integer
--- @param ident_end integer
--- @param line_of fun(pos: integer): integer
--- @param out SourceScan
--- @return integer
local function parse_mips_atom_comp_proc(source, pos, ident_end, line_of, out)
local open_paren = duffle.skip_ws_and_cmt(source, ident_end)
if source:sub(open_paren, open_paren) ~= "(" then return open_paren + 1 end
local inner, after_paren = duffle.read_parens(source, open_paren)
-- Find the LAST `{` in inner (the body brace, not any potential embedded braces in expressions).
local last_brace_pos = nil
for search_pos = #inner, 1, -1 do
if inner:sub(search_pos, search_pos) == "{" then last_brace_pos = search_pos; break end
end
if not last_brace_pos then return after_paren end
-- Use duffle.read_braces to find the matching close brace.
-- Replaces the pre-refactor hand-rolled depth tracker (~25 LOC of `if c == 123 then depth = depth + 1 ...`).
-- If close_pos is past the end of inner, the brace didn't match (malformed input); skip.
local body, close_pos = duffle.read_braces(inner, last_brace_pos)
if close_pos > #inner + 1 then return after_paren end
local raw_name = inner:match("^%s*([%w_]+)") or "?"
-- Position of body[1] in source = open_paren + 1 (start of inner) + last_brace_pos + 1 (past '{').
local body_off = open_paren + 2 + last_brace_pos
out.atoms[#out.atoms + 1] = {
line = line_of(pos), name = strip_ac_prefix(raw_name), body = body, body_off = body_off,
kind = "comp_proc", raw_name = raw_name,
ident_pos = pos, after_paren = after_paren,
}
return after_paren
end
--- Parse: `MipsCode code_<name> { <body> }` (raw atom form — offsets pass only).
--- @param source string
--- @param pos integer
--- @param ident_end integer
--- @param line_of fun(pos: integer): integer
--- @param out SourceScan
--- @return integer
local function parse_mips_code(source, pos, ident_end, line_of, out)
local next_pos = duffle.skip_ws_and_cmt(source, ident_end)
local next_ident, next_after = duffle.read_ident(source, next_pos)
if not next_ident or #next_ident <= 5 or next_ident:sub(1, 5) ~= "code_" then
return ident_end
end
local atom_name = next_ident:sub(6)
local brace_pos = duffle.scan_to_char(source, "{", next_after)
if not brace_pos then return ident_end end
local body, after_brace = duffle.read_braces(source, brace_pos)
out.raw_atoms[#out.raw_atoms + 1] = {
line = line_of(pos), name = atom_name, body = body, body_off = brace_pos + 1,
kind = "raw_atom", raw_name = atom_name,
}
return after_brace
end
--- Parse: `typedef Struct_(<name>) { <fields> }` — only emits when name starts with `Binds_`.
--- @param source string
--- @param pos integer
--- @param ident_end integer
--- @param line_of fun(pos: integer): integer
--- @param out SourceScan
--- @return integer
local function parse_typedef_binds(source, pos, ident_end, line_of, out)
local after_typedef = duffle.skip_ws_and_cmt(source, ident_end)
local id2, id2_end = duffle.read_ident(source, after_typedef)
if id2 ~= "Struct_" then return ident_end end
local open_paren = duffle.skip_ws_and_cmt(source, id2_end)
if source:sub(open_paren, open_paren) ~= "(" then return id2_end end
local inner, after_paren = duffle.read_parens(source, open_paren)
local name = duffle.trim(inner)
local brace = duffle.scan_to_char(source, "{", after_paren)
if not brace then return open_paren + 1 end
local body, after_brace = duffle.read_braces(source, brace)
if name:sub(1, 6) == "Binds_" then
local fields, byte_off = scan_binds_fields(body)
out.binds[#out.binds + 1] = { line = line_of(pos), name = name, fields = fields, bytes = byte_off }
end
return after_brace
end
--- Parse: `_Pragma("mac_X tape_atom words=N")` (operator form).
--- @param source string
--- @param pos integer
--- @param ident_end integer
--- @param line_of fun(pos: integer): integer
--- @param out SourceScan
--- @return integer
local function parse_pragma_macro(source, pos, ident_end, line_of, out)
local open_paren = duffle.skip_ws_and_cmt(source, ident_end)
if source:sub(open_paren, open_paren) ~= "(" then return open_paren + 1 end
local str, str_end = duffle.read_parens(source, open_paren)
str = duffle.trim(str)
if str:sub(1, 1) ~= '"' or str:sub(-1) ~= '"' then return str_end end
local inner = str:sub(2, -2)
local space = duffle.find_byte(inner, 32, 1)
if not space then return str_end end
local name = inner:sub(1, space - 1)
local rest = inner:sub(space + 1)
local eq = duffle.find_byte(rest, 61, 1)
if not eq then return str_end end
local key = duffle.trim(rest:sub(1, eq - 1))
local val = duffle.trim(rest:sub(eq + 1))
if key == "tape_atom words" or key == "words" then
out.macros[#out.macros + 1] = { line = line_of(pos), name = name, words = tonumber(val) or 0 }
end
return str_end
end
--- Parse: `pragma` ident (no-op — directive form `#pragma` is handled by `skip_preprocessor_line` upstream).
--- If we reach this parser it means the directive skip didn't fire, which can happen for non-#-prefixed pragma.
--- Just advance past the ident.
--- @param source string
--- @param pos integer
--- @param ident_end integer
--- @param line_of fun(pos: integer): integer
--- @param out SourceScan
--- @return integer
local function parse_pragma_dummy(source, pos, ident_end, line_of, out)
return ident_end
end
-- ════════════════════════════════════════════════════════════════════════════
-- DECL_PARSERS — data-driven construct dispatch (the plex pattern)
-- ════════════════════════════════════════════════════════════════════════════
--
-- Each entry maps a leading ident to its parser function. The main scan_source() loop is one line of dispatch:
-- local parser = DECL_PARSERS[ident]; if parser then pos = parser(...) end
--
-- Adding a new construct = 1 row here + 1 parser function above.
local DECL_PARSERS = {
MipsAtom_ = parse_mips_atom,
MipsAtomComp_ = parse_mips_atom_comp,
MipsAtomComp_Proc_ = parse_mips_atom_comp_proc,
MipsCode = parse_mips_code,
typedef = parse_typedef_binds,
_Pragma = parse_pragma_macro,
pragma = parse_pragma_dummy,
}
-- ════════════════════════════════════════════════════════════════════════════
-- The single source walker
-- ════════════════════════════════════════════════════════════════════════════
@@ -191,13 +462,16 @@ end
--- @return table -- SourceScan { atoms, raw_atoms, binds, atom_infos, macros, line_of }
local function scan_source(source)
local line_of = duffle.LineIndex(source)
local atoms = {}
local raw_atoms = {}
local binds = {}
local atom_infos = {}
local macros = {}
local pos = 1
local src_len = #source
local out = {
atoms = {},
raw_atoms = {},
binds = {},
atom_infos = {},
macros = {},
line_of = line_of,
}
local pos = 1
local src_len = #source
while pos <= src_len do
pos = duffle.skip_ws_and_cmt(source, pos)
@@ -206,250 +480,29 @@ local function scan_source(source)
-- Skip preprocessor directives (#define / #include / #pragma / etc).
-- _Pragma is an operator (not a directive) — it doesn't start with #.
local pp_pos = duffle.skip_preprocessor_line(source, pos)
if pp_pos then pos = pp_pos; goto continue end
-- Skip C qualifiers (static, const, etc.) that may precede a declaration.
pos = scan_skip_qualifiers(source, pos)
if pos > src_len then break end
local ident, ident_end = duffle.read_ident(source, pos)
-- scan: <ident>
if not ident then pos = pos + 1; goto continue end
-- ── MipsAtom_ / MipsAtomComp_ / MipsAtomComp_Proc_ ──
if ident == "MipsAtom_" or ident == "MipsAtomComp_" or ident == "MipsAtomComp_Proc_" then
local is_atom = ident == "MipsAtom_"
local is_comp = ident == "MipsAtomComp_"
local is_proc = ident == "MipsAtomComp_Proc_"
local kind = is_atom and "atom" or (is_comp and "comp_bare" or "comp_proc")
local open_paren = duffle.skip_ws_and_cmt(source, ident_end)
if source:sub(open_paren, open_paren) ~= "(" then pos = open_paren + 1; goto continue end
local inner, after_paren = duffle.read_parens(source, open_paren)
-- scan: <ident>(<args>)
if is_proc then
-- MipsAtomComp_Proc_(name, { body }) — body is inside the LAST { } in args.
local last_brace_pos
for search_pos = #inner, 1, -1 do
if inner:sub(search_pos, search_pos) == "{" then last_brace_pos = search_pos; break end
end
if last_brace_pos then
local depth = 1
local inner_pos = last_brace_pos + 1
while inner_pos <= #inner and depth > 0 do
local c = inner:byte(inner_pos)
if c == 123 then depth = depth + 1; inner_pos = inner_pos + 1
elseif c == 125 then depth = depth - 1; if depth == 0 then break end; inner_pos = inner_pos + 1
elseif c == 40 then local _, a = duffle.read_parens(inner, inner_pos); inner_pos = a
elseif c == 91 then local _, a = duffle.read_brackets(inner, inner_pos); inner_pos = a
elseif c == 34 or c == 39 then inner_pos = duffle.skip_str_or_cmt(inner, inner_pos) + 1
else inner_pos = inner_pos + 1 end
end
if depth == 0 then
-- scan: <ident>(<name>, { <body> })
local name_match = inner:match("^%s*([%w_]+)")
local raw_name = name_match or "?"
-- Strip "ac_" prefix for component names (components pass convention).
local name = raw_name
if #raw_name > 3 and raw_name:sub(1, 3) == "ac_" then
name = raw_name:sub(4)
end
local body = inner:sub(last_brace_pos + 1, inner_pos - 1)
local body_off = open_paren + 1 + last_brace_pos
atoms[#atoms + 1] = {
line = line_of(pos), name = name, body = body, body_off = body_off + 1,
kind = kind, raw_name = raw_name,
ident_pos = pos, after_paren = after_paren,
args = nil, comment = nil,
}
end
end
pos = after_paren
else
-- MipsAtom_(name) { body } OR MipsAtomComp_(name) { body }
local name_start = 1
while name_start <= #inner and inner:sub(name_start, name_start):match("[%s]") do name_start = name_start + 1 end
local name_end = name_start
while name_end <= #inner and inner:sub(name_end, name_end):match("[%w_]") do name_end = name_end + 1 end
local raw_name = inner:sub(name_start, name_end - 1)
-- scan: <ident>(<name>)
if raw_name ~= "" then
local brace = duffle.scan_to_char(source, "{", after_paren)
-- scan: <ident>(<name>) {
if brace then
local body, after_brace = duffle.read_braces(source, brace)
-- scan: <ident>(<name>) { <body> }
-- Strip "ac_" prefix for component names (components pass convention).
local disp_name = raw_name
if is_comp and #raw_name > 3 and raw_name:sub(1, 3) == "ac_" then
disp_name = raw_name:sub(4)
end
atoms[#atoms + 1] = {
line = line_of(pos), name = disp_name, body = body, body_off = brace + 1,
kind = kind, raw_name = raw_name,
ident_pos = pos, after_paren = after_paren,
args = nil, comment = nil,
}
pos = after_brace
if pp_pos then
pos = pp_pos
else
-- Skip C qualifiers (static, const, etc.) that may precede a declaration.
pos = scan_skip_qualifiers(source, pos)
if pos <= src_len then
local ident, ident_end = duffle.read_ident(source, pos)
if ident then
local parser = DECL_PARSERS[ident]
if parser then
pos = parser(source, pos, ident_end, line_of, out)
else
pos = open_paren + 1
-- Unrecognized ident — advance past it.
pos = ident_end
end
else
pos = open_paren + 1
pos = pos + 1
end
end
-- For MipsAtom_ entries: check if atom_info(...) follows.
if is_atom then
local lookahead = duffle.skip_ws_and_cmt(source, after_paren)
local look_ident, look_end = duffle.read_ident(source, lookahead)
-- scan: MipsAtom_(<name>) <look_ident>
if look_ident == "atom_info" then
local info_open = duffle.skip_ws_and_cmt(source, look_end)
if source:sub(info_open, info_open) == "(" then
local info_inner, info_after = duffle.read_parens(source, info_open)
-- scan: MipsAtom_(<name>) atom_info(<binds>, <reads>, <writes>)
-- Find the atom name from the just-parsed atom entry (last one added).
local last_atom = atoms[#atoms]
local atom_name = last_atom and last_atom.raw_name or "?"
local ai_binds, ai_reads, ai_writes = scan_atom_info_subcalls(info_inner)
atom_infos[#atom_infos + 1] = {
atom_name = atom_name, binds = ai_binds,
reads = ai_reads or {}, writes = ai_writes or {},
info_line = line_of(lookahead),
}
-- Don't advance pos past info_after — the body { ... } still needs to be skipped
-- by the brace scan below. But if there's no body (forward decl), advance.
local body_brace = duffle.scan_to_char(source, "{", info_after)
if body_brace then
local _, after_body = duffle.read_braces(source, body_brace)
pos = after_body
else
pos = info_after
end
end
end
end
goto continue
end
-- ── MipsCode code_<name> { body } (raw atom form — offsets pass only) ──
if ident == "MipsCode" then
local next_pos = duffle.skip_ws_and_cmt(source, ident_end)
local next_ident, next_after = duffle.read_ident(source, next_pos)
-- scan: MipsCode <next_ident>
if next_ident and #next_ident > 5 and next_ident:sub(1, 5) == "code_" then
local atom_name = next_ident:sub(6)
-- scan: MipsCode code_<name>
local brace_pos = duffle.scan_to_char(source, "{", next_after)
-- scan: MipsCode code_<name> {
if brace_pos then
local body, after_brace = duffle.read_braces(source, brace_pos)
-- scan: MipsCode code_<name> { <body> }
raw_atoms[#raw_atoms + 1] = {
line = line_of(pos), name = atom_name, body = body, body_off = brace_pos + 1,
kind = "raw_atom", raw_name = atom_name,
}
pos = after_brace
goto continue
end
end
pos = ident_end
goto continue
end
-- ── typedef Struct_(Binds_X) { fields } ──
if ident == "typedef" then
local after_typedef = duffle.skip_ws_and_cmt(source, ident_end)
local id2, id2_end = duffle.read_ident(source, after_typedef)
-- scan: typedef <id2>
if id2 == "Struct_" then
local open_paren = duffle.skip_ws_and_cmt(source, id2_end)
if source:sub(open_paren, open_paren) == "(" then
local inner, after_paren = duffle.read_parens(source, open_paren)
-- scan: typedef Struct_(<name>)
local name = duffle.trim(inner)
local brace = duffle.scan_to_char(source, "{", after_paren)
-- scan: typedef Struct_(<name>) {
if brace then
local body, after_brace = duffle.read_braces(source, brace)
-- scan: typedef Struct_(<name>) { <fields> }
if name:sub(1, 6) == "Binds_" then
local fields, byte_off = scan_binds_fields(body)
binds[#binds + 1] = { line = line_of(pos), name = name, fields = fields, bytes = byte_off }
end
pos = after_brace
goto continue
end
pos = open_paren + 1
goto continue
end
pos = id2_end or (after_typedef + 1)
goto continue
end
pos = ident_end
goto continue
end
-- ── _Pragma("mac_X tape_atom words=N") (operator form) ──
if ident == "_Pragma" then
local open_paren = duffle.skip_ws_and_cmt(source, ident_end)
if source:sub(open_paren, open_paren) == "(" then
local str, str_end = duffle.read_parens(source, open_paren)
-- scan: _Pragma(<string>)
str = duffle.trim(str)
if str:sub(1, 1) == '"' and str:sub(-1) == '"' then
local inner = str:sub(2, -2)
local space = duffle.find_byte(inner, 32, 1)
if space then
local name = inner:sub(1, space - 1)
local rest = inner:sub(space + 1)
local eq = duffle.find_byte(rest, 61, 1)
if eq then
local key = duffle.trim(rest:sub(1, eq - 1))
local val = duffle.trim(rest:sub(eq + 1))
if key == "tape_atom words" or key == "words" then
macros[#macros + 1] = { line = line_of(pos), name = name, words = tonumber(val) or 0 }
end
end
end
end
pos = str_end
goto continue
end
pos = open_paren + 1
goto continue
end
-- ── #pragma mac_X tape_atom words=N (directive form) ──
-- (preprocessor skip above handles # lines, but pragma is an ident here
-- only if it appeared without a leading # — which happens when the
-- preprocessor skip didn't fire because the # was on a previous line.
-- The annotation pass handles this via its own skip_preprocessor_line,
-- but scan_source handles it here by checking the ident.)
if ident == "pragma" then
-- This shouldn't normally fire — #pragma lines are skipped by
-- skip_preprocessor_line above. If we get here, it's a _Pragma
-- variant or a non-#-prefixed pragma. Just advance.
pos = ident_end
goto continue
end
-- ── Unrecognized ident — advance past it ──
pos = ident_end
::continue::
end
return {
atoms = atoms,
raw_atoms = raw_atoms,
binds = binds,
atom_infos = atom_infos,
macros = macros,
line_of = line_of,
}
return out
end
-- ════════════════════════════════════════════════════════════════════════════
scripts/passes/static_analysis.lua
+101 -80
View File
@@ -14,7 +14,7 @@
--- `["static-analysis"] = { module = "passes.static_analysis", kind = "validation", deps = {"word-counts", "components"},
--- out = { { kind = "report", path_template = "<out_root>/<basename>.static_analysis.txt" } } }`
---
--- **Conventions**: tabs (1/level), EmmyLua annotations, no regex, Lua 5.3 compatible. See `lua.md` in the ps1-ai styleguides.
--- **Conventions**: tabs (1/level), EmmyLua annotations, no regex, Lua 5.3 compatible.
-- ════════════════════════════════════════════════════════════════════════════
-- Module-scope requires + package.path setup
@@ -114,25 +114,6 @@ local OUTPUT_EXTENSION = ".static_analysis.txt"
--- @field findings Finding[] -- findings for this atom
--- @field total_cycles integer -- sum of token cycle costs
-- ════════════════════════════════════════════════════════════════════════════
-- Source scanning — delegated to duffle.scan_source (ps1_meta.lua pre-scans)
-- ════════════════════════════════════════════════════════════════════════════
--
-- The orchestrator calls duffle.scan_source once per source and stashes the fat SourceScan in src.scan.
-- validate() below reads from src.scan — no source walking in this pass.
-- ════════════════════════════════════════════════════════════════════════════
-- Body tokenizer (top-level comma splitter + per-token classification)
-- ════════════════════════════════════════════════════════════════════════════
--- Build a map: `body_relative_char_offset` -> `body_relative_line`.
--- Used by the checks to convert per-token offsets in the body to lina numbers relative to the start of `body`.
--- The atom's source-line of the body-start is added by the caller.
---
-- NOTE: `tokenize_body` and `build_body_line_index` moved to `duffle.lua` as shared
-- memoized utilities (`duffle.tokenize_body`, `duffle.build_body_line_index`).
-- The local copies were deleted; all callers now use the duffle versions.
-- ════════════════════════════════════════════════════════════════════════════
-- classify_tokens — per-token classification (the plex's pre-computed data layer)
-- ════════════════════════════════════════════════════════════════════════════
@@ -156,16 +137,30 @@ local OUTPUT_EXTENSION = ".static_analysis.txt"
-- Checks that need "how many nops before token i" use `tok_class.nop_prefix` instead of walking backwards.
--- @class TokClass
--- @field ident string -- leading identifier
--- @field nop_words integer -- 0/1/2
--- @field nop_prefix integer -- consecutive nop words before this token
--- @field is_yield boolean
--- @field is_atom_label boolean
--- @field label_name string|nil -- for atom_label(name)
--- @field is_branch boolean
--- @field branch_label string|false|nil -- for branch_*(..., atom_offset(F, label))
--- @field is_load_word boolean
--- @field is_store_word boolean
--- @field ident string -- leading identifier
--- @field nop_words integer -- 0/1/2
--- @field nop_prefix integer -- consecutive nop words before this token
--- @field is_yield boolean
--- @field is_atom_label boolean
--- @field label_name string|nil -- for atom_label(name)
--- @field is_branch boolean
--- @field branch_label string|false|nil -- for branch_*(..., atom_offset(F, label))
--- @field is_load_word boolean
--- @field is_store_word boolean
--- @field mac_format_shape string|nil -- "f3" / "g4" etc. for mac_format_X_color; nil otherwise
--- @field is_gte_store boolean -- ident matches `mac_gte_store_<shape>`
--- @field is_ot_tag boolean -- ident matches `mac_insert_ot_tag_<shape>`
--- @field writes_r_prim_cursor boolean -- store_word targeting R_PrimCursor
--- @field reads_r_tape_ptr boolean -- any token referencing R_TapePtr
--- @field o_arg1 string|nil -- first arg of O_(<a>, <b>) captures; nil for non-O_ tokens
--- @field o_arg2 string|nil -- second arg of O_(<a>, <b>) captures
--- @field s_arg1 string|nil -- arg of S_(<a>) captures; nil for non-S_ tokens
-- Patterns for O_(<arg1>, <arg2>) and S_(<arg>) captures. UNANCHORED — the substring can appear
-- anywhere in the token (e.g., `load_word(R_T0, R_TapePtr, O_(Binds_X, field))` matches at position ~24).
-- The binds_name match is deferred to check_abi_handoff (which compares tc.o_arg1 == atom.info.binds).
local O_PATTERN = "O_%(([%w_]+),%s*([%w_]+)%s*%)"
local S_PATTERN = "S_%(([%w_]+)%s*%)"
local function classify_tokens(tokens)
local n = #tokens
@@ -186,6 +181,16 @@ local function classify_tokens(tokens)
local is_load_word = ident == "load_word"
local is_store_word = ident == "store_word"
-- Per-check pre-computes (R3 lift). Each pre-compute eliminates one per-token regex/string-find
-- call from check_abi_handoff / check_gpu_portstore_shape.
local mac_format_shape = nil
local is_gte_store = false
local is_ot_tag = false
local writes_r_prim_cursor = false
local reads_r_tape_ptr = false
local o_arg1, o_arg2 = nil, nil
local s_arg1 = nil
if ident == "atom_label" then
is_atom_label = true
label_name = tok:match("^atom_label%s*%(%s*([%w_]+)%s*%)")
@@ -194,17 +199,40 @@ local function classify_tokens(tokens)
branch_label = tok:match("atom_offset%s*%([^,]+,%s*([%w_]+)%s*%)") or false
end
-- mac_format_X_color / mac_gte_store_<shape> / mac_insert_ot_tag_<shape> (used by check_gpu_portstore_shape).
local shape = ident:match("^mac_format_([%w_]+)_color$")
if shape then mac_format_shape = shape end
if ident:match("^mac_gte_store_[%w_]+$") then is_gte_store = true end
if ident:match("^mac_insert_ot_tag_[%w_]+$") then is_ot_tag = true end
-- O_(<arg1>, <arg2>) / S_(<arg>) captures (used by check_abi_handoff).
-- Cheap pattern match — anchored, fails fast on non-matching tokens.
o_arg1, o_arg2 = tok:match(O_PATTERN)
if not o_arg1 then s_arg1 = tok:match(S_PATTERN) end
-- R_TapePtr + R_PrimCursor references (used by check_abi_handoff / check_gpu_portstore_shape).
if tok:find("R_TapePtr", 1, true) then reads_r_tape_ptr = true end
if is_store_word and tok:find("R_PrimCursor", 1, true) then writes_r_prim_cursor = true end
tc[tok_idx] = {
ident = ident,
nop_words = nop_words,
nop_prefix = nop_run,
is_yield = is_yield,
is_atom_label = is_atom_label,
label_name = label_name,
is_branch = is_branch,
branch_label = branch_label,
is_load_word = is_load_word,
is_store_word = is_store_word,
ident = ident,
nop_words = nop_words,
nop_prefix = nop_run,
is_yield = is_yield,
is_atom_label = is_atom_label,
label_name = label_name,
is_branch = is_branch,
branch_label = branch_label,
is_load_word = is_load_word,
is_store_word = is_store_word,
mac_format_shape = mac_format_shape,
is_gte_store = is_gte_store,
is_ot_tag = is_ot_tag,
writes_r_prim_cursor = writes_r_prim_cursor,
reads_r_tape_ptr = reads_r_tape_ptr,
o_arg1 = o_arg1,
o_arg2 = o_arg2,
s_arg1 = s_arg1,
}
-- Advance the nop run for the NEXT token.
if nop_words > 0 then
@@ -406,29 +434,28 @@ local function check_abi_handoff(atom, pipe_ctx, findings)
local tc = atom.paths.tok_class
local found_field_set = {}
local found_advance = false
local bind_re = "O_%(" .. binds_name .. ",%s*([%w_]+)%s*%)"
for tok_idx, t in ipairs(tokens) do
local tok = t.tok
if tc[tok_idx].is_load_word then
if tok:find("R_TapePtr", 1, true) and tok:find("O_(" .. binds_name .. ",", 1, true) then
local field = tok:match(bind_re)
-- scan: load_word(R_*, R_TapePtr, O_(<Binds_X>, <field>))
if field then
found_field_set[field] = true
else
local body_line = atom.line + line_in_body[t.rel]
findings[#findings + 1] = {
atom = atom.name, line = body_line,
check = "abi_handoff", kind = "error",
msg = string.format("%s at line %d has load_word(R_TapePtr, O_(%s, <non-ident>)); expected O_(%s, <field>)",
atom.name, body_line, binds_name, binds_name),
}
end
-- Reads from tc_entry fields pre-computed by classify_tokens (R3 lift). Eliminates 3 per-token
-- string-find/match calls (R_TapePtr + O_(binds_name,...) + bind_re) → 3 O(1) field reads.
for tok_idx = 1, #tokens do
local tc_entry = tc[tok_idx]
-- scan: load_word(R_*, R_TapePtr, O_(<Binds_X>, <field>))
if tc_entry.is_load_word and tc_entry.reads_r_tape_ptr and tc_entry.o_arg1 == binds_name then
local field = tc_entry.o_arg2
if field then
found_field_set[field] = true
else
local body_line = atom.line + line_in_body[tokens[tok_idx].rel]
findings[#findings + 1] = {
atom = atom.name, line = body_line,
check = "abi_handoff", kind = "error",
msg = string.format("%s at line %d has load_word(R_TapePtr, O_(%s, <non-ident>)); expected O_(%s, <field>)",
atom.name, body_line, binds_name, binds_name),
}
end
end
if tok:find("R_TapePtr", 1, true)
and tok:find("S_(" .. binds_name .. ")", 1, true) then
-- scan: add_ui_self(R_TapePtr, S_(<Binds_X>))
-- scan: add_ui_self(R_TapePtr, S_(<Binds_X>))
if tc_entry.reads_r_tape_ptr and tc_entry.s_arg1 == binds_name then
found_advance = true
end
end
@@ -468,7 +495,6 @@ end
--- - Atoms containing a `mac_<name>(...)` call whose name is not in duffle.GP0_MACRO_CONTRIB emit a "new macro; update duffle.GP0_MACRO_CONTRIB" advisory.
---
--- Applies only to `kind = "atom"` (baked atoms). Components don't emit full primitives.
--- Stage 2: signature uniformized to `(atom, pipe_ctx, findings)` — pipe_ctx is ignored here.
local function check_gpu_portstore_shape(atom, pipe_ctx, findings)
if atom.kind ~= "atom" then return end
local tokens = atom.paths.tokens
@@ -479,31 +505,30 @@ local function check_gpu_portstore_shape(atom, pipe_ctx, findings)
local contrib = 0
local saw_format = false
local saw_prim_write = false
for tok_idx, t in ipairs(tokens) do
local tok = t.tok
local ident = tc[tok_idx].ident
-- Match `mac_format_<shape>_color(...)` and strip `_color`
-- to get the bare shape suffix (f3 / g4 / etc).
local shape = ident:match("^mac_format_([%w_]+)_color$")
-- Reads from tc_entry fields pre-computed by classify_tokens (R3 lift). Eliminates 4 per-token
-- string matches (mac_format_X_color + mac_gte_store_<shape> + mac_insert_ot_tag_<shape> + R_PrimCursor)
for tok_idx = 1, #tokens do
local tc_entry = tc[tok_idx]
local shape = tc_entry.mac_format_shape
if shape and duffle.GP0_CMD_BY_SHAPE[shape] then
if not cmd_byte then
cmd_byte = duffle.GP0_CMD_BY_SHAPE[shape]
cmd_line = atom.line + line_in_body[t.rel]
cmd_line = atom.line + line_in_body[tokens[tok_idx].rel]
end
saw_format = true
local contrib_key = "mac_format_" .. shape .. "_color"
local n = duffle.GP0_MACRO_CONTRIB[contrib_key]
local n = duffle.GP0_MACRO_CONTRIB["mac_format_" .. shape .. "_color"]
if n then contrib = contrib + n end
end
if ident:match("^mac_gte_store_[%w_]+$") then
local n = duffle.GP0_MACRO_CONTRIB[ident]
if tc_entry.is_gte_store then
local n = duffle.GP0_MACRO_CONTRIB[tc_entry.ident]
if n then contrib = contrib + n end
end
if ident:match("^mac_insert_ot_tag_[%w_]+$") then
local n = duffle.GP0_MACRO_CONTRIB[ident]
if tc_entry.is_ot_tag then
local n = duffle.GP0_MACRO_CONTRIB[tc_entry.ident]
if n then contrib = contrib + n end
end
if tc[tok_idx].is_store_word and tok:find("R_PrimCursor", 1, true) then
if tc_entry.writes_r_prim_cursor then
saw_prim_write = true
end
end
@@ -536,10 +561,6 @@ end
-- Check #5: per-atom cycle budget (uses analyze_atom_paths's unknown_macros)
-- ════════════════════════════════════════════════════════════════════════════
-- NOTE: `token_cycles`, `find_atom_labels`, `find_branch_targets` were removed
-- when `classify_tokens` (the pre-computed per-token classification) replaced them.
-- The classification lives on `atom.paths.tok_class`; analyze_atom_paths reads it.
--- Walk all paths through an atom body and return per-path cycle sums.
--- Builds a tiny CFG: each token has a "next" pointer; branches have two (fall-through + taken).
--- The BD-slot nop after a branch is absorbed into the branch's cost (MIPS-accurate: BD slot always runs),
scripts/passes/word_count_eval.lua
+17 -40
View File
@@ -28,17 +28,13 @@ local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua")
-- Constants
-- ════════════════════════════════════════════════════════════════════════════
-- Windows separator chars — used to convert `dir` output (which uses `\`) into POSIX paths (which our scripts expect).
-- Windows separator char — used by `fname:match` to recognize `.macs.h` files.
local PATH_SEP_BACKSLASH = "\\"
local PATH_SEP_FORWARD = "/"
-- Fallback glob command (subprocess). Used when `lfs` (LuaFileSystem) is not available.
-- Scoped to `code\` to avoid walking `.git/`, `toolchain/`, `build/`, etc.
local DIR_GLOB_CMD = 'dir /b /s "%s\\code\\%s" 2>nul'
-- Try to load lfs (LuaFileSystem). If available, scan_dir uses native directory enumeration (~2ms)
-- instead of spawning `dir /b /s` as a subprocess (~56ms). Built by update_deps.ps1 into toolchain/lfs/lfs.dll.
local lfs = pcall(require, "lfs") and require("lfs") or nil
-- Required native extension: lfs (LuaFileSystem). Built by `update_deps.ps1` to
-- `toolchain/lfs/lfs.dll` and wired into package.cpath by `scripts/duffle_paths.lua`.
-- If lfs is missing, `require` throws — fail loud per the build-tool convention.
local lfs = require("lfs")
-- ════════════════════════════════════════════════════════════════════════════
-- Type declarations
@@ -106,26 +102,20 @@ end
-- └────────────────────────────────────────────────────────────────────┘
--- Recursively scan a directory for files matching a glob suffix.
--- No regex per the no_regex constraint — uses plain byte matching via `dir /b /s` on Windows.
---
--- The `.macs.h` files produced by the components pass always live at `<project_root>/<module>/gen/`.
--- We can shortcut the `dir /b /s` walk by listing modules first (one `dir /b /ad`), then walking each `<module>/gen/`
--- (one `dir /b` per module, no recursion).
--- For projects with 2 modules and 0 .macs.h files, this drops the cost from ~52ms
--- (full recursive walk of the entire project tree) to ~5ms.
--- Native walk via lfs.attributes + lfs.dir: ~2ms vs ~56ms for the prior `dir /b /s` subprocess.
---
--- @param dir string -- directory to scan (absolute or relative)
--- @param suffix string -- file pattern, e.g. "*.macs.h"
--- @return string[]
-- Cache the scan_dir result per (dir, suffix) in package.loaded.
-- Each `io.popen` call on Windows is ~50-100ms of subprocess overhead, so caching the result saves a fixed cost on every build.
-- Cache the scan_dir result per (dir, suffix) in package.loaded.
-- The cache persists for the lifetime of the Lua process (cleared when ps1_meta.lua exits).
-- If a build removes/creates .macs.h files mid-process, the caller can invalidate by calling `M._invalidate_scan_cache()`.
local SCAN_CACHE_KEY = "__word_count_eval_scan_cache__"
--- Scan `code/` for files matching `suffix` (e.g. `*.macs.h`).
--- Uses `lfs` (LuaFileSystem) when available — native directory enumeration at ~2ms.
--- Falls back to `dir /b /s` subprocess (~56ms) when `lfs` is not compiled.
--- Native directory enumeration via lfs (~2ms). Zero subprocess spawns.
---
--- @param dir string -- project root directory
--- @param suffix string -- file pattern, e.g. "*.macs.h"
@@ -137,33 +127,20 @@ function M.scan_dir(dir, suffix)
if cache and cache[key] then return cache[key] end
local results = {}
if lfs then
-- Native walk: list code/<module>/gen/ for matching files. Zero subprocess spawns.
local code_dir = dir .. "/code"
if lfs.attributes(code_dir, "mode") == "directory" then
for mod_name in lfs.dir(code_dir) do
if mod_name ~= "." and mod_name ~= ".." then
local gen_path = code_dir .. "/" .. mod_name .. "/gen"
if lfs.attributes(gen_path, "mode") == "directory" then
for fname in lfs.dir(gen_path) do
if fname:match("%.macs%.h$") then
results[#results + 1] = gen_path .. "/" .. fname
end
local code_dir = dir .. "/code"
if lfs.attributes(code_dir, "mode") == "directory" then
for mod_name in lfs.dir(code_dir) do
if mod_name ~= "." and mod_name ~= ".." then
local gen_path = code_dir .. "/" .. mod_name .. "/gen"
if lfs.attributes(gen_path, "mode") == "directory" then
for fname in lfs.dir(gen_path) do
if fname:match("%.macs%.h$") then
results[#results + 1] = gen_path .. "/" .. fname
end
end
end
end
end
else
-- Fallback: single `dir /b /s` subprocess scoped to code\.
local pipe = io.popen(DIR_GLOB_CMD:format(dir, suffix))
if pipe then
for raw_line in pipe:lines() do
results[#results + 1] = raw_line:gsub(PATH_SEP_BACKSLASH, PATH_SEP_FORWARD)
end
pipe:close()
end
end
-- Cache the result (including empty results).