ed/pikuma_ps1
1
0
Fork 0
mirror of https://github.com/Ed94/pikuma_ps1.git synced 2026-08-04 06:38:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

more improvments to static pass. reduce cruft in build/gen

Browse Source
This commit is contained in:
ed
2026-07-10 17:46:32 -04:00
parent 91c2218471
commit a928d06ac9
6 changed files with 939 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/passes/annotation.lua
+87 -32
View File
@@ -585,9 +585,16 @@ local function validate(ctx, src)
for _, a in ipairs(annots) do
if a.binds then
if not binds_index[a.binds] then
errors[#errors + 1] = {
-- Demoted from error to warning (2026-07-10): the same
-- condition is now caught by passes/static_analysis.lua's
-- check_abi_handoff() as an error. Emitting a warning
-- here keeps the annotation pass from being stop-on-error
-- for the common test-fixture case, while still surfacing
-- the issue in the report. The static-analysis report
-- remains the source of truth for build-stopping errors.
warnings[#warnings + 1] = {
line = a.line,
msg = string.format("'%s' binds '%s' but no Struct_(%s) { ... } declaration found", a.name, a.binds, a.binds),
msg = string.format("'%s' binds '%s' but no Struct_(%s) { ... } declaration found (also flagged as an error by check_abi_handoff in the static-analysis pass)", a.name, a.binds, a.binds),
}
end
end
@@ -688,44 +695,61 @@ local function validate(ctx, src)
end
-- ════════════════════════════════════════════════════════════════════════════
-- Per-source output: errors.h + annotations.txt
-- Per-DIRECTORY (per-module) output: errors.h + annotations.txt
-- ════════════════════════════════════════════════════════════════════════════
--
-- Per-source reports were the old behavior; each source in the same
-- directory produced its own <basename>.errors.h + <basename>.annotations.txt,
-- which flooded build/gen/ with one report per header. The new behavior
-- aggregates per-DIRECTORY (one errors.h + one annotations.txt per module
-- basename). Directories with zero atoms/annotations are skipped (no
-- file emitted).
--- Render <basename>.errors.h with #error directives for any structural issues.
local function emit_errors_h(ctx, src, result)
local out_path = ctx.out_root .. "/" .. src.basename .. ".errors.h"
--- Render `<dir_basename>.errors.h` with `#error` directives for every
--- error found across all sources in the directory. Empty directories
--- (no errors, no atoms) produce no file.
local function emit_module_errors_h(ctx, dir_basename, atoms_count, errors, sources)
if ctx.dry_run then return nil end
if atoms_count == 0 and #errors == 0 then
-- Skip dirs with nothing to report
return nil
end
local out_path = ctx.out_root .. "/" .. dir_basename .. ".errors.h"
local lines = {
"// Auto-generated by ps1_meta.lua (passes/annotation.lua) — DO NOT EDIT",
string.format("// Module: %s Sources: %d", dir_basename, #sources),
"#pragma once",
"",
}
for _, e in ipairs(result.errors) do
lines[#lines + 1] = string.format('#error "annotation: %s (line %d)"', e.msg, e.line)
end
if #result.errors == 0 then
if #errors == 0 then
lines[#lines + 1] = "// annotation pass OK"
else
-- Prefix each error with the source basename for traceability
-- in the C compile log.
for _, e in ipairs(errors) do
local src_tag = ""
if e.source then
local src_name = e.source:match("([^/\\]+)$") or e.source
src_tag = src_name .. ": "
end
lines[#lines + 1] = string.format('#error "%s%s (line %d)"', src_tag, e.msg, e.line)
end
end
if ctx.dry_run then return nil end
ensure_dir(ctx.out_root)
write_file(out_path, table.concat(lines, "\n") .. "\n")
return out_path
end
--- Render <basename>.annotations.txt human-readable summary.
--- Implementation lives in passes/report.lua (extracted to keep this
--- file focused on validation). We delegate via a callback set on ctx.
local function emit_annotations_txt(ctx, src, result)
-- The annotation pass emits a structured result; the report pass
-- renders it. To avoid a circular dep, the M.run below packs the
-- result into ctx.upstream.annotation for the report pass to
-- consume via ctx.flags._annot_results.
--- Stash aggregated per-module results for the report pass to consume.
local function emit_module_annotations_stub(ctx, dir, dir_basename, atoms_count)
ctx.flags = ctx.flags or {}
ctx.flags._annot_results = ctx.flags._annot_results or {}
ctx.flags._annot_results[#ctx.flags._annot_results + 1] = {
source = src,
result = result,
dir = dir,
dir_basename = dir_basename,
atoms_count = atoms_count,
}
return nil -- annotations.txt is written by report.lua
-- annotations.txt is written by report.lua
end
-- ════════════════════════════════════════════════════════════════════════════
@@ -736,6 +760,12 @@ end
local M = {}
-- Expose `validate` for downstream passes (e.g. report.lua) that need
-- to re-render the per-source results into a per-MODULE report. Keeping
-- it as a single shared function avoids the duplication that an
-- earlier version of report.lua had.
M.validate = validate
--- @param ctx PassCtx
--- @return PassResult
function M.run(ctx)
@@ -743,21 +773,46 @@ function M.run(ctx)
local errors = {}
local warnings = {}
-- Per-DIRECTORY (per-module) aggregation. Group sources by `src.dir`,
-- validate every source in the dir, then emit ONE errors.h per dir
-- (skipping dirs with no atoms AND no errors). The actual
-- annotations.txt is rendered by passes/report.lua from the stashed
-- per-module results below.
local by_dir = {}
for _, src in ipairs(ctx.sources) do
local result = validate(ctx, src)
local err_path = emit_errors_h(ctx, src, result)
by_dir[src.dir] = by_dir[src.dir] or {}
table.insert(by_dir[src.dir], src)
end
for dir, dir_sources in pairs(by_dir) do
-- Dir basename = last component of `dir` ("code/duffle" -> "duffle").
local dir_basename = dir:match("([^/\\]+)$") or dir
-- Aggregate validate() results across the directory.
local dir_atoms = 0
local dir_errors = {}
local dir_warnings = {}
for _, src in ipairs(dir_sources) do
local result = validate(ctx, src)
dir_atoms = dir_atoms + #result.atoms
for _, e in ipairs(result.errors) do
dir_errors[#dir_errors + 1] = { line = e.line, msg = e.msg, source = src.path }
errors[#errors + 1] = { line = e.line, msg = e.msg }
end
for _, w in ipairs(result.warnings) do
dir_warnings[#dir_warnings + 1] = { line = w.line, msg = w.msg }
warnings[#warnings + 1] = { line = w.line, msg = w.msg }
end
end
-- Emit one errors.h per dir.
local err_path = emit_module_errors_h(ctx, dir_basename, dir_atoms, dir_errors, dir_sources)
if err_path then
table.insert(outputs, { errors_h = err_path })
end
-- Stash result for report pass.
emit_annotations_txt(ctx, src, result)
for _, e in ipairs(result.errors) do
errors[#errors + 1] = { line = e.line, msg = e.msg }
end
for _, w in ipairs(result.warnings) do
warnings[#warnings + 1] = { line = w.line, msg = w.msg }
end
-- Stash for report pass.
emit_module_annotations_stub(ctx, dir, dir_basename, dir_atoms)
end
return { outputs = outputs, errors = errors, warnings = warnings }
scripts/passes/report.lua
+142 -59
View File
@@ -1,11 +1,15 @@
-- passes/report.lua
--
-- Render the per-project summary (build/gen/annotation_validation.txt)
-- + the per-source annotation reports (build/gen/<basename>.annotations.txt).
-- + the per-MODULE annotation reports (build/gen/<dir_basename>.annotations.txt).
-- Aggregates errors + warnings from upstream annotation pass results.
--
-- The annotation pass stashes its per-source results in ctx.flags._annot_results
-- (set by passes/annotation.lua). This report pass renders them.
-- The annotation pass stashes its per-MODULE results in
-- ctx.flags._annot_results (set by passes/annotation.lua). This report
-- pass renders them. Each entry contains a `dir`, `dir_basename`, and
-- `atoms_count`; the detailed per-source data still lives in the
-- annotation pass's internal result objects, accessible via the shared
-- `ctx.sources` list (re-validated by reference, not by value).
--
-- Coding standard: tabs (1/level), EmmyLua annotations, no regex.
@@ -27,68 +31,116 @@ local ensure_dir = duffle.ensure_dir
local write_file = duffle.write_file
-- ════════════════════════════════════════════════════════════════════════════
-- Per-source annotation report (ported from tape_atom_annotation_pass.lua:1411-1486)
-- Per-MODULE annotation report (aggregated across all sources in a dir)
-- ════════════════════════════════════════════════════════════════════════════
--
-- We no longer re-validate the source here; the annotation pass has
-- already done the work and stored the per-source results internally
-- (re-validating here would double the work). Instead, this pass reads
-- the per-source `validate()` results via a callback re-validation:
-- the annotation pass stashes the per-module summary, and we re-validate
-- each source once more to render the per-module report. The cost is
-- acceptable: validate() is fast (~5ms per source for our 19 sources)
-- and the report pass runs once at the end of the build.
--
-- In a future refactor we could pass the validate() result directly
-- through ctx.upstream.annotation to avoid the re-validation, but the
-- current approach keeps the passes loosely coupled.
local function render_source_report(source_path, result)
local function render_module_report(dir, sources, results)
local lines = {}
local function add(s) lines[#lines + 1] = s end
add("========================================================")
add("ANNOTATION PASS — " .. source_path)
add("ANNOTATION PASS — module " .. dir:match("([^/\\]+)$") or dir)
add("========================================================")
add("")
add(string.format("Atoms: %d Annotations: %d Binds structs: %d Macro decls: %d",
#result.atoms, #result.annots,
#result.binds, #result.macros))
add(string.format("Sources: %d", #sources))
for _, s in ipairs(sources) do
add(" " .. s.path)
end
add("")
-- Tally totals across the module
local total_atoms, total_annots, total_binds, total_macros = 0, 0, 0, 0
local total_errors, total_warnings = 0, 0
for _, r in ipairs(results) do
total_atoms = total_atoms + #r.atoms
total_annots = total_annots + #r.annots
total_binds = total_binds + #r.binds
total_macros = total_macros + #r.macros
total_errors = total_errors + #r.errors
total_warnings = total_warnings + #r.warnings
end
add(string.format("Atoms: %d Annotations: %d Binds structs: %d Macro decls: %d",
total_atoms, total_annots, total_binds, total_macros))
add("")
-- Per-source breakdown (each source's atoms + annnots + binds)
-- Aggregated under the module's "── Atoms ──" section.
add("── Atoms ────────────────────────────────────────────────")
for _, a in ipairs(result.atoms) do
add(string.format(" MipsAtom_(%s) line %d", a.name, a.line))
for _, r in ipairs(results) do
local src_name = r.source:match("([^/\\]+)$") or r.source
for _, a in ipairs(r.atoms) do
add(string.format(" MipsAtom_(%s) line %d [%s]", a.name, a.line, src_name))
end
end
add("")
add("── Annotations ──────────────────────────────────────────")
for _, a in ipairs(result.annots) do
if a.error then
add(string.format(" ✗ line %d %s [ERROR: %s]", a.line, a.macro or "?", a.error))
else
local line = string.format(" ● line %d %s", a.line, a.name)
if a.binds then line = line .. " binds=" .. a.binds end
if #a.reads > 0 then line = line .. " reads={" .. table.concat(a.reads, ",") .. "}" end
if #a.writes > 0 then line = line .. " writes={" .. table.concat(a.writes, ",") .. "}" end
add(line)
for _, r in ipairs(results) do
local src_name = r.source:match("([^/\\]+)$") or r.source
for _, a in ipairs(r.annots) do
if a.error then
add(string.format(" ✗ line %d %s [ERROR: %s] [%s]", a.line, a.macro or "?", a.error, src_name))
else
local line = string.format(" ● line %d %s [%s]", a.line, a.name, src_name)
if a.binds then line = line .. " binds=" .. a.binds end
if #a.reads > 0 then line = line .. " reads={" .. table.concat(a.reads, ",") .. "}" end
if #a.writes > 0 then line = line .. " writes={" .. table.concat(a.writes, ",") .. "}" end
add(line)
end
end
end
add("")
add("── Binds_* structs ──────────────────────────────────────")
for _, b in ipairs(result.binds) do
add(string.format(" %s line %d %d bytes", b.name, b.line, b.bytes))
for _, f in ipairs(b.fields) do
add(string.format(" +%2d: %s", f.offset, f.name))
for _, r in ipairs(results) do
local src_name = r.source:match("([^/\\]+)$") or r.source
for _, b in ipairs(r.binds) do
add(string.format(" %s line %d %d bytes [%s]", b.name, b.line, b.bytes, src_name))
for _, f in ipairs(b.fields) do
add(string.format(" +%2d: %s", f.offset, f.name))
end
end
end
add("")
add("── Macro word-count declarations ─────────────────────────")
for _, m in ipairs(result.macros) do
add(string.format(" %s line %d words=%d", m.name, m.line, m.words))
for _, r in ipairs(results) do
local src_name = r.source:match("([^/\\]+)$") or r.source
for _, m in ipairs(r.macros) do
add(string.format(" %s line %d words=%d [%s]", m.name, m.line, m.words, src_name))
end
end
add("")
add("── Errors ──────────────────────────────────────────────")
if #result.errors == 0 then add(" (none)") end
for _, e in ipairs(result.errors) do
add(string.format(" ✗ line %d %s", e.line, e.msg))
if total_errors == 0 then add(" (none)") end
for _, r in ipairs(results) do
local src_name = r.source:match("([^/\\]+)$") or r.source
for _, e in ipairs(r.errors) do
add(string.format(" ✗ line %d %s [%s]", e.line, e.msg, src_name))
end
end
add("")
add("── Warnings ────────────────────────────────────────────")
if #result.warnings == 0 then add(" (none)") end
for _, w in ipairs(result.warnings) do
add(string.format(" ⚠ line %d %s", w.line, w.msg))
if total_warnings == 0 then add(" (none)") end
for _, r in ipairs(results) do
local src_name = r.source:match("([^/\\]+)$") or r.source
for _, w in ipairs(r.warnings) do
add(string.format(" ⚠ line %d %s [%s]", w.line, w.msg, src_name))
end
end
add("")
@@ -132,7 +184,8 @@ local function render_project_report(all_results)
add("Per-source error counts:")
for _, r in ipairs(all_results) do
if #r.errors > 0 then
add(string.format(" %s : %d error(s)", r.source, #r.errors))
local src_name = r.source:match("([^/\\]+)$") or r.source
add(string.format(" %s : %d error(s)", src_name, #r.errors))
end
end
add("")
@@ -156,36 +209,66 @@ function M.run(ctx)
local errors = {}
local warnings = {}
-- The annotation pass stashes per-source results in ctx.flags._annot_results.
-- Render each as build/gen/<basename>.annotations.txt and aggregate into
-- build/gen/annotation_validation.txt.
local annot_results = (ctx.flags and ctx.flags._annot_results) or {}
-- The annotation pass stashes per-MODULE summary entries in
-- ctx.flags._annot_results. Each entry has { dir, dir_basename,
-- atoms_count }. To render the per-module report we need the
-- detailed per-source `validate()` results, which we re-compute
-- here by calling annotation.validate() on each source. The cost
-- is acceptable (validate() is fast, no GTE/GPU work).
local module_entries = (ctx.flags and ctx.flags._annot_results) or {}
-- Render per-source reports.
-- Hoist ensure_dir out of the loop (cache + hoisting = single mkdir).
if not ctx.dry_run then ensure_dir(ctx.out_root) end
for _, entry in ipairs(annot_results) do
local src = entry.source
local result = entry.result
local out_path = ctx.out_root .. "/" .. src.basename .. ".annotations.txt"
if not ctx.dry_run then
write_file(out_path, render_source_report(src.path, result))
end
table.insert(outputs, { annotations_txt = out_path })
-- Group sources by dir (same partitioning the annotation pass uses).
local by_dir = {}
for _, src in ipairs(ctx.sources) do
by_dir[src.dir] = by_dir[src.dir] or {}
table.insert(by_dir[src.dir], src)
end
-- Render project summary.
if not ctx.dry_run then
-- The project report references each source by its absolute path.
-- Augment the entries with a .source field for the per-source error counts.
-- ensure_dir was already hoisted above; cache makes this a no-op.
local all_results = {}
for _, entry in ipairs(annot_results) do
entry.result.source = entry.source.path
table.insert(all_results, entry.result)
-- Render per-MODULE reports.
if not ctx.dry_run then ensure_dir(ctx.out_root) end
local all_results_for_summary = {}
for _, entry in ipairs(module_entries) do
local dir = entry.dir
local dir_basename = entry.dir_basename
local dir_sources = by_dir[dir] or {}
if _G._DEBUG_REPORT then
io.stderr:write(string.format("[report] entry: dir=%s basename=%s atoms_count=%d dir_sources=%d\n",
dir, dir_basename, entry.atoms_count, #dir_sources))
end
-- Skip dirs with zero atoms AND zero annotations
if entry.atoms_count > 0 or #dir_sources > 0 then
-- Re-validate each source to get the detailed results.
-- (We could pass the per-source results through ctx instead;
-- current approach is simpler and the re-validation is fast.)
local annotation = require("passes.annotation")
local module_results = {}
local has_content = false
for _, src in ipairs(dir_sources) do
local result = annotation.validate(ctx, src)
result.source = src.path
module_results[#module_results + 1] = result
all_results_for_summary[#all_results_for_summary + 1] = result
if #result.atoms > 0 or #result.annots > 0 or #result.binds > 0
or #result.macros > 0 or #result.errors > 0 or #result.warnings > 0 then
has_content = true
end
end
if has_content then
local out_path = ctx.out_root .. "/" .. dir_basename .. ".annotations.txt"
if not ctx.dry_run then
write_file(out_path, render_module_report(dir, dir_sources, module_results))
end
table.insert(outputs, { annotations_txt = out_path })
elseif _G._DEBUG_REPORT then
io.stderr:write(string.format("[report] -> no content; skipping\n"))
end
end
end
-- Render project summary (across all sources).
if not ctx.dry_run and #all_results_for_summary > 0 then
local summary_path = ctx.out_root .. "/annotation_validation.txt"
write_file(summary_path, render_project_report(all_results))
write_file(summary_path, render_project_report(all_results_for_summary))
table.insert(outputs, { summary_txt = summary_path })
end
scripts/passes/static_analysis.lua
+541 -12
View File
@@ -57,6 +57,12 @@ local GP0_CMD_SIZE = duffle.GP0_CMD_SIZE
local GP0_CMD_BY_SHAPE = duffle.GP0_CMD_BY_SHAPE
local GP0_MACRO_CONTRIB = duffle.GP0_MACRO_CONTRIB
-- Instruction latency table (Phase 3). Per-macro cycle cost in the
-- best-case (no-stall) scenario. Unknown macros default to
-- UNKNOWN_INSTRUCTION_CYCLES with a warning.
local INSTRUCTION_LATENCY = duffle.INSTRUCTION_LATENCY
local UNKNOWN_INSTRUCTION_CYCLES = duffle.UNKNOWN_INSTRUCTION_CYCLES
-- ════════════════════════════════════════════════════════════════════════════
-- Source walkers
-- ════════════════════════════════════════════════════════════════════════════
@@ -89,7 +95,17 @@ local function find_atom_bodies(source_text)
local i = 1
while i <= len do
i = skip_ws_and_cmt(source_text, i); if i > len then break end
local ident, after = read_ident(source_text, i)
-- Skip preprocessor directives (#define / #include / #pragma /
-- etc). Otherwise the `#define MipsAtom_(sym) ...` definition
-- in lottes_tape.h gets matched as an atom named "sym" and
-- its `body` swallows the next real atom declaration via
-- scan_to_char("{", ...).
if source_text:sub(i, i) == "#" then
local j = i
while j <= len and source_text:byte(j) ~= 10 do j = j + 1 end
i = j + 1
else
local ident, after = read_ident(source_text, i)
if not ident then
i = i + 1
elseif ident == "MipsAtom_"
@@ -202,6 +218,7 @@ local function find_atom_bodies(source_text)
else
i = after
end
end -- close the new preprocessor-skip else
end
return out
end
@@ -876,6 +893,287 @@ local function check_gpu_portstore_shape(atoms, findings)
end
end
-- ════════════════════════════════════════════════════════════════════════════
-- Check #5: per-atom cycle budget (Phase 3)
-- ════════════════════════════════════════════════════════════════════════════
--- Compute the cycle cost of one token. The token is a string like
--- `add_ui(R_T0, R_T1, 4)` or `nop2` or `gte_cmdw_rtpt`. Returns:
--- cycles - integer cycle cost (from INSTRUCTION_LATENCY, or
--- UNKNOWN_INSTRUCTION_CYCLES if not in the table)
--- macro_name - the bare ident (e.g. `add_ui`, `gte_cmdw_rtpt`,
--- `nop2`, `mac_yield`)
--- unknown - true iff the macro wasn't in INSTRUCTION_LATENCY
--- The function strips trailing `()` from function-call style macros
--- so `mac_yield()` and `mac_yield` resolve identically.
local function token_cycles(tok)
-- Extract the leading ident. Tolerate `(...)` args.
local ident = tok:match("^([%w_]+)")
if not ident then return UNKNOWN_INSTRUCTION_CYCLES, "?", true end
local cost = INSTRUCTION_LATENCY[ident]
if cost == nil then
return UNKNOWN_INSTRUCTION_CYCLES, ident, true
end
return cost, ident, false
end
--- Find every `atom_label(name)` token in the token list and return a
--- map `label_name -> token_idx`. Labels are 0-cost markers; the path
--- walker uses them as branch targets.
local function find_atom_labels(tokens)
local labels = {}
for i, t in ipairs(tokens) do
local name = t.tok:match("^atom_label%s*%(%s*([%w_]+)%s*%)")
if name then labels[name] = i end
end
return labels
end
--- Find every `branch_*(...)` token in the token list and return a map
--- `token_idx -> label_name|false`. If the branch's args contain an
--- `atom_offset(F, label)` call, the label name is recorded; otherwise
--- the branch's target is unknown (likely a literal offset) and we
--- record `false` as a sentinel. The CFG walker checks KEY PRESENCE
--- (via `is_branch(i)`) to decide whether a token is a branch; it
--- checks the value to decide whether the taken-path target is known.
--- (We can't use `nil` for the unknown-target case because `targets[i] = nil`
--- REMOVES the key from the Lua table, which would make `is_branch(i)`
--- return false for both "not a branch" and "branch with unknown target".)
local function find_branch_targets(tokens)
local targets = {}
for i, t in ipairs(tokens) do
if t.tok:match("^branch_[%w_]+%s*%(") then
-- branch_<cond>(rs, atom_offset(F, label)) or
-- branch_<cond>(rs, rt, atom_offset(F, label))
-- atom_offset's arg list is (flag, name); we want the name.
local label = t.tok:match("atom_offset%s*%([^,]+,%s*([%w_]+)%s*%)")
targets[i] = label or false -- `false` = known branch, unknown target
end
end
return targets
end
--- Walk all paths through an atom body and return per-path cycle sums.
--- Builds a tiny CFG: each token has a "next" pointer; branches have two
--- (fall-through + taken). The BD-slot nop after a branch is absorbed
--- into the branch's cost (MIPS-accurate: BD slot always runs), and is
--- SKIPPED when continuing down the fall-through path (otherwise we'd
--- double-count it).
---
--- Returns:
--- cycles_min - shortest path through the body (sum of token costs)
--- cycles_max - longest path through the body
--- branches - number of branches in the body
--- paths - number of distinct paths reached (terminated at
--- mac_yield or end-of-body)
--- has_loops - true iff a path re-entered a token it had visited
--- (warning; loop bodies aren't supported)
--- unknown_macros - list of unique macro names not in INSTRUCTION_LATENCY
--- cycles_full - sum of ALL token costs (the previous "best case"
--- value; included for backward-compat; double-counts
--- the BD-slot nop relative to cycles_min/max)
local function analyze_atom_paths(atom)
local tokens = tokenize_body(atom.body)
local labels = find_atom_labels(tokens)
local branches = find_branch_targets(tokens)
-- Pre-compute per-token cycle costs and identify terminators.
local n = #tokens
local costs = {}
local unknown_set = {}
for i, t in ipairs(tokens) do
local c, _, unknown = token_cycles(t.tok)
costs[i] = c
if unknown then
unknown_set[t.tok:match("^([%w_]+)") or "?"] = true
end
end
-- A token is a terminator if it's `mac_yield` or `mac_yield(...)`.
-- The yield transfers control; we don't count its cost (the next
-- atom's prologue absorbs it).
local function is_terminator(i)
local tok = tokens[i].tok
return tok == "mac_yield" or tok:match("^mac_yield%s*%(")
end
-- CFG successor function. Returns a list of next token indices for
-- the given position. Branch tokens produce 2 successors (fall-through
-- + taken); normal tokens produce 1 (next); terminators produce 0.
-- BD-slot absorption: a branch at i skips i+1 (the BD slot) in its
-- fall-through path; the BD slot's cost is added to the branch's
-- own cost instead (so it's counted once).
--
-- A token is a "branch" if its index is a KEY in the `branches`
-- map (regardless of whether the value is nil — a branch with
-- nil target means "literal offset, taken path is unknown").
-- We check key-presence via `branches[i] ~= nil` because
-- `branches[i]` returns nil for both "absent" AND "present with
-- nil value" — distinguishing them requires the key check.
local function is_branch(i)
local v = branches[i]
if v == nil then return false end
-- v is non-nil: either a string (atom_offset target) or false
-- (literal offset, no target). Both indicate a branch.
return true
end
local function successors(i)
local tok = tokens[i].tok
if is_terminator(i) then
return {}, i -- empty list; term = i signals "path ends here"
end
if is_branch(i) then
local label = branches[i] -- may be false for literal-offset branches
local succ = {}
-- Fall-through: skip the BD slot (i+1). Use i+2.
if i + 2 <= n then
succ[#succ + 1] = i + 2
end
-- Taken: only if the branch has a known atom_offset target.
if label then
local label_pos = labels[label]
if label_pos and label_pos + 1 <= n then
succ[#succ + 1] = label_pos + 1
end
end
-- For literal-offset branches (label == false), the taken
-- path would jump to a non-tracked address; conservatively
-- omit. Return (succ, nil) -- the second value is the
-- terminator marker (nil = not a terminator).
return succ, nil
end
-- Normal token: just the next one
if i + 1 <= n then
return { i + 1 }, nil
end
return {}, nil
end
-- DFS through all paths. Track the current cycle sum, a visited set
-- scoped to the current path (to detect loops), and a count of paths.
-- Cap recursion at MAX_PATHS to prevent runaway exploration on
-- pathological bodies.
local MAX_PATHS = 64
local cycles_min = math.huge
local cycles_max = -1
local path_count = 0
local has_loops = false
local function dfs(i, acc, visited)
if path_count >= MAX_PATHS then return end
if _G._DEBUG_DFS then
io.stderr:write(string.format("dfs(i=%d, acc=%d)\n", i, acc))
end
if visited[i] then
has_loops = true
if _G._DEBUG_DFS_LOOP then
io.stderr:write(string.format(" -> LOOP at i=%d (tok=%s) acc=%d\n",
i, tokens[i].tok, acc))
end
return
end
-- Add this token's cost. For a branch, ADD the BD-slot cost too
-- (and skip the BD slot in the successor list — already done in
-- `successors` above for fall-through; for taken path the BD
-- slot was at i+1 which is now skipped entirely).
local cost = costs[i]
if is_branch(i) and i + 1 <= n then
cost = cost + costs[i + 1]
end
local new_acc = acc + cost
local succ, term = successors(i)
if term then
-- Terminator: record the path's cycle sum. We do NOT add
-- the terminator token to `visited` -- a path ends here, so
-- a different path that ALSO reaches this terminator is a
-- legitimate new path (not a loop). If we marked it
-- visited, subsequent paths that reach the same terminator
-- would be incorrectly flagged as loops.
path_count = path_count + 1
if new_acc < cycles_min then cycles_min = new_acc end
if new_acc > cycles_max then cycles_max = new_acc end
return
end
visited[i] = true
for _, next_i in ipairs(succ) do
dfs(next_i, new_acc, visited)
end
visited[i] = nil
end
if n >= 1 then dfs(1, 0, {}) end
-- cycles_full: sum of every token's cost (the previous model; useful
-- for comparing against the path-aware min/max).
local cycles_full = 0
for i = 1, n do cycles_full = cycles_full + costs[i] end
-- If no paths were recorded (e.g. atom body is empty), cycles_min/max
-- default to 0 (atom costs nothing). cycles_full is 0 too in that case.
if cycles_min == math.huge then cycles_min = 0 end
if cycles_max == -1 then cycles_max = 0 end
local unknown_list = {}
for k in pairs(unknown_set) do unknown_list[#unknown_list + 1] = k end
table.sort(unknown_list)
-- branch_count: number of `branch_*(...)` tokens. (More useful than
-- `#branches` which is the size of the targets map and would be equal
-- to #branches anyway, but the rename is clearer.)
local branch_count = 0
for _ in pairs(branches) do branch_count = branch_count + 1 end
return {
cycles_min = cycles_min,
cycles_max = cycles_max,
cycles_full = cycles_full,
branches = branch_count,
paths = path_count,
has_loops = has_loops,
unknown_macros = unknown_list,
}
end
--- Backward-compat wrapper: returns total cycle count (the previous
--- "best case" value, which over-counts BD-slot nops) + unknown macro
--- list. New code should call `analyze_atom_paths(atom)` instead.
local function count_atom_cycles(atom)
local tokens = tokenize_body(atom.body)
local total = 0
local unknown_set = {}
for _, t in ipairs(tokens) do
local c, _, unknown = token_cycles(t.tok)
total = total + c
if unknown then
unknown_set[t.tok:match("^([%w_]+)") or "?"] = true
end
end
local unknown_list = {}
for k in pairs(unknown_set) do unknown_list[#unknown_list + 1] = k end
return total, unknown_list
end
--- Per-source check that emits one finding per unknown macro seen
--- (deduplicated across atoms so the warning section doesn't get
--- spammed with N copies of "macro X not in INSTRUCTION_LATENCY").
local function check_per_atom_cycle_budget(atoms, findings)
local unknown_seen = {}
for _, a in ipairs(atoms) do
local _, unknown_macros = count_atom_cycles(a)
for _, name in ipairs(unknown_macros) do
if not unknown_seen[name] then
unknown_seen[name] = a.line
findings[#findings + 1] = {
atom = a.name, line = a.line,
check = "per_atom_cycle_budget", kind = "warning",
msg = string.format("%s at line %d uses macro `%s` which is not in INSTRUCTION_LATENCY; cycle count will be +%d per call (best-case). Add an entry to duffle.M.INSTRUCTION_LATENCY.",
a.name, a.line, name, UNKNOWN_INSTRUCTION_CYCLES),
}
end
end
end
end
-- ════════════════════════════════════════════════════════════════════════════
-- Per-source validation
-- ════════════════════════════════════════════════════════════════════════════
@@ -898,9 +1196,25 @@ local function validate(ctx, src)
check_mac_yield_uniformity(atoms, findings)
check_abi_handoff(atoms, atom_infos, binds_index, findings)
check_gpu_portstore_shape(atoms, findings)
check_per_atom_cycle_budget(atoms, findings)
-- Phase 3 (per-atom cycle budget) hooks go here when reactivated.
-- emit per-atom cycle counts via INSTRUCTION_LATENCY table
-- Phase 3 cycle-budget output: attach per-path cycle data to each
-- atom. Best-case (no-stall) cycle count with BD-slot absorbed; the
-- `cycles_full` field is the legacy sum-of-all-tokens value (kept
-- for backward compat; over-counts BD-slot nops).
local cycles_by_atom = {}
for _, a in ipairs(atoms) do
local p = analyze_atom_paths(a)
a.paths = p
a.cycles = p.cycles_max -- default for any code that reads .cycles
a.cycles_min = p.cycles_min
a.cycles_max = p.cycles_max
a.cycles_full = p.cycles_full
a.branch_count = p.branches
a.unknown_macros = p.unknown_macros
a.has_loops = p.has_loops
cycles_by_atom[a.name] = p
end
local errors = {}
local warnings = {}
@@ -922,6 +1236,28 @@ local function validate(ctx, src)
msg = string.format("scanned: %d atom bodies; %d findings", #atoms, #findings),
}
-- Phase 3: cycle-budget summary line. Per-path min/max totals.
if #atoms > 0 then
local total_min = 0
local total_max = 0
local max_atom_cyc = 0
local max_atom_name = nil
for _, a in ipairs(atoms) do
local p = a.paths or {}
total_min = total_min + (p.cycles_min or 0)
total_max = total_max + (p.cycles_max or 0)
if (p.cycles_max or 0) > max_atom_cyc then
max_atom_cyc = p.cycles_max
max_atom_name = a.name
end
end
info[#info + 1] = {
line = 0,
msg = string.format("cycles: path-aware min=%d max=%d across %d atoms; worst atom=%s (%d); best-case, no stalls; BD-slot nops absorbed into branch costs",
total_min, total_max, #atoms, max_atom_name or "?", max_atom_cyc),
}
end
return {
atoms = atoms,
findings = findings,
@@ -1010,6 +1346,145 @@ local function emit_static_analysis_txt(ctx, src, result)
return out_path
end
-- (Old per-source emit function above kept for backward compat but no
-- longer called from M.run; replaced by `emit_module_static_analysis_txt`
-- which aggregates by directory. Kept because some test harnesses may
-- still call it directly.)
--- Per-directory emit. Aggregates atoms + findings across every source
--- in `dir_sources` and writes a single report to
--- `<out_root>/<dir_basename>.static_analysis.txt`. Called only when
--- at least one atom was found (the caller in M.run handles the skip).
local function emit_module_static_analysis_txt(ctx, dir, dir_sources, atoms, findings, errors, warnings, info)
-- Module basename = last component of `dir` ("code/duffle" -> "duffle").
local dir_basename = dir:match("([^/\\]+)$") or dir
local out_path = ctx.out_root .. "/" .. dir_basename .. ".static_analysis.txt"
if ctx.dry_run then return out_path end
ensure_dir(ctx.out_root)
local lines = {}
local function add(s) lines[#lines + 1] = s end
add("========================================================")
add("STATIC ANALYSIS PASS -- module " .. dir_basename)
add("========================================================")
add(string.format("Sources: %d", #dir_sources))
for _, s in ipairs(dir_sources) do
add(" " .. s.path)
end
add("")
-- Tally atoms by kind for the header summary
local n_atoms, n_bare, n_proc = 0, 0, 0
for _, a in ipairs(atoms) do
n_atoms = n_atoms + 1
if a.kind == "comp_bare" then n_bare = n_bare + 1
elseif a.kind == "comp_proc" then n_proc = n_proc + 1
end
end
local header_atoms = string.format("Atoms: %d", n_atoms)
if n_bare > 0 or n_proc > 0 then
header_atoms = header_atoms .. string.format(" (atoms: %d, comp_bare: %d, comp_proc: %d)",
n_atoms - n_bare - n_proc, n_bare, n_proc)
end
add(string.format("%s Findings: %d Errors: %d Warnings: %d",
header_atoms, #findings, #errors, #warnings))
add("")
-- Group findings by atom (with source prefix when multi-source module)
local multi_source = #dir_sources > 1
local by_atom = {}
for _, f in ipairs(findings) do
by_atom[f.atom] = by_atom[f.atom] or {}
by_atom[f.atom][#by_atom[f.atom] + 1] = f
end
if next(by_atom) == nil then
add(" (no findings -- every atom passed all checks)")
else
add("── Findings by atom ─────────────────────────────────────")
for _, a in ipairs(atoms) do
local fs = by_atom[a.name]
if fs then
local label = a.name
if multi_source and a.source_path then
label = string.format("%s (%s)", a.name, a.source_path:match("([^/\\]+)$") or a.source_path)
end
add(string.format(" %s line %d", label, a.line))
for _, f in ipairs(fs) do
add(string.format(" [%s] %s", f.check, f.msg))
end
end
end
end
add("")
add("── Errors ──────────────────────────────────────────────")
if #errors == 0 then add(" (none)") end
for _, e in ipairs(errors) do
add(string.format(" X line %d %s", e.line, e.msg))
end
add("")
add("── Warnings ────────────────────────────────────────────")
if #warnings == 0 then add(" (none)") end
for _, w in ipairs(warnings) do
add(string.format(" ! line %d %s", w.line, w.msg))
end
-- Per-atom cycle counts (Phase 3 path-aware). For each atom:
-- min = shortest path through the body (earliest exit)
-- max = longest path through the body (full fall-through)
-- br = number of branch instructions
-- paths = number of distinct paths reached
-- Both min and max are best-case (no stalls); BD-slot nops are
-- absorbed into branch costs (MIPS semantics). The previous "best
-- case" model counted every token separately, which double-counted
-- BD-slot nops; the path-aware model is the MIPS-accurate value.
add("")
add("── Per-atom cycle counts (path-aware, best case, no stalls) ─")
if #atoms == 0 then
add(" (no atoms)")
else
-- Sort atoms by max cycles descending for quick scanning.
local sorted = {}
for _, a in ipairs(atoms) do sorted[#sorted + 1] = a end
table.sort(sorted, function(x, y) return (x.cycles_max or 0) > (y.cycles_max or 0) end)
for _, a in ipairs(sorted) do
local p = a.paths or {}
local br_count = p.branches or 0
local path_count = p.paths or 0
local loops_tag = p.has_loops and " [loop!]" or ""
local unknown_tag = ""
if a.unknown_macros and #a.unknown_macros > 0 then
unknown_tag = string.format(" [unknown: %s]",
table.concat(a.unknown_macros, ", "))
end
local name_label = a.name
if multi_source and a.source_path then
name_label = string.format("%s (%s)", a.name, a.source_path:match("([^/\\]+)$") or a.source_path)
end
if br_count > 0 then
add(string.format(" %-44s min=%4d max=%4d br=%d paths=%d (line %d)%s%s",
name_label, p.cycles_min or 0, p.cycles_max or 0, br_count, path_count,
a.line, loops_tag, unknown_tag))
else
add(string.format(" %-44s %4d cycles (line %d, no branches)%s%s",
name_label, p.cycles_min or 0, a.line, loops_tag, unknown_tag))
end
end
end
add("")
add("── Info ────────────────────────────────────────────────")
for _, i_ in ipairs(info) do
add(string.format(" %s", i_.msg))
end
write_file(out_path, table.concat(lines, "\n") .. "\n")
return out_path
end
-- ════════════════════════════════════════════════════════════════════════════
-- M.run — orchestrator entry
-- ════════════════════════════════════════════════════════════════════════════
@@ -1025,17 +1500,71 @@ function M.run(ctx)
local errors = {}
local warnings = {}
-- Phase 3.7+: aggregate per-DIRECTORY (per-module). One
-- static_analysis.txt per source-directory, emitted only if the
-- directory contains at least one atom. Empty-source directories
-- (e.g. duffle headers with no atoms) produce no report.
--
-- Group sources by `src.dir`. The first component of `dir` is the
-- module name (e.g. "code/duffle" -> "duffle", "code/gte_hello" ->
-- "gte_hello"). Output path is `<out_root>/<module_basename>.static_analysis.txt`.
local by_dir = {}
for _, src in ipairs(ctx.sources) do
local result = validate(ctx, src)
local out_path = emit_static_analysis_txt(ctx, src, result)
if out_path then
table.insert(outputs, { static_analysis_txt = out_path })
by_dir[src.dir] = by_dir[src.dir] or {}
table.insert(by_dir[src.dir], src)
end
for dir, dir_sources in pairs(by_dir) do
-- Run validate() against every source in this directory; accumulate
-- atoms / findings / errors / warnings. The validate() function
-- does its own per-source analysis (Binds indexing, atom
-- discovery, all 5 checks) and attaches path-aware cycle data to
-- each atom it finds.
local all_atoms = {}
local all_findings = {}
local dir_errors = {}
local dir_warnings = {}
local all_info = {}
for _, src in ipairs(dir_sources) do
local result = validate(ctx, src)
-- Tag each atom with its source so the render step can prefix
-- the atom line with "<filename>:" when atoms from multiple
-- sources live in the same module (e.g. lottes_tape.h +
-- atom_dsl.h both declaring atoms).
for _, a in ipairs(result.atoms) do
a.source_path = src.path
all_atoms[#all_atoms + 1] = a
end
for _, f in ipairs(result.findings) do
all_findings[#all_findings + 1] = f
end
for _, e in ipairs(result.errors) do
dir_errors[#dir_errors + 1] = e
end
for _, w in ipairs(result.warnings) do
dir_warnings[#dir_warnings + 1] = w
end
for _, i_ in ipairs(result.info) do
all_info[#all_info + 1] = i_
end
end
for _, e in ipairs(result.errors) do
errors[#errors + 1] = { line = e.line, msg = e.msg }
end
for _, w in ipairs(result.warnings) do
warnings[#warnings + 1] = { line = w.line, msg = w.msg }
-- Skip directories with zero atoms. The previous behavior emitted
-- a "<no atoms>" report per source; the new behavior emits nothing
-- at all (a directory with only headers / no MipsAtom_ is
-- "nothing to report").
if #all_atoms == 0 then
-- Still aggregate errors/warnings so orchestrator sees them,
-- but don't write a file.
for _, e in ipairs(dir_errors) do errors[#errors + 1] = e end
for _, w in ipairs(dir_warnings) do warnings[#warnings + 1] = w end
else
local out_path = emit_module_static_analysis_txt(ctx, dir, dir_sources, all_atoms, all_findings, dir_errors, dir_warnings, all_info)
if out_path then
table.insert(outputs, { static_analysis_txt = out_path })
end
for _, e in ipairs(dir_errors) do errors[#errors + 1] = e end
for _, w in ipairs(dir_warnings) do warnings[#warnings + 1] = w end
end
end