ed/pikuma_ps1
1
0
Fork 0
mirror of https://github.com/Ed94/pikuma_ps1.git synced 2026-08-03 22:28:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

review pass on lua scripts related to tape atom metaprogram

Browse Source 
script running is slow need to fix.
This commit is contained in:
ed
2026-07-10 21:15:21 -04:00
parent a226b45d18
commit e9f26f89b8
11 changed files with 3257 additions and 1841 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/passes/annotation.lua
+468 -255
View File
@@ -1,32 +1,33 @@
-- passes/annotation.lua
--
-- Validate atom annotation DSL usage in source files. Reads:
-- - atom_annot / atom_init / atom_setup / atom_commit / atom_bind
-- / atom_terminate / atom_label / atom_offset / TAPE_WORDS
-- - atom_resource / atom_region / atom_group / atom_cadence / atom_async
-- - Binds_* struct declarations
-- Writes:
-- - <ctx.out_root>/<basename>.errors.h (with #error directives on findings)
-- - <ctx.out_root>/<basename>.annotations.txt (human-readable summary)
-- Ported from scripts/tape_atom_annotation_pass.lua:78-545 + 1081-1407
-- (validation only — NOT rendering, which goes to passes/report.lua).
--
-- Coding standard: tabs (1/level), EmmyLua annotations, no regex.
--- passes/annotation.lua — Atom-annotation DSL validator.
---
--- Validates `MipsAtom_(name) atom_info(atom_bind(Binds_X),
--- atom_reads(...), atom_writes(...)) { ... }` declarations in source files.
--- Also reads:
--- - `Binds_*` struct declarations (`typedef Struct_(Binds_X) { ... };`)
--- - `TAPE_WORDS(mac_X, N)` pragma directives (`#pragma` + `_Pragma`)
---
--- Writes:
--- - `<ctx.out_root>/<dir_basename>.errors.h` — one per module, with
--- `#error` directives on findings (the C compile will surface the
--- error)
--- - The annotations.txt report is rendered by `passes/report.lua`
--- from the per-module results stashed in `ctx.flags._annot_results`
---
--- **Ported from** `scripts/tape_atom_annotation_pass.lua:78-545 +
--- 1081-1407` (validation only — NOT rendering, which goes to
--- `passes/report.lua`).
---
--- **Conventions**: tabs (1/level), EmmyLua annotations, no regex,
--- Lua 5.3 compatible. See
--- `C:\projects\Pikuma\ps1-ai\conductor\code_styleguides\lua.md`.
-- ════════════════════════════════════════════════════════════════════════════
-- Module-scope requires + package.path setup
-- ════════════════════════════════════════════════════════════════════════════
local script_path = arg and arg[0] or "?"
local last_sep = 0
for i = 1, #script_path do
local c = script_path:sub(i, i)
if c == "/" or c == "\\" then last_sep = i end
end
local script_dir = last_sep == 0 and "./" or script_path:sub(1, last_sep)
package.path = script_dir .. "../?.lua;" .. script_dir .. "../?/init.lua;" .. script_dir .. "?.lua;" .. package.path
local duffle = require("duffle")
-- Bootstrap: same as entry scripts. See `ps1_meta.lua` for the rationale.
dofile((arg[0]:match("(.*[/\\])") or "./") .. "duffle_paths.lua")
local duffle = require("duffle")
local is_space = duffle.is_space
local is_alpha = duffle.is_alpha
local is_alnum = duffle.is_alnum
@@ -52,50 +53,166 @@ local TAPE_ATOM_MACROS = duffle.TAPE_ATOM_MACROS
local function is_wave_context_reg(n) return WAVE_CONTEXT_REGS[n] ~= nil end
-- ════════════════════════════════════════════════════════════════════════════
-- Constants
-- ════════════════════════════════════════════════════════════════════════════
-- Atom declaration + annotation identifiers.
local ATOM_DECL = "MipsAtom_"
local ATOM_INFO = "atom_info"
local STRUCT_TYPE = "Struct_"
local PRAGMA_IDENT = "pragma"
local PRAGMA_OPERATOR = "_Pragma"
-- Struct-name prefix + byte size of U4 fields.
local BINDS_PREFIX = "Binds_"
local BINDS_PREFIX_LEN = 6 -- = #BINDS_PREFIX
local U4_TYPE = "U4"
local U4_BYTES = 4 -- sizeof(U4)
local BINDS_FIELD_PREFIX = "R_" -- wave-context register name prefix
-- TAPE_WORDS pragma keys (the third token after #pragma).
local WORDS_KEY = "words"
local WORDS_KEY_PREFIX = "words=" -- the per-macro `words=N` form
local WORDS_KEY_PREFIX_LEN = 6 -- = #WORDS_KEY_PREFIX
local TAPE_ATOM_WORDS_KEY = "tape_atom words" -- the _Pragma form
-- ASCII byte values used in tokenization.
local BYTE_NEWLINE = 10
local BYTE_SPACE = 32
local BYTE_DQUOTE = 34
local BYTE_EQUALS = 61
local BYTE_OPEN_PAREN = 40
local BYTE_OPEN_BRACE = 123
local BYTE_OPEN_BRACK = 91
local BYTE_CLOSE_PAREN = 41
local BYTE_CLOSE_BRACE = 125
local BYTE_CLOSE_BRACK = 93
local BYTE_COMMA = 44
-- ════════════════════════════════════════════════════════════════════════════
-- Type declarations
-- ════════════════════════════════════════════════════════════════════════════
--- @class SourceFile
--- @field path string -- absolute path to the source file
--- @field text string -- the full source text
--- @field dir string -- the directory containing the source
--- @field basename string -- filename without extension
--- @class PassCtx
--- @field sources SourceFile[]
--- @field metadata_path string
--- @field shared table
--- @field shared.word_counts table<string, integer>
--- @field out_root string
--- @field project_root string
--- @field upstream table<string, table>
--- @field flags table
--- @field flags._annot_results table[] -- stashed by annotation pass; consumed by report.lua
--- @field dry_run boolean
--- @field verbose boolean
--- @class PassResult
--- @field outputs table[]
--- @field errors table[]
--- @field warnings table[]
--- @class Atom
--- @field line integer -- source line of the MipsAtom_ declaration
--- @field name string -- atom name (e.g. "cube_g4_face")
--- @class AtomAnnotation
--- @field line integer -- source line of the atom_info call
--- @field macro string -- the macro name (always "atom_info" in the new shape)
--- @field name string -- the atom name
--- @field kind string -- always "info"
--- @field binds string|nil -- Binds_X name if any
--- @field reads string[] -- R_* names (read targets)
--- @field writes string[] -- R_* names (write targets)
--- @field error string|nil -- error message if annotation was malformed
--- @field errors string[] -- nested errors from per-arg validation
--- @class BindsField
--- @field name string -- field name
--- @field offset integer -- byte offset within the Binds_X struct
--- @class BindsStruct
--- @field name string -- struct name (e.g. "Binds_Floor")
--- @field line integer -- source line of the typedef
--- @field bytes integer -- total byte size
--- @field fields BindsField[] -- the field list
--- @class MacroEntry
--- @field name string -- macro name (e.g. "mac_format_f3_color")
--- @field line integer -- source line of the TAPE_WORDS pragma
--- @field words integer -- declared word count
--- @class Finding
--- @field line integer -- source line (or 0 for pass-level)
--- @field msg string -- finding message
--- @class AnnotatedResult
--- @field atoms Atom[]
--- @field annots AtomAnnotation[]
--- @field macros MacroEntry[]
--- @field binds BindsStruct[]
--- @field errors Finding[]
--- @field warnings Finding[]
--- @field info Finding[]
--- @field pragmas table -- reserved (currently always nil; legacy compat)
--- ════════════════════════════════════════════════════════════════════════════
-- Hand-rolled split helpers (no regex patterns used)
-- ════════════════════════════════════════════════════════════════════════════
--- Split a string at top-level commas. Used inside TAPE_ATOM_* macro
--- bodies where nested parens/braces/brackets are possible.
--- @param s string
--- @return string[]
local function split_csv_top(s)
local tokens = {}
local i, start = 1, 1
local depth = 0
while i <= #s do
local c = s:sub(i, i)
if c == "(" or c == "{" or c == "[" then
local tokens = {}
local pos = 1
local chunk_a = 1
local depth = 0
local str_len = #s
while pos <= str_len do
local ch = s:byte(pos)
if ch == BYTE_OPEN_PAREN or ch == BYTE_OPEN_BRACE or ch == BYTE_OPEN_BRACK then
depth = depth + 1
i = i + 1
elseif c == ")" or c == "}" or c == "]" then
pos = pos + 1
elseif ch == BYTE_CLOSE_PAREN or ch == BYTE_CLOSE_BRACE or ch == BYTE_CLOSE_BRACK then
depth = depth - 1
i = i + 1
elseif c == "," and depth == 0 then
tokens[#tokens + 1] = s:sub(start, i - 1)
i = i + 1
start = i
pos = pos + 1
elseif ch == BYTE_COMMA and depth == 0 then
tokens[#tokens + 1] = s:sub(chunk_a, pos - 1)
pos = pos + 1
chunk_a = pos
else
i = i + 1
pos = pos + 1
end
end
local last = s:sub(start)
local last = s:sub(chunk_a)
if trim(last) ~= "" then tokens[#tokens + 1] = last end
return tokens
end
--- Split a string into whitespace-separated tokens.
--- Hand-rolled (no regex patterns).
--- @param s string
--- @return string[]
local function split_ws(s)
local tokens = {}
local i, n = 1, 1
local len = #s
while i <= len do
local pos = 1
local n = 1
local len = #s
while pos <= len do
-- Skip whitespace.
while i <= len and is_space(s:sub(i, i)) do i = i + 1 end
if i > len then break end
local start = i
while pos <= len and is_space(s:sub(pos, pos)) do pos = pos + 1 end
if pos > len then break end
local chunk_a = pos
-- Take non-whitespace run.
while i <= len and not is_space(s:sub(i, i)) do i = i + 1 end
tokens[n] = s:sub(start, i - 1)
while pos <= len and not is_space(s:sub(pos, pos)) do pos = pos + 1 end
tokens[n] = s:sub(chunk_a, pos - 1)
n = n + 1
end
return tokens
@@ -196,75 +313,120 @@ end
-- Parse TAPE_WORDS(mac_X, N) pragma directives
-- ════════════════════════════════════════════════════════════════════════════
--- Find every TAPE_WORDS(mac_X, N) pragma in source.
--- Skip preprocessor directives (lines starting with `#`).
--- Returns the position past the newline at the end of the line.
--- @param source string
--- @param pos integer
--- @return integer
local function skip_preprocessor_line(source, pos)
local str_len = #source
local scan = pos
while scan <= str_len and source:byte(scan) ~= BYTE_NEWLINE do
scan = scan + 1
end
return scan + 1
end
--- Parse `_Pragma("mac_X tape_atom words=N")` (operator form).
--- @param source string
--- @param ident_pos integer -- position of the `_Pragma` ident
--- @param after_ident integer -- position just past the ident
--- @return MacroEntry|nil, integer -- (entry or nil, new source position)
local function parse_pragma_operator(source, ident_pos, after_ident)
local open_paren = skip_ws_and_cmt(source, after_ident)
if source:byte(open_paren) ~= BYTE_OPEN_PAREN then
return nil, open_paren + 1
end
local str, str_end = read_parens(source, open_paren)
str = trim(str)
if str:sub(1, 1) ~= '"' or str:sub(-1) ~= '"' then
return nil, str_end
end
local inner = str:sub(2, -2)
local space = find_byte(inner, BYTE_SPACE, 1)
if not space then return nil, str_end end
local name = inner:sub(1, space - 1)
local rest = inner:sub(space + 1)
local eq = find_byte(rest, BYTE_EQUALS, 1)
if not eq then return nil, str_end end
local key = trim(rest:sub(1, eq - 1))
local val = trim(rest:sub(eq + 1))
if key ~= TAPE_ATOM_WORDS_KEY and key ~= WORDS_KEY then return nil, str_end end
return {
line = source:sub(1, ident_pos) and 0 or 0, -- see line_of below
name = name,
words = tonumber(val) or 0,
}, str_end
end
--- Parse `#pragma mac_X tape_atom words=N` (directive form).
--- @param source string
--- @param ident_pos integer -- position of the `pragma` ident
--- @param after_ident integer -- position just past the ident
--- @return MacroEntry|nil, integer -- (entry or nil, new source position)
local function parse_pragma_directive(source, ident_pos, after_ident)
local str_len = #source
local rest_start = skip_ws_and_cmt(source, after_ident)
local eol = rest_start
while eol <= str_len and source:byte(eol) ~= BYTE_NEWLINE do
eol = eol + 1
end
local line_text = trim(source:sub(rest_start, eol - 1))
local tokens = split_ws(line_text)
local entry
if #tokens >= 3 and tokens[2] == "tape_atom" and tokens[3]:sub(1, WORDS_KEY_PREFIX_LEN) == WORDS_KEY_PREFIX then
entry = {
name = tokens[1],
words = tonumber(tokens[3]:sub(WORDS_KEY_PREFIX_LEN + 1)) or 0,
}
elseif #tokens >= 2 and tokens[2]:sub(1, WORDS_KEY_PREFIX_LEN) == WORDS_KEY_PREFIX then
entry = {
name = tokens[1],
words = tonumber(tokens[2]:sub(WORDS_KEY_PREFIX_LEN + 1)) or 0,
}
end
if entry then
local line_of = duffle.LineIndex(source)
entry.line = line_of(ident_pos)
end
return entry, eol
end
--- Find every `TAPE_WORDS(mac_X, N)` pragma in source.
--- Accepts both forms:
--- _Pragma("mac_X tape_atom words=N") (operator form)
--- #pragma mac_X tape_atom words=N (directive form)
--- `_Pragma("mac_X tape_atom words=N")` (operator form)
--- `#pragma mac_X tape_atom words=N` (directive form)
--- @param source string
--- @return MacroEntry[]
local function find_macro_word_annotations(source)
local line_of = duffle.LineIndex(source)
local out = {}
local len = #source
local i = 1
while i <= len do
i = skip_ws_and_cmt(source, i); if i > len then break end
local out = {}
local pos = 1
local str_len = #source
while pos <= str_len do
pos = skip_ws_and_cmt(source, pos)
if pos > str_len then break end
-- Skip preprocessor directives (lines starting with #).
if source:sub(i, i) == "#" then
local j = i
while j <= len and source:sub(j, j) ~= "\n" do j = j + 1 end
i = j + 1
if source:byte(pos) == 35 then -- '#'
pos = skip_preprocessor_line(source, pos)
else
local ident, after = read_ident(source, i)
local ident, after_ident = read_ident(source, pos)
if not ident then
i = i + 1
elseif ident == "_Pragma" then
local open = skip_ws_and_cmt(source, after)
if source:sub(open, open) == "(" then
local str, str_end = read_parens(source, open)
str = trim(str)
if str:sub(1, 1) == '"' and str:sub(-1) == '"' then
local inner = str:sub(2, -2)
local space = find_byte(inner, " ", 1)
if space then
local name = inner:sub(1, space - 1)
local rest = inner:sub(space + 1)
local eq = find_byte(rest, "=", 1)
if eq then
local key = trim(rest:sub(1, eq - 1))
local val = trim(rest:sub(eq + 1))
if key == "tape_atom words" or key == "words" then
local n = tonumber(val) or 0
out[#out + 1] = {
line = line_of(i),
name = name,
words = n,
}
end
end
end
end
i = str_end
else
i = open + 1
pos = pos + 1
elseif ident == PRAGMA_OPERATOR then
local entry, new_pos = parse_pragma_operator(source, pos, after_ident)
if entry then
local line_of = duffle.LineIndex(source)
entry.line = line_of(pos)
out[#out + 1] = entry
end
elseif ident == "pragma" then
-- Directive form: `#pragma mac_X tape_atom words=N`
local rest_start = skip_ws_and_cmt(source, after)
local j = rest_start
while j <= len and source:sub(j, j) ~= "\n" do j = j + 1 end
local line_text = trim(source:sub(rest_start, j - 1))
local tokens = split_ws(line_text)
if #tokens >= 3 and tokens[2] == "tape_atom" and tokens[3]:sub(1, 6) == "words=" then
local name = tokens[1]
local n = tonumber(tokens[3]:sub(7)) or 0
out[#out + 1] = { line = line_of(i), name = name, words = n }
elseif #tokens >= 2 and tokens[2]:sub(1, 6) == "words=" then
local name = tokens[1]
local n = tonumber(tokens[2]:sub(7)) or 0
out[#out + 1] = { line = line_of(i), name = name, words = n }
end
i = j
pos = new_pos
elseif ident == PRAGMA_IDENT then
local entry, new_pos = parse_pragma_directive(source, pos, after_ident)
if entry then out[#out + 1] = entry end
pos = new_pos
else
i = after
pos = after_ident
end
end
end
@@ -275,76 +437,105 @@ end
-- Parse `typedef Struct_(Binds_X) { ... };` declarations
-- ════════════════════════════════════════════════════════════════════════════
--- Find every Binds_* struct declaration.
--- Returns a list of {line, name, fields = {{name, byte_offset}, ...}}.
--- Walk a `Binds_X` body string and extract U4 fields (name + byte offset).
--- Only U4 fields are tracked (Binds_* are always word arrays in this
--- codebase -- pointers stored as U4, indices as U4, etc.).
--- @param body string -- the brace-delimited body (without the braces)
--- @return BindsField[] -- field list
--- @return integer -- total byte size
local function parse_binds_body(body)
local fields = {}
local byte_off = 0
local pos = 1
local body_len = #body
while pos <= body_len do
pos = skip_ws_and_cmt(body, pos)
if pos > body_len then break end
local type_ident, type_after = read_ident(body, pos)
if not type_ident then
pos = pos + 1
elseif type_ident == U4_TYPE then
local field_after = skip_ws_and_cmt(body, type_after)
local fid, fafter = read_ident(body, field_after)
if fid then
fields[#fields + 1] = { name = fid, offset = byte_off }
byte_off = byte_off + U4_BYTES
end
pos = fafter or (type_after + 1)
else
pos = type_after + 1
end
end
return fields, byte_off
end
--- Try to parse a `typedef Struct_(Binds_X) { ... };` declaration.
--- Returns the parsed BindsStruct (if the form matched) and the new
--- source position. If the form didn't match, returns nil + a position
--- to continue scanning from.
--- @param source string
--- @param ident_pos integer -- position of the `typedef` ident start
--- @param after_typedef integer -- position just past `typedef`
--- @param line_of fun(pos: integer): integer
--- @return BindsStruct|nil, integer
local function parse_typedef_binds(source, ident_pos, after_typedef, line_of)
local after_type = skip_ws_and_cmt(source, after_typedef)
local type_ident, after_type_ident = read_ident(source, after_type)
if type_ident ~= STRUCT_TYPE then
return nil, after_type_ident or (after_type + 1)
end
local open_paren = skip_ws_and_cmt(source, after_type_ident)
if source:byte(open_paren) ~= BYTE_OPEN_PAREN then
return nil, open_paren + 1
end
local inner, after_paren = read_parens(source, open_paren)
local name = trim(inner)
local brace = scan_to_char(source, "{", after_paren)
if not brace then return nil, open_paren + 1 end
local body, after_brace = read_braces(source, brace)
local fields, bytes = parse_binds_body(body)
-- Only emit Binds_* structs (other Struct_ typedefs are ignored).
if name:sub(1, BINDS_PREFIX_LEN) ~= BINDS_PREFIX then
return nil, after_brace
end
return {
line = line_of(ident_pos),
name = name,
fields = fields,
bytes = bytes,
}, after_brace
end
--- Find every `Binds_*` struct declaration.
--- @param source string
--- @return BindsStruct[]
local function find_binds_structs(source)
local line_of = duffle.LineIndex(source)
local out = {}
local len = #source
local i = 1
while i <= len do
i = skip_ws_and_cmt(source, i); if i > len then break end
if source:sub(i, i) == "#" then
local j = i
while j <= len and source:sub(j, j) ~= "\n" do j = j + 1 end
i = j + 1
local out = {}
local pos = 1
local str_len = #source
while pos <= str_len do
pos = skip_ws_and_cmt(source, pos)
if pos > str_len then break end
if source:byte(pos) == 35 then -- '#'
pos = skip_preprocessor_line(source, pos)
else
local ident, after = read_ident(source, i)
local ident, after_ident = read_ident(source, pos)
if not ident then
i = i + 1
pos = pos + 1
elseif ident == "typedef" then
local j = skip_ws_and_cmt(source, after)
local id2, after2 = read_ident(source, j)
if id2 ~= "Struct_" then
i = after2 or (j + 1)
elseif id2 == "Struct_" then
local open = skip_ws_and_cmt(source, after2)
if source:sub(open, open) == "(" then
local inner, after_paren = read_parens(source, open)
local name = trim(inner)
local brace = scan_to_char(source, "{", after_paren)
if brace then
local body, after_brace = read_braces(source, brace)
local fields = {}
local byte_off = 0
local k = 1
while k <= #body do
k = skip_ws_and_cmt(body, k); if k > #body then break end
local tid, tafter = read_ident(body, k)
if not tid then
k = k + 1
elseif tid == "U4" then
local fid, fafter = read_ident(body, skip_ws_and_cmt(body, tafter))
if fid then
fields[#fields + 1] = { name = fid, offset = byte_off }
byte_off = byte_off + 4
end
k = fafter or tafter + 1
else
k = tafter + 1
end
end
-- Only emit Binds_* structs.
if name:sub(1, 6) == "Binds_" then
out[#out + 1] = {
line = line_of(i),
name = name,
fields = fields,
bytes = byte_off,
}
end
i = after_brace
else
i = open + 1
end
else
i = open + 1
end
else
i = after2 or (j + 1)
end
local binds_struct, new_pos = parse_typedef_binds(source, pos, after_ident, line_of)
if binds_struct then out[#out + 1] = binds_struct end
pos = new_pos
else
i = after
pos = after_ident
end
end
end
@@ -355,40 +546,58 @@ end
-- Find every MipsAtom_(name) { ... } declaration in source
-- ════════════════════════════════════════════════════════════════════════════
--- Read the next identifier token from `s` starting at `pos`, where the
--- identifier is a contiguous run of `[a-zA-Z0-9_]` characters (no
--- underscore-starting alpha-only constraint). Returns the ident + the
--- position just past it, or nil + pos if no identifier starts there.
--- @param s string
--- @param pos integer
--- @return string|nil, integer
local function read_alnum_ident(s, pos)
local str_len = #s
while pos <= str_len and is_space(s:sub(pos, pos)) do pos = pos + 1 end
local start = pos
while pos <= str_len and is_alnum(s:sub(pos, pos)) do pos = pos + 1 end
if pos == start then return nil, pos end
return s:sub(start, pos - 1), pos
end
--- Find every `MipsAtom_(name)` declaration in source. (Just the name +
--- source line; the body is parsed separately by `parse_mips_atom`.)
--- @param source string
--- @return Atom[]
local function find_atom_names(source)
local line_of = duffle.LineIndex(source)
local out = {}
local len = #source
local i = 1
while i <= len do
i = skip_ws_and_cmt(source, i); if i > len then break end
local ident, after = read_ident(source, i)
local out = {}
local pos = 1
local str_len = #source
while pos <= str_len do
pos = skip_ws_and_cmt(source, pos)
if pos > str_len then break end
local ident, after_ident = read_ident(source, pos)
if not ident then
i = i + 1
elseif ident == "MipsAtom_" then
local open = skip_ws_and_cmt(source, after)
if source:sub(open, open) == "(" then
local inner, after_paren = read_parens(source, open)
local a = 1
while a <= #inner and is_space(inner:sub(a, a)) do a = a + 1 end
local b = a
while b <= #inner and is_alnum(inner:sub(b, b)) do b = b + 1 end
local name = inner:sub(a, b - 1)
if name ~= "" then
out[#out + 1] = { line = line_of(i), name = name }
pos = pos + 1
elseif ident ~= ATOM_DECL then
pos = after_ident
else
local open_paren = skip_ws_and_cmt(source, after_ident)
if source:byte(open_paren) ~= BYTE_OPEN_PAREN then
pos = open_paren + 1
else
local inner, after_paren = read_parens(source, open_paren)
local name, _ = read_alnum_ident(inner, 1)
if name and name ~= "" then
out[#out + 1] = { line = line_of(pos), name = name }
end
local brace = scan_to_char(source, "{", after_paren)
if brace then
local _, after_brace = read_braces(source, brace)
i = after_brace
pos = after_brace
else
i = open + 1
pos = open_paren + 1
end
else
i = open + 1
end
else
i = after
end
end
return out
@@ -451,72 +660,76 @@ local function new_annot_entry(line, ident, name, kind)
}
end
--- Find every MipsAtom_(name) declaration in source, then look for an
--- immediately-following atom_info(...) call. If present, parse its
--- sub-calls into a normalized annotation entry linked to the MipsAtom_
--- name. If no atom_info follows, emit NO annotation entry (atoms
--- without annotations are valid in the new minimal shape).
--- Try to parse an `atom_info(...)` call right after the `MipsAtom_(name)`
--- parens. Returns the annotation entry (if present) and the new source
--- position past the atom_info call. Returns nil if no atom_info follows.
--- @param source string
--- @param atom_name string
--- @param after_mipsatom_paren integer -- position past the MipsAtom_(...) close paren
--- @param line_of fun(pos: integer): integer
--- @return AtomAnnotation|nil, integer -- (entry or nil, new source position)
local function parse_atom_info_call(source, atom_name, after_mipsatom_paren, line_of)
local lookahead = skip_ws_and_cmt(source, after_mipsatom_paren)
local look_ident, look_after = read_ident(source, lookahead)
if look_ident ~= ATOM_INFO then return nil, after_mipsatom_paren end
local info_open = skip_ws_and_cmt(source, look_after)
if source:byte(info_open) ~= BYTE_OPEN_PAREN then
return nil, info_open + 1
end
local info_inner, info_after = read_parens(source, info_open)
local args = parse_atom_annot_args(info_inner)
local entry = new_annot_entry(line_of(lookahead), ATOM_INFO, atom_name, "info")
ANNOT_ARG_HANDLERS.info(entry, args)
return entry, info_after
end
--- Find every `MipsAtom_(name) atom_info(...) { ... };` annotation in source.
--- Returns a list of annotation entries. Atoms without a following
--- `atom_info(...)` call produce NO entry (atoms without annotations are
--- valid in the new minimal shape).
--- @param source string
--- @return AtomAnnotation[]
local function find_atom_annotations(source)
local line_of = duffle.LineIndex(source)
local annots = {}
local len = #source
local i = 1
while i <= len do
i = skip_ws_and_cmt(source, i); if i > len then break end
local annots = {}
local pos = 1
local str_len = #source
while pos <= str_len do
pos = skip_ws_and_cmt(source, pos)
if pos > str_len then break end
-- Skip preprocessor directives (lines starting with #).
if source:sub(i, i) == "#" then
local j = i
while j <= len and source:sub(j, j) ~= "\n" do j = j + 1 end
i = j + 1
goto continue
end
local ident, after = read_ident(source, i)
if not ident then
i = i + 1
elseif ident == "MipsAtom_" then
local open = skip_ws_and_cmt(source, after)
if source:sub(open, open) ~= "(" then
i = open + 1
goto continue
end
local inner, after_paren = read_parens(source, open)
local a = 1
while a <= #inner and is_space(inner:sub(a, a)) do a = a + 1 end
local b = a
while b <= #inner and is_alnum(inner:sub(b, b)) do b = b + 1 end
local name = inner:sub(a, b - 1)
-- Look for atom_info(...) right after MipsAtom_(name).
local lookahead = skip_ws_and_cmt(source, after_paren)
local look_ident, look_after = read_ident(source, lookahead)
if look_ident == "atom_info" then
local info_open = skip_ws_and_cmt(source, look_after)
if source:sub(info_open, info_open) == "(" then
local info_inner, info_after = read_parens(source, info_open)
local args = parse_atom_annot_args(info_inner)
local entry = new_annot_entry(line_of(lookahead), "atom_info", name, "info")
ANNOT_ARG_HANDLERS.info(entry, args)
annots[#annots + 1] = entry
i = info_after
if source:byte(pos) == 35 then -- '#'
pos = skip_preprocessor_line(source, pos)
else
local ident, after_ident = read_ident(source, pos)
if not ident then
pos = pos + 1
elseif ident == ATOM_DECL then
local open_paren = skip_ws_and_cmt(source, after_ident)
if source:byte(open_paren) ~= BYTE_OPEN_PAREN then
pos = open_paren + 1
else
i = info_open + 1
local inner, after_paren = read_parens(source, open_paren)
local name, _ = read_alnum_ident(inner, 1)
local entry, new_pos = parse_atom_info_call(source, name, after_paren, line_of)
if entry then annots[#annots + 1] = entry end
pos = new_pos
-- Skip past the body { ... } if present.
local brace = scan_to_char(source, "{", pos)
if brace then
local _, after_brace = read_braces(source, brace)
pos = after_brace
end
end
else
-- No atom_info follows this MipsAtom_. Valid in new shape.
i = after_paren
pos = after_ident
end
-- Skip past the body { ... } if present.
local brace = scan_to_char(source, "{", i)
if brace then
local _, after_brace = read_braces(source, brace)
i = after_brace
end
else
i = after
end
::continue::
end
return annots
end
scripts/passes/components.lua
+598 -420
View File
File diff suppressed because it is too large Load Diff
scripts/passes/duffle_paths.lua
+75
View File
@@ -0,0 +1,75 @@
--- duffle_paths.lua — Single-line bootstrap helper for the tape-atom
--- Lua scripts.
---
--- Each entry script (ps1_meta.lua, word_count_eval.lua, and the 5
--- passes/*.lua files) starts with:
---
--- ```lua
--- local duffle = dofile((arg[0]:match("(.*[/\\])") or "./") .. "duffle_paths.lua")
--- ```
---
--- That single line: (a) locates this helper via `arg[0]`, (b) loads
--- it (which sets `package.path` + `package.cpath` via `git rev-parse`),
--- (c) returns the `M` table (a wrapper around the setup function).
--- After this line, `require("duffle")` and `require("passes.X")` both
--- resolve normally.
---
--- **Why a helper instead of inline?**
--- - The 8-line path-setup boilerplate was duplicated across 7 entry
--- scripts (one per file). Single source of truth here.
--- - Mirrors the build script's pattern in `build_psyq.ps1`:
--- `$path_root = split-path -Path $PSScriptRoot -Parent;` then
--- derive everything from there.
---
--- **Why `git rev-parse --show-toplevel`?**
--- Hardcoding `C:\\projects\\Pikuma\\ps1\\...` breaks portability. Git
--- gives us the canonical repo root regardless of where the repo lives
--- on disk.
local M = {}
--- Resolve the repo root via git. Returns a normalized path with a
--- trailing forward-slash, or nil if not in a git repo.
--- @return string|nil
local function find_repo_root()
local p = io.popen("git rev-parse --show-toplevel 2>nul")
local root
if p then
root = p:read("*l")
p:close()
end
if not root or root == "" then return nil end
-- Normalize to forward slashes (Windows accepts both, but mixed
-- `\` + `/` confuses LuaJIT's file APIs).
root = root:gsub("\\", "/")
if not root:match("/$") then root = root .. "/" end
return root
end
--- Set `package.path` (for `require("duffle")` + `require("passes.X")`)
--- and `package.cpath` (for `lpeg.dll` on Windows).
function M.setup()
local repo_root = find_repo_root()
if not repo_root then
io.stderr:write("[duffle_paths] git rev-parse failed -- not in a git repo?\n")
os.exit(2)
end
local scripts_dir = repo_root .. "scripts/"
local passes_dir = repo_root .. "scripts/passes/"
package.path = scripts_dir .. "?.lua;"
.. scripts_dir .. "?/init.lua;"
.. passes_dir .. "?.lua;"
.. passes_dir .. "?/init.lua;"
.. package.path
if package.config:sub(1, 1) == "\\" then
package.cpath = repo_root .. "toolchain/luajit-2.1/lib/lua/5.1/?.dll;"
.. package.cpath
end
end
-- Run the setup as a side effect.
M.setup()
return M
scripts/passes/offsets.lua
+387 -235
View File
@@ -1,286 +1,405 @@
-- passes/offsets.lua
--
-- Generate <module>/gen/<basename>.offsets.h with branch offset
-- immediates for every atom_offset(F, T) reference in atom bodies.
--
-- The branch offset regression we just fixed in commit 98e27c2 must
-- NOT return. The fix was in duffle.lua's split_top_level_commas +
-- tape_atom_annotation_pass.lua's compute_component_word_count.
-- word_count_eval.count_token_words preserves the fix.
--
-- THIS MODULE ALSO REQUIRES the recent fix to duffle.lua's
-- split_top_level_commas (the second-half of the 98e27c2 fix):
-- top-level comments must be appended to the previous token, not
-- stripped, so the emit path preserves `// trailing comment` text
-- for convert_line_comments_to_block to convert to `/* */`.
--
-- Coding standard: tabs (1/level), EmmyLua annotations, no regex.
--- passes/offsets.lua — Branch-offset generator.
---
--- Scans every source for `MipsAtom_(name) { ... }` (and the raw
--- `MipsCode code_<name> { ... }` form) declarations, computes the
--- word offset from each `atom_offset(F, T)` marker to its target
--- `atom_label(T)` declaration, and emits `<dir_basename>.offsets.h`
--- with one `#define _atom_offset_F_T = N` per branch.
---
--- The offset is `target_word - branch_word - 1` (the standard MIPS
--- branch-immediate encoding: branch_offset = relative_pc_in_words - 1).
---
--- **Conventions**: tabs (1/level), EmmyLua annotations, no regex,
--- Lua 5.3 compatible. See
--- `C:\projects\Pikuma\ps1-ai\conductor\code_styleguides\lua.md`.
-- ════════════════════════════════════════════════════════════════════════════
-- Module-scope requires + package.path setup
-- ════════════════════════════════════════════════════════════════════════════
local script_path = arg and arg[0] or "?"
local last_sep = 0
for i = 1, #script_path do
local c = script_path:sub(i, i)
if c == "/" or c == "\\" then last_sep = i end
end
local script_dir = last_sep == 0 and "./" or script_path:sub(1, last_sep)
package.path = script_dir .. "../?.lua;" .. script_dir .. "../?/init.lua;" .. script_dir .. "?.lua;" .. package.path
local duffle = require("duffle")
local trim = duffle.trim
local read_ident = duffle.read_ident
local is_space = duffle.is_space
local is_alpha = duffle.is_alpha
local is_alnum = duffle.is_alnum
local skip_ws_and_cmt = duffle.skip_ws_and_cmt
local skip_str_or_cmt = duffle.skip_str_or_cmt
local split_top_level_commas = duffle.split_top_level_commas
local read_parens = duffle.read_parens
local read_braces = duffle.read_braces
local write_file = duffle.write_file
local dirname = duffle.dirname
local basename_no_ext = duffle.basename_no_ext
local ensure_dir = duffle.ensure_dir
local word_count_eval = require("word_count_eval")
local count_token_words = word_count_eval.count_token_words
-- Resolve `arg[0]` to an absolute-ish script directory so that
-- `require("duffle")` resolves against `scripts/` regardless of CWD.
-- Note: this boilerplate is duplicated in 6 other entry scripts; a
-- Phase-6 extraction target (`duffle.setup_package_path()`).
-- Bootstrap: see `ps1_meta.lua` for the rationale.
dofile((arg[0]:match("(.*[/\\])") or "./") .. "duffle_paths.lua")
local duffle = require("duffle")
local word_count_eval = require("word_count_eval")
local count_token_words = word_count_eval.count_token_words
-- ════════════════════════════════════════════════════════════════════════════
-- Local helpers (ported from offset_gen.meta.lua lines 67-93)
-- Constants
-- ════════════════════════════════════════════════════════════════════════════
-- C qualifier keywords that may precede a `MipsAtom_` declaration
-- (and should be skipped by `skip_qualifiers`).
local QUALIFIER_KEYWORDS = {
["static"] = true, ["const"] = true, ["volatile"] = true,
["extern"] = true, ["register"] = true, ["auto"] = true,
["inline"] = true, ["typedef"] = true,
["internal"] = true, ["LP_"] = true, ["global"] = true, ["gkknown"] = true,
}
-- Atom declaration identifiers.
local ATOM_PREFIX = "MipsAtom_"
local CODE_DECL = "MipsCode"
local CODE_RAW_PREFIX = "code_" -- raw atom form: `MipsCode code_<name> { ... }`
local CODE_RAW_PREFIX_LEN = 5 -- = #CODE_RAW_PREFIX
-- Marker-call identifiers inside atom bodies.
local LABEL_MARKER = "atom_label"
local OFFSET_MARKER = "atom_offset"
-- Offset macro/enum naming prefixes (the emitted header uses these).
local OFFSET_MACRO_PREFIX = "_atom_offset_"
local OFFSET_ENUM_PREFIX = "atom_offset_"
-- Column width for the `#define _atom_offset_F_T = N` alignment.
local OFFSET_MACRO_COL = 44
-- ════════════════════════════════════════════════════════════════════════════
-- Type declarations
-- ════════════════════════════════════════════════════════════════════════════
--- @class SourceFile
--- @field path string -- absolute path to the source file
--- @field text string -- the full source text
--- @field dir string -- the directory containing the source
--- @field basename string -- filename without extension
--- @class PassCtx
--- @field sources SourceFile[] -- all source files in the build
--- @field metadata_path string -- path to word_count.metadata.h
--- @field shared table -- cross-pass shared state
--- @field shared.word_counts table -- macro name -> word count
--- @field out_root string -- output root (e.g. "build/gen")
--- @field project_root string -- project root (e.g. "code/")
--- @field upstream table<string, table> -- per-pass upstream outputs
--- @field flags table -- CLI flags
--- @field dry_run boolean -- if true, compute but don't write
--- @field verbose boolean -- if true, log diagnostic info
--- @class PassResult
--- @field outputs table[] -- {kind=, path=} entries describing emit files
--- @field errors table[] -- {line=, msg=} entries; build-stops
--- @field warnings table[] -- {line=, msg=} entries; build-succeeds
--- @class Atom
--- @field name string -- atom name (e.g. "cube_g4_face")
--- @field body string -- the brace-delimited body (without the braces)
--- @class BranchOffset
--- @field tag string -- the marker tag (e.g. "F" in `atom_offset(F, T)`)
--- @field target string -- the target label name (e.g. "T" in `atom_offset(F, T)`)
--- @field pos integer -- the branch's word position within the atom body
--- @field offset integer -- computed `target_word - branch_word - 1`
--- @class AtomData
--- @field name string -- atom name
--- @field total_words integer -- total word count of the atom body
--- @field offsets BranchOffset[] -- per-branch offset list
-- ════════════════════════════════════════════════════════════════════════════
-- Local helpers
-- ════════════════════════════════════════════════════════════════════════════
-- Returns true if `s` starts with `prefix`.
-- @param s string
-- @param prefix string
-- @return boolean
local function starts_with(s, prefix)
if #s < #prefix then return false end
for i = 1, #prefix do
if s:sub(i, i) ~= prefix:sub(i, i) then return false end
for pos = 1, #prefix do
if s:sub(pos, pos) ~= prefix:sub(pos, pos) then return false end
end
return true
end
local function to_upper(s) return s:upper() end
-- Replace every non-alphanumeric char in `s` with underscore.
-- @param s string
-- @return string
local function to_alnum_underscore(s)
local out = ""
for i = 1, #s do
local c = s:sub(i, i)
if is_alnum(c) then out = out .. c
else out = out .. "_" end
for pos = 1, #s do
local ch = s:sub(pos, pos)
if duffle.is_alnum(ch) then out = out .. ch else out = out .. "_" end
end
return out
end
local function pad_right(s, w) return s .. string.rep(" ", w - #s) end
-- Right-pad `s` with spaces to width `w`. If `s` is already `w` or
-- wider, no padding is added.
-- @param s string
-- @param w integer
-- @return string
local function pad_right(s, w)
return s .. string.rep(" ", math.max(0, w - #s))
end
-- ════════════════════════════════════════════════════════════════════════════
-- Marker-call helpers (ported from offset_gen.meta.lua lines 148-205)
-- Marker-call helpers
-- ════════════════════════════════════════════════════════════════════════════
--- Extract comma-separated identifier args from a parenthesized group
--- after a function-like macro call.
-- Extract comma-separated identifier args from a parenthesized group
-- after a function-like macro call. Returns (args, after_paren) where
-- `after_paren` is the position just past the closing `)`, or nil if
-- `token` did not start with `(`.
-- @param token string
-- @param after_ident integer
-- @return string[], integer|nil
local function extract_ident_args(token, after_ident)
local arg_start = skip_ws_and_cmt(token, after_ident)
local arg_start = duffle.skip_ws_and_cmt(token, after_ident)
if token:sub(arg_start, arg_start) ~= "(" then return {}, nil end
local inner, after_paren = read_parens(token, arg_start)
local inner, after_paren = duffle.read_parens(token, arg_start)
local args = {}
local n = 1
local len = #inner
while n <= len do
n = skip_ws_and_cmt(inner, n)
if n > len then break end
local ident, after = read_ident(inner, n)
local pos = 1
local inner_len = #inner
while pos <= inner_len do
pos = duffle.skip_ws_and_cmt(inner, pos)
if pos > inner_len then break end
local ident, after = duffle.read_ident(inner, pos)
if ident and ident ~= "" then
table.insert(args, ident)
n = after
pos = after
else
n = n + 1
pos = pos + 1
end
n = skip_ws_and_cmt(inner, n)
if n <= len and inner:sub(n, n) == "," then n = n + 1 end
pos = duffle.skip_ws_and_cmt(inner, pos)
if pos <= inner_len and inner:sub(pos, pos) == "," then pos = pos + 1 end
end
return args, after_paren
end
-- (internal) Record a `atom_label(name)` marker — `at_pos` is the
-- branch-free word position within the atom body.
-- @param labels table<string, integer>
-- @param args string[]
-- @param at_pos integer
local function record_label_marker(labels, args, at_pos)
if #args >= 1 then labels[args[1]] = at_pos end
end
-- (internal) Record a `atom_offset(tag, target)` marker.
-- @param branches table[] -- list of {pos=, target=, tag=}
-- @param args string[]
-- @param at_pos integer
local function record_offset_marker(branches, args, at_pos)
if #args >= 2 then
table.insert(branches, { pos = at_pos, target = args[2], tag = args[1] })
end
end
--- Scan a single token for atom_label/atom_offset markers, walking through
--- balanced groups transparently (so nested calls are found).
--- @param token string
--- @param at_pos integer -- the branch-free word position of this token in the body
--- @param labels table<string, integer>
--- @param branches table[]
local function scan_for_atom_markers(token, at_pos, labels, branches)
local i = 1
local len = #token
while i <= len do
i = skip_ws_and_cmt(token, i)
if i > len then break end
local c = token:sub(i, i)
if is_alpha(c) then
local ident, after = read_ident(token, i)
if ident == "atom_label" then
local pos = 1
local tok_len = #token
while pos <= tok_len do
pos = duffle.skip_ws_and_cmt(token, pos)
if pos > tok_len then break end
local ch = token:sub(pos, pos)
if duffle.is_alpha(ch) then
local ident, after = duffle.read_ident(token, pos)
if ident == LABEL_MARKER then
local args, after_paren = extract_ident_args(token, after)
if #args >= 1 then labels[args[1]] = at_pos end
if after_paren then i = after_paren else i = after end
elseif ident == "atom_offset" then
record_label_marker(labels, args, at_pos)
pos = after_paren or after
elseif ident == OFFSET_MARKER then
local args, after_paren = extract_ident_args(token, after)
if #args >= 2 then table.insert(branches, {pos = at_pos, target = args[2], tag = args[1]}) end
if after_paren then i = after_paren else i = after end
record_offset_marker(branches, args, at_pos)
pos = after_paren or after
else
i = after
pos = after
end
else
local nx = skip_str_or_cmt(token, i)
if nx > i then i = nx else i = i + 1 end
local nx = duffle.skip_str_or_cmt(token, pos)
pos = (nx > pos) and nx or (pos + 1)
end
end
end
--- Find the end position (just past the closing ')') of the first
--- atom_label/atom_offset call in `tok`. Returns 0 if no such call.
--- @param tok string
--- @return integer -- 0 if no marker call found; otherwise end-1 (just past ')')
local function find_marker_call_end(tok)
local i = 1
local len = #tok
while i <= len do
i = skip_ws_and_cmt(tok, i)
if i > len then break end
local c = tok:sub(i, i)
if is_space(c) then
i = i + 1
elseif c == "/" then
local pos = 1
local tok_len = #tok
while pos <= tok_len do
pos = duffle.skip_ws_and_cmt(tok, pos)
if pos > tok_len then break end
local ch = tok:sub(pos, pos)
if duffle.is_space(ch) then
pos = pos + 1
elseif ch == "/" then
-- comment — skip past it (delegated to duffle.skip_str_or_cmt)
local nx = skip_str_or_cmt(tok, i)
if nx > i then i = nx else i = i + 1 end
local nx = duffle.skip_str_or_cmt(tok, pos)
pos = (nx > pos) and nx or (pos + 1)
else
local ident, after = read_ident(tok, i)
if ident == "atom_label" or ident == "atom_offset" then
local j = skip_ws_and_cmt(tok, after)
if tok:sub(j, j) == "(" then
local _, end_paren = read_parens(tok, j)
local ident, after_ident = duffle.read_ident(tok, pos)
if ident == LABEL_MARKER or ident == OFFSET_MARKER then
local open_paren = duffle.skip_ws_and_cmt(tok, after_ident)
if tok:sub(open_paren, open_paren) == "(" then
local _, end_paren = duffle.read_parens(tok, open_paren)
return end_paren - 1
end
return 0
end
i = after or (i + 1)
pos = after_ident or (pos + 1)
end
end
return 0
end
-- ════════════════════════════════════════════════════════════════════════════
-- Atom scanner (ported from offset_gen.meta.lua lines 245-321)
-- Atom scanner
-- ════════════════════════════════════════════════════════════════════════════
--- Skip C qualifier keywords (static, const, etc.) and return the position
--- past the last qualifier.
local function skip_qualifiers(source, i)
local keywords = {
["static"] = true, ["const"] = true, ["volatile"] = true,
["extern"] = true, ["register"] = true, ["auto"] = true,
["inline"] = true, ["typedef"] = true,
["internal"]= true, ["LP_"] = true, ["global"] = true, ["gkknown"] = true,
}
--- Skip C qualifier keywords (`static`, `const`, etc.) and return the
--- position past the last qualifier.
--- @param source string
--- @param pos integer
--- @return integer
local function skip_qualifiers(source, pos)
while true do
i = skip_ws_and_cmt(source, i)
local ident, after = read_ident(source, i)
if not ident then return i end
if keywords[ident] then i = after else return i end
pos = duffle.skip_ws_and_cmt(source, pos)
local ident, after = duffle.read_ident(source, pos)
if not ident then return pos end
if QUALIFIER_KEYWORDS[ident] then pos = after else return pos end
end
end
--- Find every MipsAtom_(name) { ... } in a source.
-- (internal) Try to parse the wrapped atom form: `MipsAtom_(<name>) { ... }`.
-- Returns the parsed Atom (name + body + position past body), or nil if
-- the form didn't match.
-- @param source_text string
-- @param after_pos integer -- position just past `MipsAtom_`
-- @return Atom|nil
local function try_wrapped_atom(source_text, after_pos)
local paren_pos = duffle.skip_ws_and_cmt(source_text, after_pos)
if source_text:sub(paren_pos, paren_pos) ~= "(" then return nil end
local inner, after_paren = duffle.read_parens(source_text, paren_pos)
local name_start = 1
while name_start <= #inner and duffle.is_space(inner:sub(name_start, name_start)) do
name_start = name_start + 1
end
local name_end = name_start
while name_end <= #inner and duffle.is_alnum(inner:sub(name_end, name_end)) do
name_end = name_end + 1
end
local name = inner:sub(name_start, name_end - 1)
if name == "" then return nil end
local brace_pos = duffle.scan_to_char(source_text, "{", after_paren)
if not brace_pos then return nil end
local body, after_brace = duffle.read_braces(source_text, brace_pos)
return { name = name, body = body, after_brace = after_brace }
end
-- (internal) Try to parse the raw atom form: `MipsCode code_<name> { ... }`.
-- @param source_text string
-- @param after_pos integer -- position just past `MipsCode`
-- @return Atom|nil
local function try_raw_atom(source_text, after_pos)
local next_pos = duffle.skip_ws_and_cmt(source_text, after_pos)
local next_ident, next_after = duffle.read_ident(source_text, next_pos)
if not next_ident then return nil end
if not starts_with(next_ident, CODE_RAW_PREFIX) then return nil end
if #next_ident <= CODE_RAW_PREFIX_LEN then return nil end
local atom_name = next_ident:sub(CODE_RAW_PREFIX_LEN + 1)
local brace_pos = duffle.scan_to_char(source_text, "{", next_after)
if not brace_pos then return nil end
local body, after_brace = duffle.read_braces(source_text, brace_pos)
return { name = atom_name, body = body, after_brace = after_brace }
end
--- Find every `MipsAtom_(name) { ... }` (or raw `MipsCode code_<name> { ... }`)
--- declaration in a source.
--- @param source_text string
--- @return Atom[]
local function find_atoms(source_text)
local atoms = {}
local len = #source_text
local i = 1
local atoms = {}
local pos = 1
local src_len = #source_text
local function try_wrapped(after_pos)
local paren_pos = skip_ws_and_cmt(source_text, after_pos)
if source_text:sub(paren_pos, paren_pos) ~= "(" then return nil end
local inner, after_paren = read_parens(source_text, paren_pos)
local n = 1
while n <= #inner and is_space(inner:sub(n, n)) do n = n + 1 end
local ns = n
while n <= #inner and is_alnum(inner:sub(n, n)) do n = n + 1 end
local name = inner:sub(ns, n - 1)
if name == "" then return nil end
-- Find the brace after the parens.
local brace_pos = duffle.scan_to_char(source_text, "{", after_paren)
if not brace_pos then return nil end
local body, after_brace = read_braces(source_text, brace_pos)
return {name = name, body = body, after_brace = after_brace}
end
while pos <= src_len do
pos = duffle.skip_ws_and_cmt(source_text, pos); if pos > src_len then break end
pos = skip_qualifiers(source_text, pos); if pos > src_len then break end
local function try_raw(after_pos)
local next_pos = skip_ws_and_cmt(source_text, after_pos)
local next_ident, next_after = read_ident(source_text, next_pos)
if not next_ident then return nil end
if not starts_with(next_ident, "code_") then return nil end
if #next_ident <= 5 then return nil end
local atom_name = next_ident:sub(6)
local brace_pos = duffle.scan_to_char(source_text, "{", next_after)
if not brace_pos then return nil end
local body, after_brace = read_braces(source_text, brace_pos)
return {name = atom_name, body = body, after_brace = after_brace}
end
while i <= len do
i = skip_ws_and_cmt(source_text, i); if i > len then break end
i = skip_qualifiers(source_text, i); if i > len then break end
local ident, after = read_ident(source_text, i)
local ident, after = duffle.read_ident(source_text, pos)
if not ident then
i = i + 1
elseif ident == "MipsAtom_" then
local atom = try_wrapped(after)
pos = pos + 1
elseif ident == ATOM_PREFIX then
local atom = try_wrapped_atom(source_text, after)
if atom then
table.insert(atoms, {name = atom.name, body = atom.body})
i = atom.after_brace
atoms[#atoms + 1] = { name = atom.name, body = atom.body }
pos = atom.after_brace
else
i = i + 1
pos = pos + 1
end
elseif ident == "MipsCode" then
local atom = try_raw(after)
elseif ident == CODE_DECL then
local atom = try_raw_atom(source_text, after)
if atom then
table.insert(atoms, {name = atom.name, body = atom.body})
i = atom.after_brace
atoms[#atoms + 1] = { name = atom.name, body = atom.body }
pos = atom.after_brace
else
i = after
pos = after
end
else
i = after
pos = after
end
end
return atoms
end
-- ════════════════════════════════════════════════════════════════════════════
-- Per-atom body scan (ported from offset_gen.meta.lua lines 207-239)
-- Per-atom body scan
-- ════════════════════════════════════════════════════════════════════════════
-- (internal) Count words emitted by the rest of `tok` after a marker call
-- (the marker call itself emits 0 words, but the source pattern may bundle
-- the marker with the next instruction on the same line, separated by no
-- top-level comma). Returns the word count contributed by that rest.
-- @param tok string
-- @param word_counts table
-- @return integer
local function count_marker_rest(tok, word_counts)
local marker_end = find_marker_call_end(tok)
if marker_end <= 0 or marker_end >= #tok then return 0 end
local rest = duffle.trim(tok:sub(marker_end + 1))
if rest == "" then return 0 end
return count_token_words(rest, word_counts)
end
-- (internal) Is this token a marker call (`atom_label` or `atom_offset`)?
-- @param tok string
-- @return boolean
local function is_marker_token(tok)
local leading_ident = duffle.read_ident(tok, 1)
return leading_ident == LABEL_MARKER or leading_ident == OFFSET_MARKER
end
--- Scan an atom body for labels + branches, count total words.
--- Returns (labels, branches, total_words).
--- @param body string
--- @param word_counts table
--- @return table<string, integer>, table[], integer
local function scan_atom_body(body, word_counts)
local pos = 0
local labels = {}
local branches = {}
for _, tok in ipairs(split_top_level_commas(body)) do
local k = 1
local tlen = #tok
while k <= tlen and is_space(tok:sub(k, k)) do k = k + 1 end
local leading_ident = read_ident(tok, k)
if leading_ident == "atom_label" or leading_ident == "atom_offset" then
for _, tok in ipairs(duffle.split_top_level_commas(body)) do
if is_marker_token(tok) then
-- Marker call: record at the current pos, do NOT advance pos.
-- But the source pattern may bundle the marker with the next
-- instruction on a new line (no top-level comma between them).
-- In that case, the rest of `tok` after the marker call is
-- a real instruction that must still be counted.
scan_for_atom_markers(tok, pos, labels, branches)
local marker_end = find_marker_call_end(tok)
if marker_end > 0 and marker_end < #tok then
local rest = trim(tok:sub(marker_end + 1))
if rest ~= "" then
local rest_words = count_token_words(rest, word_counts)
pos = pos + rest_words
end
end
pos = pos + count_marker_rest(tok, word_counts)
else
local words = count_token_words(tok, word_counts)
scan_for_atom_markers(tok, pos, labels, branches)
@@ -291,10 +410,14 @@ local function scan_atom_body(body, word_counts)
end
-- ════════════════════════════════════════════════════════════════════════════
-- Offset computation + header generation (ported lines 327-383)
-- Offset computation + header generation
-- ════════════════════════════════════════════════════════════════════════════
--- Compute branch offsets as (target_word - branch_word - 1).
-- Compute branch offsets as `target_word - branch_word - 1` (the
-- standard MIPS branch-immediate encoding).
-- @param labels table<string, integer>
-- @param branches table[]
-- @return BranchOffset[]
local function compute_offsets(labels, branches)
local results = {}
for _, br in ipairs(branches) do
@@ -302,17 +425,55 @@ local function compute_offsets(labels, branches)
if not target then
error("Branch target '" .. br.target .. "' has no atom_label (at word " .. br.pos .. ")")
end
table.insert(results, {target = br.target, tag = br.tag, offset = target - br.pos - 1})
results[#results + 1] = { target = br.target, tag = br.tag, offset = target - br.pos - 1 }
end
return results
end
--- Generate the per-source .offsets.h header.
-- (internal) Build a constant-table entry `{macro_name, enum_name, value}`
-- from a BranchOffset.
-- @param r BranchOffset
-- @return table
local function make_offset_const(r)
return {
macro_name = OFFSET_MACRO_PREFIX .. r.tag .. "_" .. r.target,
enum_name = OFFSET_ENUM_PREFIX .. r.tag .. "_" .. r.target,
value = r.offset,
}
end
-- (internal) Emit one atom's offset constants + enum into the lines buffer.
-- @param add fun(s: string)
-- @param atom AtomData
local function emit_atom_offsets(add, atom)
if #atom.offsets == 0 then return end
add("// --- atom: " .. atom.name .. " (" .. atom.total_words .. " words) ---")
add("")
local consts = {}
for _, r in ipairs(atom.offsets) do
consts[#consts + 1] = make_offset_const(r)
end
for _, c in ipairs(consts) do
add("#define " .. pad_right(c.macro_name, OFFSET_MACRO_COL) .. " " .. c.value)
end
add("")
add("enum {")
for _, c in ipairs(consts) do
add(" " .. c.enum_name .. " = " .. c.macro_name .. ",")
end
add("};")
add("")
end
-- Generate the per-source .offsets.h header.
-- @param source_path string
-- @param atoms_data AtomData[]
-- @return string
local function generate_header(source_path, atoms_data)
local basename = basename_no_ext(source_path)
local basename = duffle.basename_no_ext(source_path)
local lines = {}
local function add(s) table.insert(lines, s) end
local function add(s) lines[#lines + 1] = s end
add("// Auto-generated by ps1_meta.lua (passes/offsets.lua) — DO NOT EDIT")
add("// Source: " .. source_path)
@@ -322,28 +483,7 @@ local function generate_header(source_path, atoms_data)
add("")
add("")
for _, atom in ipairs(atoms_data) do
if #atom.offsets > 0 then
add("// --- atom: " .. atom.name .. " (" .. atom.total_words .. " words) ---")
add("")
local consts = {}
for _, r in ipairs(atom.offsets) do
table.insert(consts, {
macro_name = "_atom_offset_" .. r.tag .. "_" .. r.target,
enum_name = "atom_offset_" .. r.tag .. "_" .. r.target,
value = r.offset,
})
end
for _, c in ipairs(consts) do
add("#define " .. pad_right(c.macro_name, 44) .. " " .. c.value)
end
add("")
add("enum {")
for _, c in ipairs(consts) do
add(" " .. c.enum_name .. " = " .. c.macro_name .. ",")
end
add("};")
add("")
end
emit_atom_offsets(add, atom)
end
add("#pragma endregion " .. basename)
add("")
@@ -351,13 +491,41 @@ local function generate_header(source_path, atoms_data)
end
-- ════════════════════════════════════════════════════════════════════════════
-- M.run — orchestrator entry
-- M — module exports
-- ════════════════════════════════════════════════════════════════════════════
--- @class M
local M = {}
-- (internal) Process one source: find atoms, scan bodies, write header.
-- Returns the offsets_h path if a header was written, or nil.
-- @param ctx PassCtx
-- @param src SourceFile
-- @return string|nil -- the offsets_h path
local function process_source(ctx, src)
local atoms = find_atoms(src.text)
if #atoms == 0 then return nil end
local atoms_data = {}
for _, atom in ipairs(atoms) do
local labels, branches, total = scan_atom_body(atom.body, ctx.shared.word_counts)
atoms_data[#atoms_data + 1] = {
name = atom.name,
total_words = total,
offsets = compute_offsets(labels, branches),
}
end
local out_path = src.dir .. "/gen/" .. duffle.basename_no_ext(src.dir) .. ".offsets.h"
if not ctx.dry_run then
duffle.ensure_dir(duffle.dirname(out_path))
duffle.write_file(out_path, generate_header(src.path, atoms_data))
end
return out_path
end
--- Run the offsets pass. For each source, emits a per-module
--- `<dir_basename>.offsets.h` containing `#define _atom_offset_F_T = N`
--- constants for every `atom_offset(F, T)` reference in the source's atoms.
--- @param ctx PassCtx
--- @return PassResult
function M.run(ctx)
@@ -366,25 +534,9 @@ function M.run(ctx)
local warnings = {}
for _, src in ipairs(ctx.sources) do
local atoms = find_atoms(src.text)
if #atoms > 0 then
local atoms_data = {}
for _, atom in ipairs(atoms) do
local labels, branches, total = scan_atom_body(atom.body, ctx.shared.word_counts)
local offsets = compute_offsets(labels, branches)
table.insert(atoms_data, {
name = atom.name,
total_words = total,
offsets = offsets,
})
end
local out_path = src.dir .. "/gen/" .. basename_no_ext(src.dir) .. ".offsets.h"
if not ctx.dry_run then
ensure_dir(dirname(out_path))
write_file(out_path, generate_header(src.path, atoms_data))
end
table.insert(outputs, { offsets_h = out_path })
local out_path = process_source(ctx, src)
if out_path then
outputs[#outputs + 1] = { offsets_h = out_path }
end
end
scripts/passes/report.lua
+320 -136
View File
@@ -1,66 +1,174 @@
-- passes/report.lua
--
-- Render the per-project summary (build/gen/annotation_validation.txt)
-- + the per-MODULE annotation reports (build/gen/<dir_basename>.annotations.txt).
-- Aggregates errors + warnings from upstream annotation pass results.
--
-- The annotation pass stashes its per-MODULE results in
-- ctx.flags._annot_results (set by passes/annotation.lua). This report
-- pass renders them. Each entry contains a `dir`, `dir_basename`, and
-- `atoms_count`; the detailed per-source data still lives in the
-- annotation pass's internal result objects, accessible via the shared
-- `ctx.sources` list (re-validated by reference, not by value).
--
-- Coding standard: tabs (1/level), EmmyLua annotations, no regex.
--- passes/report.lua — Per-MODULE annotation report renderer +
--- project-wide summary writer.
---
--- Two output files per build:
--- - `build/gen/<dir_basename>.annotations.txt` — one per source-directory
--- containing atoms; aggregates across all sources in the directory.
--- - `build/gen/annotation_validation.txt` — the project summary.
---
--- The annotation pass stashes per-MODULE summary entries in
--- `ctx.flags._annot_results` (set by `passes/annotation.lua`). This
--- pass re-validates each source via `annotation.validate()` to get
--- the detailed per-source results needed for the report. The cost is
--- acceptable: `validate()` is fast (~5ms per source) and runs once.
---
--- **Conventions**: tabs (1/level), EmmyLua annotations, no regex,
--- Lua 5.3 compatible. See
--- `C:\projects\Pikuma\ps1-ai\conductor\code_styleguides\lua.md`.
-- ════════════════════════════════════════════════════════════════════════════
-- Module-scope requires + package.path setup
-- ════════════════════════════════════════════════════════════════════════════
local script_path = arg and arg[0] or "?"
local last_sep = 0
for i = 1, #script_path do
local c = script_path:sub(i, i)
if c == "/" or c == "\\" then last_sep = i end
end
local script_dir = last_sep == 0 and "./" or script_path:sub(1, last_sep)
package.path = script_dir .. "../?.lua;" .. script_dir .. "../?/init.lua;" .. script_dir .. "?.lua;" .. package.path
-- Resolve `arg[0]` to an absolute-ish script directory so that
-- `require("duffle")` resolves against `scripts/` regardless of CWD.
-- Note: this boilerplate is duplicated in 6 other entry scripts; a
-- Phase-6 extraction target (`duffle.setup_package_path()`).
-- Bootstrap: see `ps1_meta.lua` for the rationale.
dofile((arg[0]:match("(.*[/\\])") or "./") .. "duffle_paths.lua")
local duffle = require("duffle")
local duffle = require("duffle")
local ensure_dir = duffle.ensure_dir
local write_file = duffle.write_file
-- ════════════════════════════════════════════════════════════════════════════
-- Constants
-- ════════════════════════════════════════════════════════════════════════════
-- Section separators used in the rendered text reports. The thin rules
-- are hand-tuned to align with the per-section content width; do not
-- change without also checking the section renderers below.
local RULE_THICK = "========================================================"
local SECTION_HEADER_ATOMS = "── Atoms ────────────────────────────────────────────────"
local SECTION_HEADER_ANNOTS = "── Annotations ──────────────────────────────────────────"
local SECTION_HEADER_BINDS = "── Binds_* structs ──────────────────────────────────────"
local SECTION_HEADER_MACROS = "── Macro word-count declarations ─────────────────────────"
local SECTION_HEADER_ERRORS = "── Errors ──────────────────────────────────────────────"
local SECTION_HEADER_WARNINGS = "── Warnings ────────────────────────────────────────────"
-- Lua pattern that captures the basename (last path segment) of a
-- forward- or back-slash separated path.
local BASENAME_PATTERN = "([^/\\]+)$"
-- Debug flag name — set to truthy in `_G` to enable verbose logging.
local DEBUG_FLAG = "_DEBUG_REPORT"
-- Pass identifier for log messages.
local PASS_NAME = "report"
-- ════════════════════════════════════════════════════════════════════════════
-- Type declarations
-- ════════════════════════════════════════════════════════════════════════════
--- @class SourceFile
--- @field path string -- absolute path to the source file
--- @field text string -- the full source text
--- @field dir string -- the directory containing the source
--- @field basename string -- filename without extension
--- @class PassCtx
--- @field sources SourceFile[] -- all source files in the build
--- @field metadata_path string -- path to word_count.metadata.h
--- @field shared table -- cross-pass shared state
--- @field out_root string -- output root (e.g. "build/gen")
--- @field project_root string -- project root (e.g. "code/")
--- @field upstream table<string, table> -- per-pass upstream outputs
--- @field flags table -- CLI flags + per-pass stash
--- @field flags._annot_results ModuleEntry[] -- stashed by annotation pass
--- @field dry_run boolean -- if true, compute but don't write
--- @field verbose boolean -- if true, log diagnostic info
--- @class PassResult
--- @field outputs table[] -- {kind=, path=} entries describing emit files
--- @field errors table[] -- {line=, msg=} entries; build-stops
--- @field warnings table[] -- {line=, msg=} entries; build-succeeds
-- Shapes produced by `passes/annotation.lua`'s `M.validate()`.
--- @class AtomEntry
--- @field name string -- atom name (e.g. "cube_g4_face")
--- @field line integer -- source line of the atom declaration
--- @class AnnotEntry
--- @field line integer -- source line
--- @field macro string -- the macro name (e.g. "atom_reads")
--- @field name string -- the atom name (if a `name(...)` was given)
--- @field kind string -- "atom_info" | "atom_bind" | ...
--- @field binds string|nil -- Binds_X name if any
--- @field reads string[] -- R_* names (read targets)
--- @field writes string[] -- R_* names (write targets)
--- @field error string|nil -- error message if annotation was malformed
--- @class BindsField
--- @field name string -- field name
--- @field offset integer -- byte offset within the Binds_X struct
--- @class BindsStruct
--- @field name string -- struct name (e.g. "Binds_Floor")
--- @field line integer -- source line of the typedef
--- @field bytes integer -- total byte size
--- @field fields BindsField[] -- the field list
--- @class MacroEntry
--- @field name string -- macro name (e.g. "WORD_COUNT(my_macro, 4)")
--- @field line integer -- source line
--- @field words integer -- declared word count
--- @class Finding
--- @field line integer -- source line
--- @field msg string -- finding message
--- @class AnnotationResult
--- @field source string -- set by this pass; original source path
--- @field atoms AtomEntry[] -- atom declarations in this source
--- @field annots AnnotEntry[] -- annotation entries
--- @field macros MacroEntry[] -- macro word-count declarations
--- @field binds BindsStruct[] -- Binds_* struct declarations
--- @field errors Finding[] -- errors from validation
--- @field warnings Finding[] -- warnings from validation
--- @field info table -- info summary (not rendered here)
--- @class ModuleEntry
--- @field dir string -- absolute directory path
--- @field dir_basename string -- basename (e.g. "duffle", "gte_hello")
--- @field atoms_count integer -- pre-counted atoms for filtering
--- @class ModuleReport
--- @field dir string -- module directory
--- @field sources SourceFile[] -- sources in this module
--- @field results AnnotationResult[] -- per-source validate() results
--- @class ProjectReport
--- @field results AnnotationResult[] -- all per-source results
-- ════════════════════════════════════════════════════════════════════════════
-- Per-MODULE annotation report (aggregated across all sources in a dir)
-- ════════════════════════════════════════════════════════════════════════════
--
-- We no longer re-validate the source here; the annotation pass has
-- already done the work and stored the per-source results internally
-- (re-validating here would double the work). Instead, this pass reads
-- the per-source `validate()` results via a callback re-validation:
-- the annotation pass stashes the per-module summary, and we re-validate
-- each source once more to render the per-module report. The cost is
-- acceptable: validate() is fast (~5ms per source for our 19 sources)
-- and the report pass runs once at the end of the build.
--
-- In a future refactor we could pass the validate() result directly
-- through ctx.upstream.annotation to avoid the re-validation, but the
-- current approach keeps the passes loosely coupled.
local function render_module_report(dir, sources, results)
local lines = {}
local function add(s) lines[#lines + 1] = s end
-- Extract the basename (last path segment) of a forward- or back-slash
-- separated path. Returns the input unchanged if no separator is found.
-- @param path string
-- @return string
local function source_basename(path)
return path:match(BASENAME_PATTERN) or path
end
add("========================================================")
add("ANNOTATION PASS — module " .. dir:match("([^/\\]+)$") or dir)
add("========================================================")
add(string.format("Sources: %d", #sources))
for _, s in ipairs(sources) do
add(" " .. s.path)
-- (internal) Format a single annotation entry as one rendered line.
-- @param a AnnotEntry
-- @param src_name string
-- @return string
local function format_annot_line(a, src_name)
if a.error then
return string.format(" ✗ line %d %s [ERROR: %s] [%s]", a.line, a.macro or "?", a.error, src_name)
end
add("")
local line = string.format(" ● line %d %s [%s]", a.line, a.name, src_name)
if a.binds then line = line .. " binds=" .. a.binds end
if #a.reads > 0 then line = line .. " reads={" .. table.concat(a.reads, ",") .. "}" end
if #a.writes > 0 then line = line .. " writes={" .. table.concat(a.writes, ",") .. "}" end
return line
end
-- Tally totals across the module
-- (internal) Tally totals across all results in a module.
-- @param results AnnotationResult[]
-- @return integer, integer, integer, integer, integer, integer
local function tally_module_totals(results)
local total_atoms, total_annots, total_binds, total_macros = 0, 0, 0, 0
local total_errors, total_warnings = 0, 0
for _, r in ipairs(results) do
@@ -71,41 +179,38 @@ local function render_module_report(dir, sources, results)
total_errors = total_errors + #r.errors
total_warnings = total_warnings + #r.warnings
end
add(string.format("Atoms: %d Annotations: %d Binds structs: %d Macro decls: %d",
total_atoms, total_annots, total_binds, total_macros))
add("")
return total_atoms, total_annots, total_binds, total_macros, total_errors, total_warnings
end
-- Per-source breakdown (each source's atoms + annnots + binds)
-- Aggregated under the module's "── Atoms ──" section.
add("── Atoms ────────────────────────────────────────────────")
-- (internal) Section renderer: per-source atom declarations.
local function render_module_atoms_section(add, results)
add(SECTION_HEADER_ATOMS)
for _, r in ipairs(results) do
local src_name = r.source:match("([^/\\]+)$") or r.source
local src_name = source_basename(r.source)
for _, a in ipairs(r.atoms) do
add(string.format(" MipsAtom_(%s) line %d [%s]", a.name, a.line, src_name))
end
end
add("")
end
add("── Annotations ──────────────────────────────────────────")
-- (internal) Section renderer: per-source annotation entries.
local function render_module_annots_section(add, results)
add(SECTION_HEADER_ANNOTS)
for _, r in ipairs(results) do
local src_name = r.source:match("([^/\\]+)$") or r.source
local src_name = source_basename(r.source)
for _, a in ipairs(r.annots) do
if a.error then
add(string.format(" ✗ line %d %s [ERROR: %s] [%s]", a.line, a.macro or "?", a.error, src_name))
else
local line = string.format(" ● line %d %s [%s]", a.line, a.name, src_name)
if a.binds then line = line .. " binds=" .. a.binds end
if #a.reads > 0 then line = line .. " reads={" .. table.concat(a.reads, ",") .. "}" end
if #a.writes > 0 then line = line .. " writes={" .. table.concat(a.writes, ",") .. "}" end
add(line)
end
add(format_annot_line(a, src_name))
end
end
add("")
end
add("── Binds_* structs ──────────────────────────────────────")
-- (internal) Section renderer: per-source Binds_* struct declarations.
local function render_module_binds_section(add, results)
add(SECTION_HEADER_BINDS)
for _, r in ipairs(results) do
local src_name = r.source:match("([^/\\]+)$") or r.source
local src_name = source_basename(r.source)
for _, b in ipairs(r.binds) do
add(string.format(" %s line %d %d bytes [%s]", b.name, b.line, b.bytes, src_name))
for _, f in ipairs(b.fields) do
@@ -114,35 +219,79 @@ local function render_module_report(dir, sources, results)
end
end
add("")
end
add("── Macro word-count declarations ─────────────────────────")
-- (internal) Section renderer: per-source macro word-count declarations.
local function render_module_macros_section(add, results)
add(SECTION_HEADER_MACROS)
for _, r in ipairs(results) do
local src_name = r.source:match("([^/\\]+)$") or r.source
local src_name = source_basename(r.source)
for _, m in ipairs(r.macros) do
add(string.format(" %s line %d words=%d [%s]", m.name, m.line, m.words, src_name))
end
end
add("")
end
add("── Errors ──────────────────────────────────────────────")
if total_errors == 0 then add(" (none)") end
for _, r in ipairs(results) do
local src_name = r.source:match("([^/\\]+)$") or r.source
for _, e in ipairs(r.errors) do
add(string.format(" ✗ line %d %s [%s]", e.line, e.msg, src_name))
-- (internal) Section renderer: per-source errors (one-line + "(none)" if empty).
local function render_module_errors_section(add, results, total_errors)
add(SECTION_HEADER_ERRORS)
if total_errors == 0 then
add(" (none)")
else
for _, r in ipairs(results) do
local src_name = source_basename(r.source)
for _, e in ipairs(r.errors) do
add(string.format(" ✗ line %d %s [%s]", e.line, e.msg, src_name))
end
end
end
add("")
end
add("── Warnings ────────────────────────────────────────────")
if total_warnings == 0 then add(" (none)") end
for _, r in ipairs(results) do
local src_name = r.source:match("([^/\\]+)$") or r.source
for _, w in ipairs(r.warnings) do
add(string.format(" ⚠ line %d %s [%s]", w.line, w.msg, src_name))
-- (internal) Section renderer: per-source warnings (one-line + "(none)" if empty).
local function render_module_warnings_section(add, results, total_warnings)
add(SECTION_HEADER_WARNINGS)
if total_warnings == 0 then
add(" (none)")
else
for _, r in ipairs(results) do
local src_name = source_basename(r.source)
for _, w in ipairs(r.warnings) do
add(string.format(" ⚠ line %d %s [%s]", w.line, w.msg, src_name))
end
end
end
add("")
end
--- Render the per-MODULE annotation report (one `<dir_basename>.annotations.txt`).
--- @param dir string -- module directory path
--- @param sources SourceFile[] -- sources in this module
--- @param results AnnotationResult[] -- per-source validate() results
--- @return string -- the rendered report text
local function render_module_report(dir, sources, results)
local lines = {}
local function add(s) lines[#lines + 1] = s end
add(RULE_THICK)
add("ANNOTATION PASS — module " .. source_basename(dir))
add(RULE_THICK)
add(string.format("Sources: %d", #sources))
for _, s in ipairs(sources) do add(" " .. s.path) end
add("")
local total_atoms, total_annots, total_binds, total_macros, total_errors, total_warnings = tally_module_totals(results)
add(string.format("Atoms: %d Annotations: %d Binds structs: %d Macro decls: %d",
total_atoms, total_annots, total_binds, total_macros))
add("")
render_module_atoms_section(add, results)
render_module_annots_section(add, results)
render_module_binds_section(add, results)
render_module_macros_section(add, results)
render_module_errors_section(add, results, total_errors)
render_module_warnings_section(add, results, total_warnings)
return table.concat(lines, "\n") .. "\n"
end
@@ -151,13 +300,17 @@ end
-- Per-project summary (ported from tape_atom_annotation_pass.lua:1488-1528)
-- ════════════════════════════════════════════════════════════════════════════
--- Render the per-project summary (`build/gen/annotation_validation.txt`).
--- Aggregates totals across all sources; lists per-source error counts
--- if any source has errors.
--- @param all_results AnnotationResult[]
--- @return string
local function render_project_report(all_results)
local lines = {}
local function add(s) lines[#lines + 1] = s end
local total_atoms, total_annots, total_macros, total_binds = 0, 0, 0, 0
local total_errors, total_warnings = 0, 0
for _, r in ipairs(all_results) do
total_atoms = total_atoms + #r.atoms
total_annots = total_annots + #r.annots
@@ -167,9 +320,9 @@ local function render_project_report(all_results)
total_warnings = total_warnings + #r.warnings
end
add("========================================================")
add(RULE_THICK)
add("ANNOTATION VALIDATION — project summary")
add("========================================================")
add(RULE_THICK)
add("")
add(string.format("Atoms: %d", total_atoms))
add(string.format("Annotations: %d", total_annots))
@@ -184,7 +337,7 @@ local function render_project_report(all_results)
add("Per-source error counts:")
for _, r in ipairs(all_results) do
if #r.errors > 0 then
local src_name = r.source:match("([^/\\]+)$") or r.source
local src_name = source_basename(r.source)
add(string.format(" %s : %d error(s)", src_name, #r.errors))
end
end
@@ -195,13 +348,72 @@ local function render_project_report(all_results)
end
-- ════════════════════════════════════════════════════════════════════════════
-- M.run — orchestrator entry
-- Orchestration helpers
-- ════════════════════════════════════════════════════════════════════════════
--- @class M
-- Group source files by their `dir` field. Used to mirror the
-- per-DIRECTORY partitioning the annotation pass uses.
-- @param sources SourceFile[]
-- @return table<string, SourceFile[]> -- map of dir -> sources in that dir
local function group_sources_by_dir(sources)
local by_dir = {}
for _, src in ipairs(sources) do
by_dir[src.dir] = by_dir[src.dir] or {}
table.insert(by_dir[src.dir], src)
end
return by_dir
end
-- (internal) Validate each source in `dir_sources` via the annotation pass,
-- tagging each result with `result.source = src.path` for downstream rendering.
-- Returns the list of module results + the flat list of all results (for the
-- project-wide summary).
-- @param ctx PassCtx
-- @param dir_sources SourceFile[]
-- @return AnnotationResult[], AnnotationResult[]
local function validate_module_sources(ctx, dir_sources)
local annotation = require("passes.annotation")
local module_results = {}
local all_results = {}
for _, src in ipairs(dir_sources) do
local result = annotation.validate(ctx, src)
result.source = src.path
module_results[#module_results + 1] = result
all_results[#all_results + 1] = result
end
return module_results, all_results
end
-- (internal) Does this module's results contain anything worth emitting?
-- @param module_results AnnotationResult[]
-- @return boolean
local function module_has_content(module_results)
for _, r in ipairs(module_results) do
if #r.atoms > 0 or #r.annots > 0 or #r.binds > 0
or #r.macros > 0 or #r.errors > 0 or #r.warnings > 0 then
return true
end
end
return false
end
-- (internal) Log a debug message if `_G[DEBUG_FLAG]` is truthy.
-- @param fmt string
local function debug_log(fmt, ...)
if _G[DEBUG_FLAG] then
io.stderr:write(string.format("[%s] " .. fmt, PASS_NAME, ...))
end
end
-- ════════════════════════════════════════════════════════════════════════════
-- M — module exports
-- ════════════════════════════════════════════════════════════════════════════
local M = {}
--- Run the report pass. Renders one `<dir_basename>.annotations.txt`
--- per source-directory that has content, plus the project-wide
--- `annotation_validation.txt` summary.
--- @param ctx PassCtx
--- @return PassResult
function M.run(ctx)
@@ -209,67 +421,39 @@ function M.run(ctx)
local errors = {}
local warnings = {}
-- The annotation pass stashes per-MODULE summary entries in
-- ctx.flags._annot_results. Each entry has { dir, dir_basename,
-- atoms_count }. To render the per-module report we need the
-- detailed per-source `validate()` results, which we re-compute
-- here by calling annotation.validate() on each source. The cost
-- is acceptable (validate() is fast, no GTE/GPU work).
local module_entries = (ctx.flags and ctx.flags._annot_results) or {}
local by_dir = group_sources_by_dir(ctx.sources)
-- Group sources by dir (same partitioning the annotation pass uses).
local by_dir = {}
for _, src in ipairs(ctx.sources) do
by_dir[src.dir] = by_dir[src.dir] or {}
table.insert(by_dir[src.dir], src)
end
if not ctx.dry_run then duffle.ensure_dir(ctx.out_root) end
-- Render per-MODULE reports.
if not ctx.dry_run then ensure_dir(ctx.out_root) end
local all_results_for_summary = {}
for _, entry in ipairs(module_entries) do
local dir = entry.dir
local dir_basename = entry.dir_basename
local dir_sources = by_dir[dir] or {}
if _G._DEBUG_REPORT then
io.stderr:write(string.format("[report] entry: dir=%s basename=%s atoms_count=%d dir_sources=%d\n",
dir, dir_basename, entry.atoms_count, #dir_sources))
end
-- Skip dirs with zero atoms AND zero annotations
if entry.atoms_count > 0 or #dir_sources > 0 then
-- Re-validate each source to get the detailed results.
-- (We could pass the per-source results through ctx instead;
-- current approach is simpler and the re-validation is fast.)
local annotation = require("passes.annotation")
local module_results = {}
local has_content = false
for _, src in ipairs(dir_sources) do
local result = annotation.validate(ctx, src)
result.source = src.path
module_results[#module_results + 1] = result
all_results_for_summary[#all_results_for_summary + 1] = result
if #result.atoms > 0 or #result.annots > 0 or #result.binds > 0
or #result.macros > 0 or #result.errors > 0 or #result.warnings > 0 then
has_content = true
end
debug_log("entry: dir=%s basename=%s atoms_count=%d dir_sources=%d\n",
entry.dir, entry.dir_basename, entry.atoms_count, #(by_dir[entry.dir] or {}))
if entry.atoms_count > 0 or #(by_dir[entry.dir] or {}) > 0 then
local dir_sources = by_dir[entry.dir] or {}
local module_results, all_results = validate_module_sources(ctx, dir_sources)
for _, r in ipairs(all_results) do
all_results_for_summary[#all_results_for_summary + 1] = r
end
if has_content then
local out_path = ctx.out_root .. "/" .. dir_basename .. ".annotations.txt"
if module_has_content(module_results) then
local out_path = ctx.out_root .. "/" .. entry.dir_basename .. ".annotations.txt"
if not ctx.dry_run then
write_file(out_path, render_module_report(dir, dir_sources, module_results))
duffle.write_file(out_path, render_module_report(entry.dir, dir_sources, module_results))
end
table.insert(outputs, { annotations_txt = out_path })
elseif _G._DEBUG_REPORT then
io.stderr:write(string.format("[report] -> no content; skipping\n"))
outputs[#outputs + 1] = { annotations_txt = out_path }
else
debug_log(" -> no content; skipping\n")
end
end
end
-- Render project summary (across all sources).
if not ctx.dry_run and #all_results_for_summary > 0 then
local summary_path = ctx.out_root .. "/annotation_validation.txt"
write_file(summary_path, render_project_report(all_results_for_summary))
table.insert(outputs, { summary_txt = summary_path })
duffle.write_file(summary_path, render_project_report(all_results_for_summary))
outputs[#outputs + 1] = { summary_txt = summary_path }
end
return { outputs = outputs, errors = errors, warnings = warnings }
scripts/passes/static_analysis.lua
+201 -130
View File
@@ -1,67 +1,138 @@
-- passes/static_analysis.lua
--
-- Per-atom static-analysis checks for the tape-atom build pipeline.
-- Currently ships Phase 1 checks (GTE pipeline-fill + mac_yield
-- uniformity). Phases 2/3 (ABI handoff discipline, GPU port-store
-- shape, per-atom cycle budget) extend this file.
--
-- Workspace boundary: same conventions as annotation.lua
-- - primitives from duffle.lua (read_parens, read_braces, scan_to_char, LineIndex)
-- - LPeg not needed: pure hand-rolled string scanning
-- - 5.3-compatible (no <close>, no continue keyword)
-- - no :match/:gmatch
-- - tab indent, EmmyLua @class/@param annotations
--
-- The orchestrator (ps1_meta.lua) wires this module in via the
-- PASSES table:
-- ["static-analysis"] = {
-- module = "passes.static_analysis",
-- kind = "validation", -- errors stop the build
-- deps = {"word-counts", "components"},
-- out = { { kind = "report",
-- path_template = "<out_root>/<basename>.static_analysis.txt" } },
-- }
--- passes/static_analysis.lua — Per-atom static-analysis checks.
---
--- The 5 checks currently shipped:
--- 1. **GTE pipeline-fill** — every `gte_cmdw_*` invocation must be
--- preceded by the minimum number of `nop` words (per
--- `duffle.duffle.GTE_PIPELINE_LATENCY`) so the COP2 pipeline latency is
--- fully retired before the command issues.
--- 2. **mac_yield uniformity** — every atom body must contain exactly
--- one `mac_yield()` call (control transfer pattern).
--- 3. **ABI handoff** — every `atom_bind(Binds_X)` must reference a
--- `typedef Struct_(Binds_X) { ... }` declaration.
--- 4. **GPU port-store shape** — per-shape (`f3`/`f4`/`g4`/etc.) the
--- sum of `mac_format_X_color` + `mac_gte_store_X_*` +
--- `mac_insert_ot_tag_X` words must equal the GP0 cmd's expected
--- packet size.
--- 5. **per-atom cycle budget** — sum each atom body's instruction
--- latencies (per `duffle.duffle.INSTRUCTION_LATENCY`); report total.
---
--- The orchestrator (`ps1_meta.lua`) wires this module in via the
--- PASSES table:
--- `["static-analysis"] = { module = "passes.static_analysis",
--- kind = "validation",
--- deps = {"word-counts", "components"},
--- out = { { kind = "report",
--- path_template = "<out_root>/<basename>.static_analysis.txt" } } }`
---
--- **Conventions**: tabs (1/level), EmmyLua annotations, no regex,
--- Lua 5.3 compatible. See
--- `C:\projects\Pikuma\ps1-ai\conductor\code_styleguides\lua.md`.
-- ════════════════════════════════════════════════════════════════════════════
-- Module-scope requires + package.path setup
-- ════════════════════════════════════════════════════════════════════════════
local script_path = arg and arg[0] or "?"
local last_sep = 0
for i = 1, #script_path do
local c = script_path:sub(i, i)
if c == "/" or c == "\\" then last_sep = i end
end
local script_dir = last_sep == 0 and "./" or script_path:sub(1, last_sep)
package.path = script_dir .. "../?.lua;" .. script_dir .. "../?/init.lua;" .. script_dir .. "?.lua;" .. package.path
-- `duffle.setup_package_path()` resolves `arg[0]` and prepends `scripts/`
-- (and `scripts/passes/`) to `package.path`, so `require("duffle")`
-- resolves regardless of CWD. See `duffle.lua` for the implementation.
local duffle = require("duffle")
local read_ident = duffle.read_ident
local skip_ws_and_cmt = duffle.skip_ws_and_cmt
local read_parens = duffle.read_parens
local read_braces = duffle.read_braces
local read_brackets = duffle.read_brackets
local scan_to_char = duffle.scan_to_char
local split_top_level_commas = duffle.split_top_level_commas
local trim = duffle.trim
local ensure_dir = duffle.ensure_dir
local write_file = duffle.write_file
local basename_no_ext = duffle.basename_no_ext
-- Bootstrap: see `ps1_meta.lua` for the rationale.
dofile((arg[0]:match("(.*[/\\])") or "./") .. "duffle_paths.lua")
local duffle = require("duffle")
-- Latency table lives in duffle.lua (shared between this pass + future
-- per-atom cycle-budget pass). Lazily read on first use.
local GTE_PIPELINE_LATENCY = duffle.GTE_PIPELINE_LATENCY
-- Domain tables (single source of truth in duffle.lua).
-- GP0 domain tables (Phase 2 check #4). Same source as GTE_PIPELINE_LATENCY.
local GP0_CMD_SIZE = duffle.GP0_CMD_SIZE
local GP0_CMD_BY_SHAPE = duffle.GP0_CMD_BY_SHAPE
local GP0_MACRO_CONTRIB = duffle.GP0_MACRO_CONTRIB
-- Instruction latency table (Phase 3). Per-macro cycle cost in the
-- best-case (no-stall) scenario. Unknown macros default to
-- UNKNOWN_INSTRUCTION_CYCLES with a warning.
local INSTRUCTION_LATENCY = duffle.INSTRUCTION_LATENCY
local UNKNOWN_INSTRUCTION_CYCLES = duffle.UNKNOWN_INSTRUCTION_CYCLES
-- ════════════════════════════════════════════════════════════════════════════
-- Constants
-- ════════════════════════════════════════════════════════════════════════════
-- Atom declaration + component declaration identifiers.
local ATOM_DECL = "MipsAtom_"
local ATOM_COMP = "MipsAtomComp_"
local ATOM_COMP_PROC = "MipsAtomComp_Proc_"
-- Marker-call identifiers inside atom bodies.
local ATOM_LABEL = "atom_label"
local ATOM_OFFSET = "atom_offset"
local ATOM_INFO = "atom_info"
local ATOM_BIND = "atom_bind"
local ATOM_READS = "atom_reads"
local ATOM_WRITES = "atom_writes"
local ATOM_YIELD = "mac_yield"
local WORD_COUNT_PRAGMA = "WORD_COUNT("
-- ASCII byte values used in tokenization.
local BYTE_NEWLINE = 10
local BYTE_HASH = 35 -- '#'
local BYTE_OPEN_PAREN = 40
local BYTE_OPEN_BRACE = 123
local BYTE_OPEN_BRACK = 91
local BYTE_SEMI = 59
-- Per-check output paths (relative to ctx.out_root).
local OUTPUT_EXTENSION = ".static_analysis.txt"
-- ════════════════════════════════════════════════════════════════════════════
-- Type declarations
-- ════════════════════════════════════════════════════════════════════════════
--- @class SourceFile
--- @field path string -- absolute path to the source file
--- @field text string -- the full source text
--- @field dir string -- the directory containing the source
--- @field basename string -- filename without extension
--- @class PassCtx
--- @field sources SourceFile[]
--- @field metadata_path string
--- @field shared table
--- @field shared.word_counts table<string, integer>
--- @field out_root string
--- @field project_root string
--- @field upstream table<string, table>
--- @field flags table
--- @field dry_run boolean
--- @field verbose boolean
--- @class PassResult
--- @field outputs table[]
--- @field errors table[]
--- @field warnings table[]
--- @alias AtomName string -- lower_snake_case atom name
--- @alias MacroName string -- lower_snake_case macro identifier
--- @alias CheckName string -- "gte_pipeline_fill" | "mac_yield_uniformity" | "abi_handoff" | "gpu_port_store_shape" | "per_atom_cycle_budget"
--- @class AtomBody
--- @field line integer -- source line of the atom declaration
--- @field name AtomName -- atom name (e.g. "cube_g4_face")
--- @field body string -- the brace-delimited body (without the braces)
--- @field body_off integer -- char offset of body[1] in source
--- @field kind string -- "atom" | "comp_bare" | "comp_proc"
--- @class Token
--- @field tok string -- the raw token text (trimmed)
--- @field line integer -- source line of the token's start
--- @field ident string|nil -- the leading ident of the token (if any)
--- @field kind string -- "n_words" | "mac_yield" | "gte_cmdw" | "mac_format" | "mac_gte_store" | "mac_insert_ot_tag" | "atom_label" | "atom_offset" | "other"
--- @class Finding
--- @field line integer -- source line of the finding
--- @field atom AtomName -- the atom this finding is for (or "")
--- @field check CheckName -- the check identifier
--- @field kind string -- "error" | "warning" | "info"
--- @field msg string -- the finding message
--- @class AtomAnalysis
--- @field atom AtomBody
--- @field tokens Token[] -- the tokens in the atom body, annotated
--- @field findings Finding[] -- findings for this atom
--- @field total_cycles integer -- sum of token cycle costs (Phase 3)
-- ════════════════════════════════════════════════════════════════════════════
-- Source walkers
@@ -94,18 +165,18 @@ local function find_atom_bodies(source_text)
local len = #source_text
local i = 1
while i <= len do
i = skip_ws_and_cmt(source_text, i); if i > len then break end
i = duffle.skip_ws_and_cmt(source_text, i); if i > len then break end
-- Skip preprocessor directives (#define / #include / #pragma /
-- etc). Otherwise the `#define MipsAtom_(sym) ...` definition
-- in lottes_tape.h gets matched as an atom named "sym" and
-- its `body` swallows the next real atom declaration via
-- scan_to_char("{", ...).
-- duffle.scan_to_char("{", ...).
if source_text:sub(i, i) == "#" then
local j = i
while j <= len and source_text:byte(j) ~= 10 do j = j + 1 end
i = j + 1
else
local ident, after = read_ident(source_text, i)
local ident, after = duffle.read_ident(source_text, i)
if not ident then
i = i + 1
elseif ident == "MipsAtom_"
@@ -119,11 +190,11 @@ local function find_atom_bodies(source_text)
else kind = "comp_proc"
end
local open = skip_ws_and_cmt(source_text, after)
local open = duffle.skip_ws_and_cmt(source_text, after)
if source_text:sub(open, open) ~= "(" then
i = open + 1
else
local inner, after_paren = read_parens(source_text, open)
local inner, after_paren = duffle.read_parens(source_text, open)
if kind == "comp_proc" then
-- MipsAtomComp_Proc_(sym, { body })
@@ -152,11 +223,11 @@ local function find_atom_bodies(source_text)
if depth == 0 then break end
j = j + 1
elseif c == 40 then
local _, a = read_parens(inner, j); j = a
local _, a = duffle.read_parens(inner, j); j = a
elseif c == 91 then
local _, a = read_brackets(inner, j); j = a
local _, a = duffle.read_brackets(inner, j); j = a
elseif c == 34 or c == 39 then
j = duffle.skip_str_or_cmt(inner, j) + 1
j = duffle.duffle.skip_str_or_cmt(inner, j) + 1
else
j = j + 1
end
@@ -197,9 +268,9 @@ local function find_atom_bodies(source_text)
if name == "" then
i = open + 1
else
local brace = scan_to_char(source_text, "{", after_paren)
local brace = duffle.scan_to_char(source_text, "{", after_paren)
if brace then
local body, after_brace = read_braces(source_text, brace)
local body, after_brace = duffle.read_braces(source_text, brace)
local body_off = brace + 1
out[#out + 1] = {
line = line_of(i),
@@ -262,11 +333,11 @@ end
-- the trailing comma, and `nop` becomes its own token. So no special
-- handling is needed here.)
local function nop_word_count(token)
local s = trim(token)
local s = duffle.trim(token)
-- strip trailing comma(s) (defensive against raw text via, but our
-- tokenize_body already strips them; this is a safety net)
s = s:gsub(",$", "")
s = trim(s)
s = duffle.trim(s)
if s == "nop" then return 1 end
if s == "nop2" then return 2 end
return 0
@@ -283,7 +354,7 @@ local function tokenize_body(body)
local rel = 1
while rel <= len do
-- Find next non-whitespace, non-comment start
local ws_end = skip_ws_and_cmt(body, rel)
local ws_end = duffle.skip_ws_and_cmt(body, rel)
if ws_end > rel then
rel = ws_end
end
@@ -299,19 +370,19 @@ local function tokenize_body(body)
if c == 10 then break end -- '\n'
if c == 59 then break end -- ';'
if c == 40 then -- '('
local _, a = read_parens(body, i); i = a
local _, a = duffle.read_parens(body, i); i = a
elseif c == 123 then -- '{'
local _, a = read_braces(body, i); i = a
local _, a = duffle.read_braces(body, i); i = a
elseif c == 91 then -- '['
local _, a = read_brackets(body, i); i = a
local _, a = duffle.read_brackets(body, i); i = a
elseif c == 34 or c == 39 then -- '"' or '\''
i = duffle.skip_str_or_cmt(body, i) + 1
i = duffle.duffle.skip_str_or_cmt(body, i) + 1
else
i = i + 1
end
end
-- Extract token [rel .. i-1]
local tok = trim(body:sub(rel, i - 1))
local tok = duffle.trim(body:sub(rel, i - 1))
if tok ~= "" then
out[#out + 1] = { tok = tok, rel = rel }
end
@@ -319,7 +390,7 @@ local function tokenize_body(body)
if i <= len then
i = i + 1
-- Also skip whitespace before next token
local w = skip_ws_and_cmt(body, i)
local w = duffle.skip_ws_and_cmt(body, i)
if w > i then i = w end
end
rel = i
@@ -333,7 +404,7 @@ end
--- Walk the token list. Whenever we hit a `gte_cmdw_<X>` token, count
--- consecutive nop words starting at the next token. If count < the
--- minimum declared in `GTE_PIPELINE_LATENCY[X]`, record a finding.
--- minimum declared in `duffle.GTE_PIPELINE_LATENCY[X]`, record a finding.
---
--- Aliases (`gte_cmdw_rotate_translate_perspective_single` etc.) are
--- resolved against the lookup table directly; if a macro name is not
@@ -344,7 +415,7 @@ local function check_gte_pipeline_fill(atoms, findings, line_of)
-- count consecutive `nop` words IMMEDIATELY PRECEDING it (the
-- source-level `nop2, gte_cmdw_X` idiom provides the pre-pipeline
-- fill that gte.h's wrapper functions provide internally). If
-- count < `GTE_PIPELINE_LATENCY[X]`, record a finding.
-- count < `duffle.GTE_PIPELINE_LATENCY[X]`, record a finding.
--
-- We count nops going backwards from the cmdw token, stopping at
-- the first non-nop token. Tokens like `mem_share` or `port_write`
@@ -363,7 +434,7 @@ local function check_gte_pipeline_fill(atoms, findings, line_of)
or tok:match("^(gte_cmdw_[%w_]+)%s*$")
if cmdw_full then
local variant = cmdw_full:match("^gte_cmdw_(.+)$")
local need = GTE_PIPELINE_LATENCY[cmdw_full]
local need = duffle.GTE_PIPELINE_LATENCY[cmdw_full]
if need == nil then
-- alias or new gte_cmdw_<X> not yet in latency table
local line = a.line + line_in_body[tokens[ti].rel]
@@ -373,7 +444,7 @@ local function check_gte_pipeline_fill(atoms, findings, line_of)
check = "gte_pipeline_fill",
kind = "warning",
msg = string.format(
"%s at line %d uses `gte_cmdw_%s` but that macro is not in duffle.GTE_PIPELINE_LATENCY -- add a min_nops entry",
"%s at line %d uses `gte_cmdw_%s` but that macro is not in duffle.duffle.GTE_PIPELINE_LATENCY -- add a min_nops entry",
a.name, line, variant),
}
ti = ti + 1
@@ -539,42 +610,42 @@ local function find_binds_structs(source_text)
local len = #source_text
local i = 1
while i <= len do
i = skip_ws_and_cmt(source_text, i); if i > len then break end
i = duffle.skip_ws_and_cmt(source_text, i); if i > len then break end
if source_text:sub(i, i) == "#" then
local j = i
while j <= len and source_text:byte(j) ~= 10 do j = j + 1 end
i = j + 1
else
local ident, after = read_ident(source_text, i)
local ident, after = duffle.read_ident(source_text, i)
if not ident then
i = i + 1
elseif ident == "typedef" then
local j = skip_ws_and_cmt(source_text, after)
local id2, after2 = read_ident(source_text, j)
local j = duffle.skip_ws_and_cmt(source_text, after)
local id2, after2 = duffle.read_ident(source_text, j)
if id2 ~= "Struct_" then
i = after2 or (j + 1)
else
local open = skip_ws_and_cmt(source_text, after2)
local open = duffle.skip_ws_and_cmt(source_text, after2)
if source_text:sub(open, open) ~= "(" then
i = open + 1
else
local inner, after_paren = read_parens(source_text, open)
local name = trim(inner)
local brace = scan_to_char(source_text, "{", after_paren)
local inner, after_paren = duffle.read_parens(source_text, open)
local name = duffle.trim(inner)
local brace = duffle.scan_to_char(source_text, "{", after_paren)
if not brace then
i = open + 1
else
local body, after_brace = read_braces(source_text, brace)
local body, after_brace = duffle.read_braces(source_text, brace)
local fields = {}
local byte_off = 0
local k = 1
while k <= #body do
k = skip_ws_and_cmt(body, k); if k > #body then break end
local tid, tafter = read_ident(body, k)
k = duffle.skip_ws_and_cmt(body, k); if k > #body then break end
local tid, tafter = duffle.read_ident(body, k)
if not tid then
k = k + 1
elseif tid == "U4" then
local fid, fafter = read_ident(body, skip_ws_and_cmt(body, tafter))
local fid, fafter = duffle.read_ident(body, duffle.skip_ws_and_cmt(body, tafter))
if fid then
fields[#fields + 1] = { name = fid, offset = byte_off }
byte_off = byte_off + 4
@@ -610,60 +681,60 @@ local function find_atom_info(source_text)
local len = #source_text
local i = 1
while i <= len do
i = skip_ws_and_cmt(source_text, i); if i > len then break end
i = duffle.skip_ws_and_cmt(source_text, i); if i > len then break end
if source_text:sub(i, i) == "#" then
local j = i
while j <= len and source_text:byte(j) ~= 10 do j = j + 1 end
i = j + 1
else
local ident, after = read_ident(source_text, i)
local ident, after = duffle.read_ident(source_text, i)
if not ident then
i = i + 1
elseif ident == "MipsAtom_" then
local open = skip_ws_and_cmt(source_text, after)
local open = duffle.skip_ws_and_cmt(source_text, after)
if source_text:sub(open, open) ~= "(" then
i = open + 1
else
local inner, after_paren = read_parens(source_text, open)
local inner, after_paren = duffle.read_parens(source_text, open)
local a = 1
while a <= #inner and inner:sub(a, a):match("[%s]") do a = a + 1 end
local b = a
while b <= #inner and inner:sub(b, b):match("[%w_]") do b = b + 1 end
local atom_name = inner:sub(a, b - 1)
local lookahead = skip_ws_and_cmt(source_text, after_paren)
local look_ident, look_after = read_ident(source_text, lookahead)
local lookahead = duffle.skip_ws_and_cmt(source_text, after_paren)
local look_ident, look_after = duffle.read_ident(source_text, lookahead)
if look_ident == "atom_info" then
local info_open = skip_ws_and_cmt(source_text, look_after)
local info_open = duffle.skip_ws_and_cmt(source_text, look_after)
if source_text:sub(info_open, info_open) == "(" then
local info_inner, info_after = read_parens(source_text, info_open)
local info_inner, info_after = duffle.read_parens(source_text, info_open)
local binds, reads, writes = nil, nil, nil
local j = 1
while j <= #info_inner do
j = skip_ws_and_cmt(info_inner, j); if j > #info_inner then break end
local sub_ident, sub_after = read_ident(info_inner, j)
j = duffle.skip_ws_and_cmt(info_inner, j); if j > #info_inner then break end
local sub_ident, sub_after = duffle.read_ident(info_inner, j)
if not sub_ident then
j = j + 1
elseif sub_ident == "atom_bind" then
local sub_open = skip_ws_and_cmt(info_inner, sub_after)
local sub_open = duffle.skip_ws_and_cmt(info_inner, sub_after)
if info_inner:sub(sub_open, sub_open) == "(" then
local sub_inner, sub_after2 = read_parens(info_inner, sub_open)
binds = trim(sub_inner)
local sub_inner, sub_after2 = duffle.read_parens(info_inner, sub_open)
binds = duffle.trim(sub_inner)
j = sub_after2
else
j = sub_open + 1
end
elseif sub_ident == "atom_reads" or sub_ident == "atom_writes" then
local kind = sub_ident
local sub_open = skip_ws_and_cmt(info_inner, sub_after)
local sub_open = duffle.skip_ws_and_cmt(info_inner, sub_after)
if info_inner:sub(sub_open, sub_open) == "(" then
local sub_inner, sub_after2 = read_parens(info_inner, sub_open)
local sub_inner, sub_after2 = duffle.read_parens(info_inner, sub_open)
local regs = {}
local p = 1
while p <= #sub_inner do
p = skip_ws_and_cmt(sub_inner, p); if p > #sub_inner then break end
local pid, pa = read_ident(sub_inner, p)
p = duffle.skip_ws_and_cmt(sub_inner, p); if p > #sub_inner then break end
local pid, pa = duffle.read_ident(sub_inner, p)
if pid then
regs[#regs + 1] = trim(pid)
regs[#regs + 1] = duffle.trim(pid)
p = pa
else
p = p + 1
@@ -816,14 +887,14 @@ end
--- For every baked atom body, detect which GP0 primitive it's emitting
--- (first `mac_format_<shape>_color` call). Sum contributions from
--- `mac_format_X_color` + `mac_gte_store_X_post_*` + `mac_insert_ot_tag_X`.
--- Compare to GP0_CMD_SIZE[cmd_byte]. Mismatch = error.
--- Compare to duffle.GP0_CMD_SIZE[cmd_byte]. Mismatch = error.
---
--- Soft behavior (warnings):
--- - Atoms emitting a primitive via raw `store_word(R_PrimCursor, ...)`
--- (no `mac_format_X_color` call) emit a "manual packet assembly"
--- advisory. Cannot auto-validate.
--- - Atoms containing a `mac_<name>(...)` call whose name is not in
--- GP0_MACRO_CONTRIB emit a "new macro; update GP0_MACRO_CONTRIB"
--- duffle.GP0_MACRO_CONTRIB emit a "new macro; update duffle.GP0_MACRO_CONTRIB"
--- advisory.
---
--- Applies only to `kind = "atom"` (baked atoms). Components don't
@@ -844,24 +915,24 @@ local function check_gpu_portstore_shape(atoms, findings)
-- to get the bare shape suffix (f3 / g4 / etc).
local shape = tok:match("^mac_format_([%w_]+)_color%s*%(")
or tok:match("^mac_format_([%w_]+)_color%s*$")
if shape and GP0_CMD_BY_SHAPE[shape] then
if shape and duffle.GP0_CMD_BY_SHAPE[shape] then
if not cmd_byte then
cmd_byte = GP0_CMD_BY_SHAPE[shape]
cmd_byte = duffle.GP0_CMD_BY_SHAPE[shape]
cmd_line = a.line + line_in_body[t.rel]
end
saw_format = true
local contrib_key = "mac_format_" .. shape .. "_color"
local n = GP0_MACRO_CONTRIB[contrib_key]
local n = duffle.GP0_MACRO_CONTRIB[contrib_key]
if n then contrib = contrib + n end
end
local gte_store = tok:match("^mac_gte_store_[%w_]+")
if gte_store then
local n = GP0_MACRO_CONTRIB[gte_store]
local n = duffle.GP0_MACRO_CONTRIB[gte_store]
if n then contrib = contrib + n end
end
local ot_tag = tok:match("^mac_insert_ot_tag_([%w_]+)")
if ot_tag then
local n = GP0_MACRO_CONTRIB["mac_insert_ot_tag_" .. ot_tag]
local n = duffle.GP0_MACRO_CONTRIB["mac_insert_ot_tag_" .. ot_tag]
if n then contrib = contrib + n end
end
if tok:match("^store_word%s*%(") and tok:find("R_PrimCursor", 1, true) then
@@ -879,7 +950,7 @@ local function check_gpu_portstore_shape(atoms, findings)
}
end
else
local expected = GP0_CMD_SIZE[cmd_byte]
local expected = duffle.GP0_CMD_SIZE[cmd_byte]
if contrib ~= expected then
findings[#findings + 1] = {
atom = a.name, line = cmd_line or a.line,
@@ -899,20 +970,20 @@ end
--- Compute the cycle cost of one token. The token is a string like
--- `add_ui(R_T0, R_T1, 4)` or `nop2` or `gte_cmdw_rtpt`. Returns:
--- cycles - integer cycle cost (from INSTRUCTION_LATENCY, or
--- UNKNOWN_INSTRUCTION_CYCLES if not in the table)
--- cycles - integer cycle cost (from duffle.INSTRUCTION_LATENCY, or
--- duffle.UNKNOWN_INSTRUCTION_CYCLES if not in the table)
--- macro_name - the bare ident (e.g. `add_ui`, `gte_cmdw_rtpt`,
--- `nop2`, `mac_yield`)
--- unknown - true iff the macro wasn't in INSTRUCTION_LATENCY
--- unknown - true iff the macro wasn't in duffle.INSTRUCTION_LATENCY
--- The function strips trailing `()` from function-call style macros
--- so `mac_yield()` and `mac_yield` resolve identically.
local function token_cycles(tok)
-- Extract the leading ident. Tolerate `(...)` args.
local ident = tok:match("^([%w_]+)")
if not ident then return UNKNOWN_INSTRUCTION_CYCLES, "?", true end
local cost = INSTRUCTION_LATENCY[ident]
if not ident then return duffle.UNKNOWN_INSTRUCTION_CYCLES, "?", true end
local cost = duffle.INSTRUCTION_LATENCY[ident]
if cost == nil then
return UNKNOWN_INSTRUCTION_CYCLES, ident, true
return duffle.UNKNOWN_INSTRUCTION_CYCLES, ident, true
end
return cost, ident, false
end
@@ -968,7 +1039,7 @@ end
--- mac_yield or end-of-body)
--- has_loops - true iff a path re-entered a token it had visited
--- (warning; loop bodies aren't supported)
--- unknown_macros - list of unique macro names not in INSTRUCTION_LATENCY
--- unknown_macros - list of unique macro names not in duffle.INSTRUCTION_LATENCY
--- cycles_full - sum of ALL token costs (the previous "best case"
--- value; included for backward-compat; double-counts
--- the BD-slot nop relative to cycles_min/max)
@@ -1155,7 +1226,7 @@ end
--- Per-source check that emits one finding per unknown macro seen
--- (deduplicated across atoms so the warning section doesn't get
--- spammed with N copies of "macro X not in INSTRUCTION_LATENCY").
--- spammed with N copies of "macro X not in duffle.INSTRUCTION_LATENCY").
local function check_per_atom_cycle_budget(atoms, findings)
local unknown_seen = {}
for _, a in ipairs(atoms) do
@@ -1166,8 +1237,8 @@ local function check_per_atom_cycle_budget(atoms, findings)
findings[#findings + 1] = {
atom = a.name, line = a.line,
check = "per_atom_cycle_budget", kind = "warning",
msg = string.format("%s at line %d uses macro `%s` which is not in INSTRUCTION_LATENCY; cycle count will be +%d per call (best-case). Add an entry to duffle.M.INSTRUCTION_LATENCY.",
a.name, a.line, name, UNKNOWN_INSTRUCTION_CYCLES),
msg = string.format("%s at line %d uses macro `%s` which is not in duffle.INSTRUCTION_LATENCY; cycle count will be +%d per call (best-case). Add an entry to duffle.M.duffle.INSTRUCTION_LATENCY.",
a.name, a.line, name, duffle.UNKNOWN_INSTRUCTION_CYCLES),
}
end
end
@@ -1284,7 +1355,7 @@ end
local function emit_static_analysis_txt(ctx, src, result)
local out_path = ctx.out_root .. "/" .. src.basename .. ".static_analysis.txt"
if ctx.dry_run then return out_path end
ensure_dir(ctx.out_root)
duffle.ensure_dir(ctx.out_root)
local lines = {}
local function add(s) lines[#lines + 1] = s end
@@ -1352,7 +1423,7 @@ local function emit_static_analysis_txt(ctx, src, result)
add(string.format(" %s", i_.msg))
end
write_file(out_path, table.concat(lines, "\n") .. "\n")
duffle.write_file(out_path, table.concat(lines, "\n") .. "\n")
return out_path
end
@@ -1370,7 +1441,7 @@ local function emit_module_static_analysis_txt(ctx, dir, dir_sources, atoms, fin
local dir_basename = dir:match("([^/\\]+)$") or dir
local out_path = ctx.out_root .. "/" .. dir_basename .. ".static_analysis.txt"
if ctx.dry_run then return out_path end
ensure_dir(ctx.out_root)
duffle.ensure_dir(ctx.out_root)
local lines = {}
local function add(s) lines[#lines + 1] = s end
@@ -1545,7 +1616,7 @@ local function emit_module_static_analysis_txt(ctx, dir, dir_sources, atoms, fin
end
end
write_file(out_path, table.concat(lines, "\n") .. "\n")
duffle.write_file(out_path, table.concat(lines, "\n") .. "\n")
return out_path
end
scripts/passes/word_count_eval.lua
+251
View File
@@ -0,0 +1,251 @@
--- word_count_eval.lua — Word-counting logic for the tape-atom metaprogram
--- pipeline.
---
--- Three responsibilities:
--- 1. **Public utilities** (used by `passes/components.lua`,
--- `passes/offsets.lua`, `passes/annotation.lua`):
--- - `M.count_token_words(token, wc)` — words emitted by one token
--- - `M.scan_dir(dir, suffix)` — glob walk for *.macs.h
--- - `M.count_body_words(body, wc)` — words emitted by an atom body
--- 2. **Pass entry** `M.run(ctx)` — loads metadata.h + *.macs.h into
--- `ctx.shared.word_counts` for downstream passes.
--- 3. **Internal helpers** for the body scanner.
---
--- **Conventions**: tabs (1/level), EmmyLua annotations, no regex,
--- Lua 5.3 compatible. See
--- `C:\projects\Pikuma\ps1-ai\conductor\code_styleguides\lua.md`.
-- ════════════════════════════════════════════════════════════════════════════
-- Module-scope requires + package.path setup
-- ════════════════════════════════════════════════════════════════════════════
-- Resolve `arg[0]` to an absolute-ish script directory so that
-- `require("duffle")` resolves against `scripts/` regardless of CWD.
-- Note: this boilerplate is duplicated in 6 other entry scripts; a
-- Phase-6 extraction target (`duffle.setup_package_path()`).
-- Bootstrap: see `ps1_meta.lua` for the rationale.
dofile((arg[0]:match("(.*[/\\])") or "./") .. "duffle_paths.lua")
local duffle = require("duffle")
-- ════════════════════════════════════════════════════════════════════════════
-- Constants
-- ════════════════════════════════════════════════════════════════════════════
-- Windows separator chars — used to convert `dir /b /s` output (which uses
-- `\`) into POSIX paths (which our scripts expect).
local PATH_SEP_BACKSLASH = "\\"
local PATH_SEP_FORWARD = "/"
-- Glob command for Windows directory walk. `dir /b /s` lists all matching
-- files recursively with bare paths (no headers); `2>nul` discards the
-- "file not found" stderr when nothing matches.
local DIR_GLOB_CMD = 'dir /b /s "%s\\%s" 2>nul'
-- ════════════════════════════════════════════════════════════════════════════
-- Type declarations
-- ════════════════════════════════════════════════════════════════════════════
--- @class WordCounts
--- @field [string] integer -- macro name -> word count
--- @class SourceFile
--- @field path string -- absolute path to the source file
--- @field text string -- the full source text
--- @field dir string -- the directory containing the source
--- @field basename string -- filename without extension
--- @class PassCtx
--- @field sources SourceFile[] -- all source files in the build
--- @field metadata_path string -- path to word_count.metadata.h
--- @field shared table -- cross-pass shared state
--- @field shared.word_counts WordCounts -- populated by this pass
--- @field out_root string -- output root (e.g. "build/gen")
--- @field project_root string -- project root (e.g. "code/")
--- @field upstream table<string, table> -- per-pass upstream outputs
--- @field flags table -- CLI flags
--- @field dry_run boolean -- if true, compute but don't write
--- @field verbose boolean -- if true, log diagnostic info
--- @class PassResult
--- @field outputs table[] -- {kind=, path=} entries describing emit files
--- @field errors table[] -- {line=, msg=} entries; build-stops
--- @field warnings table[] -- {line=, msg=} entries; build-succeeds
-- ════════════════════════════════════════════════════════════════════════════
-- Module exports
-- ════════════════════════════════════════════════════════════════════════════
local M = {}
-- ┌────────────────────────────────────────────────────────────────────┐
-- │ Shared utility: count_token_words │
-- └────────────────────────────────────────────────────────────────────┘
--- Count words emitted by a single comma-separated token inside an atom body.
--- For most tokens (regular MIPS instructions) this returns 1.
--- For `mac_X(...)` calls, this returns the resolved word count from `wc`
--- (recursively if needed). For `nop2` etc., returns wc[name].
--- For unknown macros, returns 1 and (optionally) warns.
---
--- @param token string -- a single token from split_top_level_commas
--- @param wc WordCounts -- the shared word-count table
--- @return integer
function M.count_token_words(token, wc)
local s = duffle.trim(token)
if s == "" then return 0 end
local name, after = duffle.read_ident(s, 1)
if not name then return 1 end
if wc[name] then return wc[name] end
local paren_pos = duffle.skip_ws_and_cmt(s, after)
if s:sub(paren_pos, paren_pos) == "(" then
io.stderr:write(" warning: unknown macro '" .. name .. "', assuming 1 word\n")
end
return 1
end
-- ┌────────────────────────────────────────────────────────────────────┐
-- │ Shared utility: scan_dir │
-- └────────────────────────────────────────────────────────────────────┘
--- Recursively scan a directory for files matching a glob suffix.
--- No regex per the no_regex constraint — uses plain byte matching
--- via `dir /b /s` on Windows.
---
--- @param dir string -- directory to scan (absolute or relative)
--- @param suffix string -- file pattern, e.g. "*.macs.h"
--- @return string[]
function M.scan_dir(dir, suffix)
local results = {}
local pipe = io.popen(DIR_GLOB_CMD:format(dir, suffix))
if not pipe then return results end
for raw_line in pipe:lines() do
local path = raw_line:gsub(PATH_SEP_BACKSLASH, PATH_SEP_FORWARD)
results[#results + 1] = path
end
pipe:close()
return results
end
-- ┌────────────────────────────────────────────────────────────────────┐
-- │ Shared utility: count_body_words │
-- └────────────────────────────────────────────────────────────────────┘
--- Count words emitted by an entire atom body (a brace-delimited block).
--- Splits by top-level commas; for each token, delegates to count_token_words.
--- Handles `atom_label(name)` / `atom_offset(tag, name)` markers (record at
--- current pos, do NOT advance pos; if the marker call bundles an instruction
--- after it, count that instruction too).
---
--- @param body string -- brace-delimited atom body (without braces)
--- @param wc WordCounts -- the shared word-count table
--- @return integer -- total words
function M.count_body_words(body, wc)
local total = 0
for _, tok in ipairs(duffle.split_top_level_commas(body)) do
local pos = 1
local tok_len = #tok
while pos <= tok_len and duffle.is_space(tok:sub(pos, pos)) do
pos = pos + 1
end
local leading_ident = duffle.read_ident(tok, pos)
local is_marker = leading_ident == "atom_label" or leading_ident == "atom_offset"
if is_marker then
-- Marker call: record at current pos, do NOT advance pos.
-- But the source pattern may bundle the marker with the next
-- instruction on a new line (no top-level comma between them).
-- In that case, the rest of `tok` after the marker call is
-- a real instruction that must still be counted.
local marker_end = M.find_marker_call_end(tok)
if marker_end > 0 and marker_end < #tok then
local rest = duffle.trim(tok:sub(marker_end + 1))
if rest ~= "" then
total = total + M.count_token_words(rest, wc)
end
end
else
total = total + M.count_token_words(tok, wc)
end
end
return total
end
--- Find the end position (just past the closing ')') of the first
--- atom_label/atom_offset call in `tok`. Returns 0 if no such call.
--- Internal helper for count_body_words.
---
--- @param tok string
--- @return integer -- 0 if no marker call found
function M.find_marker_call_end(tok)
local pos = 1
local tok_len = #tok
while pos <= tok_len do
pos = duffle.skip_ws_and_cmt(tok, pos)
if pos > tok_len then break end
local ch = tok:sub(pos, pos)
if duffle.is_space(ch) then
pos = pos + 1
elseif ch == "/" then
-- comment — skip past it (delegated to duffle.skip_str_or_cmt)
local nx = duffle.skip_str_or_cmt(tok, pos)
pos = (nx > pos) and nx or (pos + 1)
else
local ident, after_ident = duffle.read_ident(tok, pos)
local marker_end = find_marker_end(tok, ident, after_ident)
if marker_end > 0 then return marker_end end
pos = after_ident or (pos + 1)
end
end
return 0
end
-- (internal) If `ident` is `atom_label`/`atom_offset` followed by `(...)`,
-- return the position just past the closing ')'. Otherwise 0.
-- @param tok string
-- @param ident string|nil
-- @param after_ident integer
-- @return integer
local function find_marker_end(tok, ident, after_ident)
if ident ~= "atom_label" and ident ~= "atom_offset" then return 0 end
local open_paren = duffle.skip_ws_and_cmt(tok, after_ident)
if tok:sub(open_paren, open_paren) ~= "(" then return 0 end
local _, end_paren = duffle.read_parens(tok, open_paren)
return end_paren - 1
end
-- ┌────────────────────────────────────────────────────────────────────┐
-- │ Pass entry: M.run(ctx) — "word-counts" pass │
-- └────────────────────────────────────────────────────────────────────┘
--- Load metadata.h + scan for existing *.macs.h files into
--- ctx.shared.word_counts. Loading the .macs.h files is idempotent:
--- entries from later (current-build) .macs.h files override
--- metadata.h entries of the same name.
---
--- @param ctx PassCtx
--- @return PassResult
function M.run(ctx)
local wc = {}
-- 1. Load metadata.h (the encoding-macro source of truth).
local meta_counts = duffle.load_word_counts(ctx.metadata_path)
for name, count in pairs(meta_counts) do wc[name] = count end
-- 2. Scan project_root recursively for *.macs.h files (component-macro source).
local macs_files = M.scan_dir(ctx.project_root, "*.macs.h")
for _, macs_path in ipairs(macs_files) do
local ok, mc = pcall(duffle.load_word_counts, macs_path)
if not ok then
io.stderr:write(string.format("[word_count_eval] parse error in '%s': %s\n", macs_path, tostring(mc)))
elseif type(mc) ~= "table" then
io.stderr:write(string.format("[word_count_eval] '%s' did not return a table (got %s)\n", macs_path, type(mc)))
else
for name, count in pairs(mc) do wc[name] = count end
end
end
ctx.shared.word_counts = wc
return { outputs = {}, errors = {}, warnings = {} }
end
return M