mostly comment review (lua metaprogram)

This commit is contained in:
ed
2026-07-11 01:47:38 -04:00
parent a0d22700db
commit 91a91b3495
9 changed files with 386 additions and 557 deletions
+62 -103
View File
@@ -138,16 +138,13 @@ end
-- Section 0: LPeg patterns (compiled once at module load) -- Section 0: LPeg patterns (compiled once at module load)
-- ════════════════════════════════════════════════════════════════════════════ -- ════════════════════════════════════════════════════════════════════════════
-- --
-- LPeg is a required dependency (PEG library, no regex). It's loaded -- LPeg is a required dependency (PEG library, no regex).
-- via `package.cpath` (configured by `duffle_paths.lua` to find -- It's loaded via `package.cpath` (configured by `duffle_paths.lua` to find `toolchain/lpeg/lpeg.dll`).
-- `toolchain/lpeg/lpeg.dll`). There's no hand-rolled fallback — the -- There's no hand-rolled fallback. The original two-tier design added complexity for a 5-10x speedup that's
-- original two-tier design added complexity for a 5-10x speedup that's -- only relevant at the high-level scanner stage; the byte-by-byte helpers in Section 1 are sufficient for the classification primitives.
-- only relevant at the high-level scanner stage; the byte-by-byte
-- helpers in Section 1 are sufficient for the classification primitives.
-- --
-- If the require fails, fail loud with an actionable message (per -- If the require fails, fail loud with an actionable message. The build script (`update_deps.ps1`) builds lpeg.dll into `toolchain/lpeg/`;
-- lua.md §9). The build script (`update_deps.ps1`) builds lpeg.dll -- if it's missing, run `update_deps.ps1`.
-- into `toolchain/lpeg/`; if it's missing, run `update_deps.ps1`.
local lpeg_ok, lpeg = pcall(require, "lpeg") local lpeg_ok, lpeg = pcall(require, "lpeg")
if not lpeg_ok then if not lpeg_ok then
io.stderr:write("[duffle] require('lpeg') failed: ", lpeg, "\n") io.stderr:write("[duffle] require('lpeg') failed: ", lpeg, "\n")
@@ -195,8 +192,7 @@ local lpeg_scan_to_target_pat = function(target) return (P(1) - P(target))^0 en
-- is_space(c), is_alpha(c), etc. — accept a single-char STRING (legacy) -- is_space(c), is_alpha(c), etc. — accept a single-char STRING (legacy)
-- is_space_byte(b), is_alpha_byte(b), etc. — accept a single-byte INTEGER -- is_space_byte(b), is_alpha_byte(b), etc. — accept a single-byte INTEGER
-- --
-- The byte-based versions are 5-10x faster in tight loops because they -- The byte-based versions are 5-10x faster in tight loops because they avoid the string allocation per s:sub(pos, pos) call.
-- avoid the string allocation per s:sub(pos, pos) call.
-- Whitespace characters per C locale. -- Whitespace characters per C locale.
function M.is_space_byte(b) return b == BYTE_SPACE or b == BYTE_TAB or b == BYTE_NEWLINE or b == BYTE_CR or b == BYTE_VT or b == BYTE_FF end function M.is_space_byte(b) return b == BYTE_SPACE or b == BYTE_TAB or b == BYTE_NEWLINE or b == BYTE_CR or b == BYTE_VT or b == BYTE_FF end
@@ -301,11 +297,10 @@ function M.write_file(path, content)
f:write(content); f:close() f:write(content); f:close()
end end
-- Cache of directories already verified to exist in this process. Each -- Cache of directories already verified to exist in this process.
-- ensure_dir() call may otherwise spawn a `cmd.exe mkdir` (50-100ms -- Each ensure_dir() call may otherwise spawn a `cmd.exe mkdir` (50-100ms per call on Windows) — calling it inside per-source loops added 1.5+
-- per call on Windows) — calling it inside per-source loops added 1.5+ -- seconds to the report pass. Cache makes ensure_dir idempotent within the process lifetime.
-- seconds to the report pass. Cache makes ensure_dir idempotent within -- (safe across passes; the dir state doesn't change).
-- the process lifetime (safe across passes; the dir state doesn't change).
local _ensured_dirs = {} local _ensured_dirs = {}
function M.ensure_dir(path) function M.ensure_dir(path)
@@ -324,8 +319,7 @@ function M._reset_ensured_dirs() _ensured_dirs = {} end
-- ════════════════════════════════════════════════════════════════════════════ -- ════════════════════════════════════════════════════════════════════════════
-- Skip a string or C-style comment starting at position `pos`. -- Skip a string or C-style comment starting at position `pos`.
-- Returns the position just past the construct, or `pos` unchanged if -- Returns the position just past the construct, or `pos` unchanged if no string/comment starts there. LPeg-backed.
-- no string/comment starts there. LPeg-backed.
function M.skip_str_or_cmt(s, pos) function M.skip_str_or_cmt(s, pos)
return lpeg.match(lpeg_str_or_cmt_pat, s, pos) or pos return lpeg.match(lpeg_str_or_cmt_pat, s, pos) or pos
end end
@@ -336,21 +330,17 @@ function M.skip_ws_and_cmt(s, pos)
return lpeg.match(lpeg_ws_and_cmt_pat, s, pos) or pos return lpeg.match(lpeg_ws_and_cmt_pat, s, pos) or pos
end end
-- Read a C-style identifier (alpha followed by zero+ alnum) starting at -- Read a C-style identifier (alpha followed by zero+ alnum) starting at position `pos`.
-- position `pos`. Returns the identifier string + the position just past -- Returns the identifier string + the position just past it, or nil + pos if no identifier starts here. LPeg-backed.
-- it, or nil + pos if no identifier starts here. LPeg-backed.
function M.read_ident(s, pos) function M.read_ident(s, pos)
local result = lpeg.match(lpeg_ident_pat, s, pos) local result = lpeg.match(lpeg_ident_pat, s, pos)
if result then return result, pos + #result end if result then return result, pos + #result end
return nil, pos return nil, pos
end end
-- Read a balanced-delimited group (parens, braces, or brackets) starting -- Read a balanced-delimited group (parens, braces, or brackets) starting at position `pos`.
-- at position `pos`. Returns the inner content (between the delimiters) + -- Returns the inner content (between the delimiters) + the position
-- the position just past the closing delimiter, or nil + pos if `s[pos]` -- just past the closing delimiter, or nil + pos if `s[pos]` isn't `open_char`.
-- isn't `open_char`.
--
-- (Hand-rolled; the depth counting makes pure LPeg awkward here.)
function M.read_balanced(s, open_char, close_char, pos) function M.read_balanced(s, open_char, close_char, pos)
local open_byte = open_char:byte() local open_byte = open_char:byte()
if s:byte(pos) ~= open_byte then return nil, pos end if s:byte(pos) ~= open_byte then return nil, pos end
@@ -380,8 +370,8 @@ M.read_parens = function(s, pos) return M.read_balanced(s, "(", ")", pos) end
M.read_braces = function(s, pos) return M.read_balanced(s, "{", "}", pos) end M.read_braces = function(s, pos) return M.read_balanced(s, "{", "}", pos) end
M.read_brackets = function(s, pos) return M.read_balanced(s, "[", "]", pos) end M.read_brackets = function(s, pos) return M.read_balanced(s, "[", "]", pos) end
-- Scan forward from position `start` until we find a specific single byte -- Scan forward from position `start` until we find a specific single byte `target`,
-- `target`, transparently stepping over balanced parens/braces/brackets. -- transparently stepping over balanced parens/braces/brackets.
-- Returns the position of `target`, or nil if not found. -- Returns the position of `target`, or nil if not found.
function M.scan_to_char(s, target, start) function M.scan_to_char(s, target, start)
local target_byte = target:byte() local target_byte = target:byte()
@@ -403,23 +393,18 @@ end
-- Split a brace-body into top-level comma-separated tokens. Honors nested -- Split a brace-body into top-level comma-separated tokens. Honors nested
-- parens/braces/brackets and skips strings/comments. -- parens/braces/brackets and skips strings/comments.
-- --
-- FIX (2026-07-09): split at top-level NEWLINES and SEMICOLONS too, AND -- FIX (2026-07-09): split at top-level NEWLINES and SEMICOLONS too, AND emit a token break after a top-level comment/string.
-- emit a token break after a top-level comment/string. Previous behavior -- Previous behavior glued the macro call after a comment into the same token, so `word_count_of_token` only saw the
-- glued the macro call after a comment into the same token, so -- leading ident (often nil after stripping the comment), undercounting the body. See Phase 1 of the branch-offset regression investigation.
-- `word_count_of_token` only saw the leading ident (often nil after -- Pure-comment / pure-string chunks (which now appear between real statements) are filtered out so they contribute 0 words instead of 1.
-- stripping the comment), undercounting the body. See Phase 1 of the
-- branch-offset regression investigation. Pure-comment / pure-string
-- chunks (which now appear between real statements) are filtered out so
-- they contribute 0 words instead of 1.
function M.split_top_level_commas(body) function M.split_top_level_commas(body)
local tokens = {} local tokens = {}
local pos = 1 local pos = 1
local body_len = #body local body_len = #body
local token_start = 1 local token_start = 1
-- True iff `chunk` contains any non-whitespace, non-comment, non-string -- True iff `chunk` contains any non-whitespace, non-comment, non-string content
-- content (i.e., real token material). Walks through ws + comments -- (i.e., real token material). Walks through ws + comments individually so a chunk like " /* trailing */ shift_lleft(...)"
-- individually so a chunk like " /* trailing */ shift_lleft(...)"
-- is correctly classified as having real content (the macro call). -- is correctly classified as having real content (the macro call).
local function has_real_content(chunk) local function has_real_content(chunk)
local scan = 1 local scan = 1
@@ -449,12 +434,11 @@ function M.split_top_level_commas(body)
-- Pure comment/string chunk at top level (no preceding instruction content within this chunk). -- Pure comment/string chunk at top level (no preceding instruction content within this chunk).
-- APPEND it to the LAST token so emit-context callers (components.lua build_component_lines) -- APPEND it to the LAST token so emit-context callers (components.lua build_component_lines)
-- can convert `// trailing comment` to `/* */` and emit it with the macro body. -- can convert `// trailing comment` to `/* */` and emit it with the macro body.
-- For word counting, count_token_words only inspects the leading ident, so a trailing comment doesn't -- For word counting, count_token_words only inspects the leading ident, so a trailing comment doesn't affect the count.
-- affect the count.
-- --
-- This is the second-half fix to commit 98e27c2: the first fix correctly broke top-level comments -- This is the second-half fix to commit 98e27c2: the first fix correctly broke top-level comments
-- off from the NEXT statement (fixing macro-call word counts); -- off from the NEXT statement (fixing macro-call word counts);
-- this fix preserves them on the PREVIOUS statement (restoring the comments in the emitted .macs.h output). -- This fix preserves them on the PREVIOUS statement (restoring the comments in the emitted .macs.h output).
tokens[#tokens] = tokens[#tokens] .. chunk tokens[#tokens] = tokens[#tokens] .. chunk
end end
end end
@@ -577,34 +561,23 @@ M.TAPE_ATOM_MACROS = {
-- GTE pipeline-fill latency table (static-analysis Phase 1). -- GTE pipeline-fill latency table (static-analysis Phase 1).
-- --
-- For each `gte_cmdw_*` macro in code/duffle/gte.h, the minimum number -- For each `gte_cmdw_*` macro in code/duffle/gte.h, the minimum number of consecutive COP2 "nop" words that MUST appear
-- of consecutive COP2 "nop" words that MUST appear before any other -- before any other COP2 read or non-nop instruction (so the GTE pipeline latency is fully retired).
-- COP2 read or non-nop instruction (so the GTE pipeline latency is -- Latencies are sourced from the doxygen comments in gte.h
-- fully retired). Latencies are sourced from the doxygen comments -- (e.g. `* @brief Rotate, Translate and Perspective Triple (23 cycles)` with body `Two nop words fill the COP2 pipeline latency`).
-- in gte.h (e.g. `* @brief Rotate, Translate and Perspective Triple
-- (23 cycles)` with body `Two nop words fill the COP2 pipeline
-- latency`).
-- --
-- The check (`scripts/passes/static_analysis.lua :: -- The check (`scripts/passes/static_analysis.lua :: check_gte_pipeline_fill`) walks each atom body,
-- check_gte_pipeline_fill`) walks each atom body, counts the -- counts the consecutive nop words after every `gte_cmdw_*` invocation, and reports a finding if the count is below this minimum.
-- consecutive nop words after every `gte_cmdw_*` invocation, and -- Aliases are dereferenced before lookup (gté_cmdw_rtps_alias -> gte_cmdw_rtps -> 2).
-- reports a finding if the count is below this minimum. Aliases
-- are dereferenced before lookup (gté_cmdw_rtps_alias ->
-- gte_cmdw_rtps -> 2).
-- --
-- Values verified against PSX-SPX gte.txt (rtpt 23cy / 8cy per divide -- Values verified against PSX-SPX gte.txt (rtpt 23cy / 8cy per divide => 2 nops; nclip 8cy => 2 nops; avsz3/avsz4 14cy => 2 nops;
-- => 2 nops; nclip 8cy => 2 nops; avsz3/avsz4 14cy => 2 nops; op -- op single-cycle atomic => 0 nops; mvmva 8cy matrix-vector => 2 nops).
-- single-cycle atomic => 0 nops; mvmva 8cy matrix-vector => 2 nops).
M.GTE_PIPELINE_LATENCY = { M.GTE_PIPELINE_LATENCY = {
-- Minimum number of consecutive `nop` words that must appear -- Minimum number of consecutive `nop` words that must appear IMMEDIATELY BEFORE a `gte_cmdw_<X>` invocation
-- IMMEDIATELY BEFORE a `gte_cmdw_<X>` invocation -- to retire -- to retire any preceding `lwc2` / `swc2` / pre-existing C2 state writes before the GTE pipeline starts reading
-- any preceding `lwc2` / `swc2` / pre-existing C2 state writes -- from V0/V1/V2 or MAC0..3 / OTZ / IR0..3 at the command's issue cycle.
-- before the GTE pipeline starts reading from V0/V1/V2 or
-- MAC0..3 / OTZ / IR0..3 at the command's issue cycle.
--
-- Values are from the doxygen comments in code/duffle/gte.h and
-- cross-checked against PSX-SPX `geometrytransformationenginegte.md`:
-- --
-- Values are from the doxygen comments in code/duffle/gte.h and cross-checked against PSX-SPX `geometrytransformationenginegte.md`:
-- cmd cycles min pre-nops rationale -- cmd cycles min pre-nops rationale
-- rtps 14 2 8c per perspective divide + 6c for IR1..4 + mac write -- rtps 14 2 8c per perspective divide + 6c for IR1..4 + mac write
-- rptt 22 2 3x rtps worth of pipeline depth -- rptt 22 2 3x rtps worth of pipeline depth
@@ -614,22 +587,16 @@ M.GTE_PIPELINE_LATENCY = {
-- mvmva 8 2 IR1..4 write + matrix work -- mvmva 8 2 IR1..4 write + matrix work
-- op 5 0 output to MAC0 only (atomic 5c calc) -- op 5 0 output to MAC0 only (atomic 5c calc)
-- --
-- The `gte_rtpt()` / `gte_nclip()` / `gte_avsz3()` wrapper macros in -- The `gte_rtpt()` / `gte_nclip()` / `gte_avsz3()` wrapper macros in gte.h emit the pre-cmd nops internally (asm_words(nop, nop, ...)),
-- gte.h emit the pre-cmd nops internally (asm_words(nop, nop, ...)), -- but THOSE WRAPPERS ARE NOT USED INSIDE ATOM BODIES in this codebase.
-- but THOSE WRAPPERS ARE NOT USED INSIDE ATOM BODIES in this -- Every MipsAtom_(name) body uses raw `nop2, gte_cmdw_<X>, ...` form instead -- that `nop2,` is the pre-fill this check validates.
-- codebase. Every MipsAtom_(name) body uses raw `nop2, -- So values here must reflect the source-level convention, NOT the wrapper-internal pre-fill (which is invisible at the source level).
-- gte_cmdw_<X>, ...` form instead -- that `nop2,` is the pre-fill
-- this check validates. So values here must reflect the source-level
-- convention, NOT the wrapper-internal pre-fill (which is invisible
-- at the source level).
-- --
-- Existing clean-atom bodies (cube_g4_face, floor_f3_face, -- Existing clean-atom bodies (cube_g4_face, floor_f3_face, diag_gte) all emit `nop2,` before every `gte_cmdw_<X>` (which matches values >= 2).
-- diag_gte) all emit `nop2,` before every `gte_cmdw_<X>` (which -- The check passes them all.
-- matches values >= 2). The check passes them all.
-- --
-- Aliases are listed separately because source code may use either -- Aliases are listed separately because source code may use either the alias or the canonical name.
-- the alias or the canonical name. The check looks up the EXACT -- The check looks up the EXACT macro text, so both forms must be in the table.
-- macro text, so both forms must be in the table.
-- Canonical macros (from code/duffle/gte.h) -- Canonical macros (from code/duffle/gte.h)
["gte_cmdw_rtps"] = 2, ["gte_cmdw_rtps"] = 2,
@@ -676,8 +643,7 @@ M.GP0_CMD_SIZE = {
} }
-- Shape suffix (after `ac_format_` / `mac_format_` prefix) -> GP0 cmd byte. -- Shape suffix (after `ac_format_` / `mac_format_` prefix) -> GP0 cmd byte.
-- Lets the static-analysis check derive the cmd byte from a macro name -- Lets the static-analysis check derive the cmd byte from a macro name like `mac_format_g4_color` -> `g4` -> 0x38 -> 9 expected words.
-- like `mac_format_g4_color` -> `g4` -> 0x38 -> 9 expected words.
M.GP0_CMD_BY_SHAPE = { M.GP0_CMD_BY_SHAPE = {
["f3"] = 0x20, ["ft3"] = 0x24, ["f3"] = 0x20, ["ft3"] = 0x24,
["f4"] = 0x28, ["ft4"] = 0x2C, ["f4"] = 0x28, ["ft4"] = 0x2C,
@@ -685,11 +651,9 @@ M.GP0_CMD_BY_SHAPE = {
["g4"] = 0x38, ["gt4"] = 0x3C, ["g4"] = 0x38, ["gt4"] = 0x3C,
} }
-- Per-macro prim-buffer contribution (NOT .text instruction count -- -- Per-macro prim-buffer contribution
-- this is "how many 32-bit words does this macro write to the primitive -- (NOT .text instruction count this is "how many 32-bit words does this macro write to the primitive being built in main RAM").
-- being built in main RAM"). Sum across `mac_format_X_color` + -- Sum across `mac_format_X_color` + `mac_gte_store_X_post_*` + `mac_insert_ot_tag_X` calls in an atom body must equal GP0_CMD_SIZE[GP0_CMD_BY_SHAPE[shape]].
-- `mac_gte_store_X_post_*` + `mac_insert_ot_tag_X` calls in an atom body
-- must equal GP0_CMD_SIZE[GP0_CMD_BY_SHAPE[shape]].
M.GP0_MACRO_CONTRIB = { M.GP0_MACRO_CONTRIB = {
["mac_format_f3_color"] = 1, ["mac_format_f3_color"] = 1,
["mac_format_g3_color"] = 3, ["mac_format_g3_color"] = 3,
@@ -702,26 +666,21 @@ M.GP0_MACRO_CONTRIB = {
["mac_insert_ot_tag_g4"] = 1, ["mac_insert_ot_tag_g4"] = 1,
} }
-- Per-macro cycle cost (best-case, no stalls). Used by the static-analysis -- Per-macro cycle cost (best-case, no stalls). Used by the static-analysis `count_atom_cycles` pass (Phase 3) to emit per-atom cycle budgets.
-- `count_atom_cycles` pass (Phase 3) to emit per-atom cycle budgets. The -- The counts cover the EXPANDED instruction sequence the macro emits (NOT just the token it appears as in source).
-- counts cover the EXPANDED instruction sequence the macro emits (NOT just -- For example:
-- the token it appears as in source). For example:
--
-- mac_pack_color_word(off, cmd, r, g, b) emits: -- mac_pack_color_word(off, cmd, r, g, b) emits:
-- load_upper_i(R_AT, (cmd << 8) | b) -- 1 cycle -- load_upper_i(R_AT, (cmd << 8) | b) -- 1 cycle
-- or_i_self(R_AT, (g << 8) | r) -- 1 cycle -- or_i_self(R_AT, (g << 8) | r) -- 1 cycle
-- store_word(R_AT, R_PrimCursor, off) -- 1 cycle -- store_word(R_AT, R_PrimCursor, off) -- 1 cycle
-- = 3 cycles total -- = 3 cycles total
-- --
-- mac_yield emits a control-transfer sequence (load_word, add_ui_self, -- mac_yield emits a control-transfer sequence (load_word, add_ui_self, jump_reg, nop)
-- jump_reg, nop) which "yields control" -- the atom body's cycle budget -- which "yields control" the atom body's cycle budget doesn't include the yield's cost (we model it as 0;
-- doesn't include the yield's cost (we model it as 0; runtime cost -- runtime cost becomes part of the NEXT atom's prologue).
-- becomes part of the NEXT atom's prologue).
-- --
-- GTE command values are the GTE instruction's intrinsic cycles (the -- GTE command values are the GTE instruction's intrinsic cycles (the latency AFTER any pre-cmd `nop2` has retired).
-- latency AFTER any pre-cmd `nop2` has retired). When the source emits -- When the source emits `nop2, gte_cmdw_X` the nops' cycles are added separately (1+1) plus the gte_cmdw_X value here:
-- `nop2, gte_cmdw_X` the nops' cycles are added separately (1+1) plus
-- the gte_cmdw_X value here:
-- rtpt = 21 + 2 nops = 23 total cycles (matches PSX-SPX) -- rtpt = 21 + 2 nops = 23 total cycles (matches PSX-SPX)
-- rtps = 12 + 2 nops = 14 total -- rtps = 12 + 2 nops = 14 total
-- nclip = 6 + 2 nops = 8 total -- nclip = 6 + 2 nops = 8 total
@@ -847,8 +806,8 @@ M.INSTRUCTION_LATENCY = {
["atom_writes"] = 0, ["atom_writes"] = 0,
} }
-- Default cycle cost for unknown macros. The static-analysis pass adds 1 -- Default cycle cost for unknown macros.
-- cycle per unknown token and emits a "new macro; update INSTRUCTION_LATENCY" -- The static-analysis pass adds 1 cycle per unknown token and emits a "new macro; update INSTRUCTION_LATENCY"
-- advisory so the cycle budget stays accurate as the codebase grows. -- advisory so the cycle budget stays accurate as the codebase grows.
M.UNKNOWN_INSTRUCTION_CYCLES = 1 M.UNKNOWN_INSTRUCTION_CYCLES = 1
+22 -35
View File
@@ -187,7 +187,6 @@ local function split_csv_top(s)
end end
--- Split a string into whitespace-separated tokens. --- Split a string into whitespace-separated tokens.
--- Hand-rolled (no regex patterns).
--- @param s string --- @param s string
--- @return string[] --- @return string[]
local function split_ws(s) local function split_ws(s)
@@ -212,10 +211,8 @@ end
-- Parse TAPE_ATOM_ANNOT(...) calls -- Parse TAPE_ATOM_ANNOT(...) calls
-- ════════════════════════════════════════════════════════════════════════════ -- ════════════════════════════════════════════════════════════════════════════
-- Recognize a `atom_bind(...)`, `atom_reads(...)`, or `atom_writes(...)` -- Recognize a `atom_bind(...)`, `atom_reads(...)`, or `atom_writes(...)` sub-call embedded inside an atom_info arg list.
-- sub-call embedded inside an atom_info arg list. -- Returns the kind ("atom_bind" / "atom_reads" / "atom_writes") and the inner content, or nil if the token isn't a recognized sub-call form.
-- Returns the kind ("atom_bind" / "atom_reads" / "atom_writes") and the inner content,
-- or nil if the token isn't a recognized sub-call form.
-- Flattened via a prefix lookup instead of a nested if/elseif chain. -- Flattened via a prefix lookup instead of a nested if/elseif chain.
local REGS_CALL_PREFIX = { local REGS_CALL_PREFIX = {
["atom_writes("] = { kind = "atom_writes", inner_offset = 13 }, ["atom_writes("] = { kind = "atom_writes", inner_offset = 13 },
@@ -451,9 +448,8 @@ local function parse_binds_body(body)
end end
--- Try to parse a `typedef Struct_(Binds_X) { ... };` declaration. --- Try to parse a `typedef Struct_(Binds_X) { ... };` declaration.
--- Returns the parsed BindsStruct (if the form matched) and the new --- Returns the parsed BindsStruct (if the form matched) and the new source position.
--- source position. If the form didn't match, returns nil + a position --- If the form didn't match, returns nil + a position to continue scanning from.
--- to continue scanning from.
--- @param source string --- @param source string
--- @param ident_pos integer -- position of the `typedef` ident start --- @param ident_pos integer -- position of the `typedef` ident start
--- @param after_typedef integer -- position just past `typedef` --- @param after_typedef integer -- position just past `typedef`
@@ -527,10 +523,9 @@ end
-- Find every MipsAtom_(name) { ... } declaration in source -- Find every MipsAtom_(name) { ... } declaration in source
-- ════════════════════════════════════════════════════════════════════════════ -- ════════════════════════════════════════════════════════════════════════════
--- Read the next identifier token from `s` starting at `pos`, where the --- Read the next identifier token from `s` starting at `pos`, where the identifier is a contiguous run of `[a-zA-Z0-9_]`
--- identifier is a contiguous run of `[a-zA-Z0-9_]` characters (no --- characters (no underscore-starting alpha-only constraint).
--- underscore-starting alpha-only constraint). Returns the ident + the --- Returns the ident + the position just past it, or nil + pos if no identifier starts there.
--- position just past it, or nil + pos if no identifier starts there.
--- @param s string --- @param s string
--- @param pos integer --- @param pos integer
--- @return string|nil, integer --- @return string|nil, integer
@@ -541,8 +536,8 @@ local function read_alnum_ident(s, pos)
return s:sub(start, pos - 1), pos return s:sub(start, pos - 1), pos
end end
--- Find every `MipsAtom_(name)` declaration in source. (Just the name + --- Find every `MipsAtom_(name)` declaration in source.
--- source line; the body is parsed separately by `parse_mips_atom`.) --- (Just the name + source line; the body is parsed separately by `parse_mips_atom`.)
--- @param source string --- @param source string
--- @return Atom[] --- @return Atom[]
local function find_atom_names(source) local function find_atom_names(source)
@@ -648,9 +643,8 @@ local function parse_atom_info_call(source, atom_name, after_mipsatom_paren, lin
end end
--- Find every `MipsAtom_(name) atom_info(...) { ... };` annotation in source. --- Find every `MipsAtom_(name) atom_info(...) { ... };` annotation in source.
--- Returns a list of annotation entries. Atoms without a following --- Returns a list of annotation entries. Atoms without a following `atom_info(...)` call produce NO entry
--- `atom_info(...)` call produce NO entry (atoms without annotations are --- (atoms without annotations are valid in the new minimal shape).
--- valid in the new minimal shape).
--- @param source string --- @param source string
--- @return AtomAnnotation[] --- @return AtomAnnotation[]
local function find_atom_annotations(source) local function find_atom_annotations(source)
@@ -861,16 +855,13 @@ end
-- Per-DIRECTORY (per-module) output: errors.h + annotations.txt -- Per-DIRECTORY (per-module) output: errors.h + annotations.txt
-- ════════════════════════════════════════════════════════════════════════════ -- ════════════════════════════════════════════════════════════════════════════
-- --
-- Per-source reports were the old behavior; each source in the same -- Per-source reports were the old behavior; each source in the same directory produced its own <basename>.errors.h + <basename>.annotations.txt,
-- directory produced its own <basename>.errors.h + <basename>.annotations.txt, -- which flooded build/gen/ with one report per header.
-- which flooded build/gen/ with one report per header. The new behavior -- Aggregates per-DIRECTORY (one errors.h + one annotations.txt per module basename).
-- aggregates per-DIRECTORY (one errors.h + one annotations.txt per module -- Directories with zero atoms/annotations are skipped (no file emitted).
-- basename). Directories with zero atoms/annotations are skipped (no
-- file emitted).
--- Render `<dir_basename>.errors.h` with `#error` directives for every --- Render `<dir_basename>.errors.h` with `#error` directives for every error found across all sources in the directory.
--- error found across all sources in the directory. Empty directories --- Empty directories (no errors, no atoms) produce no file.
--- (no errors, no atoms) produce no file.
local function emit_module_errors_h(ctx, dir_basename, atoms_count, errors, sources) local function emit_module_errors_h(ctx, dir_basename, atoms_count, errors, sources)
if ctx.dry_run then return nil end if ctx.dry_run then return nil end
if atoms_count == 0 and #errors == 0 then if atoms_count == 0 and #errors == 0 then
@@ -923,10 +914,8 @@ end
local M = {} local M = {}
-- Expose `validate` for downstream passes (e.g. report.lua) that need -- Expose `validate` for downstream passes (e.g. report.lua) that need to re-render the per-source results into a per-MODULE report.
-- to re-render the per-source results into a per-MODULE report. Keeping -- Keeping it as a single shared function avoids the duplication that an earlier version of report.lua had.
-- it as a single shared function avoids the duplication that an
-- earlier version of report.lua had.
M.validate = validate M.validate = validate
--- @param ctx PassCtx --- @param ctx PassCtx
@@ -936,11 +925,9 @@ function M.run(ctx)
local errors = {} local errors = {}
local warnings = {} local warnings = {}
-- Per-DIRECTORY (per-module) aggregation. Group sources by `src.dir`, -- Per-DIRECTORY (per-module) aggregation. Group sources by `src.dir`, validate every source in the dir, then emit ONE errors.h per dir
-- validate every source in the dir, then emit ONE errors.h per dir -- (skipping dirs with no atoms AND no errors).
-- (skipping dirs with no atoms AND no errors). The actual -- The actual annotations.txt is rendered by passes/report.lua from the stashed per-module results below.
-- annotations.txt is rendered by passes/report.lua from the stashed
-- per-module results below.
local by_dir = {} local by_dir = {}
for _, src in ipairs(ctx.sources) do for _, src in ipairs(ctx.sources) do
by_dir[src.dir] = by_dir[src.dir] or {} by_dir[src.dir] = by_dir[src.dir] or {}
+55 -91
View File
@@ -1,8 +1,7 @@
--- passes/components.lua — Component-macro header generator. --- passes/components.lua — Component-macro header generator.
--- ---
--- Walks every source for `MipsAtomComp_(ac_X) { body }` --- Walks every source for `MipsAtomComp_(ac_X) { body }` (and the function-form `MipsAtomComp_Proc_(ac_X, { body })`) declarations and
--- (and the function-form `MipsAtomComp_Proc_(ac_X, { body })`) declarations --- emits a per-directory `<dir_basename>.macs.h` containing one `#define mac_X(sig) \` macro per component + `WORD_COUNT(mac_X, N)`
--- and emits a per-directory `<dir_basename>.macs.h` containing one `#define mac_X(sig) \` macro per component + `WORD_COUNT(mac_X, N)`
--- entries for downstream offset computation. --- entries for downstream offset computation.
--- ---
--- **Conventions**: tabs (1/level), EmmyLua annotations, no regex, --- **Conventions**: tabs (1/level), EmmyLua annotations, no regex,
@@ -116,8 +115,7 @@ local function to_absolute_path(path)
local cwd = p:read("*l") local cwd = p:read("*l")
p:close() p:close()
if not cwd then return path end if not cwd then return path end
-- Normalize forward slashes to backslashes (Windows convention) on -- Normalize forward slashes to backslashes (Windows convention) on both the cwd AND the relative path tail, so the join is uniform.
-- both the cwd AND the relative path tail, so the join is uniform.
cwd = cwd:gsub("/", "\\") cwd = cwd:gsub("/", "\\")
local tail = (path:gsub("/", "\\")) local tail = (path:gsub("/", "\\"))
return cwd .. "\\" .. tail return cwd .. "\\" .. tail
@@ -149,16 +147,13 @@ local function find_last_name_open_paren(source, name, before_pos)
return last_idx return last_idx
end end
--- Find the args of the function declaration that immediately precedes --- Find the args of the function declaration that immediately precedes a `MipsAtomComp_Proc_` invocation of the given name.
--- a `MipsAtomComp_Proc_` invocation of the given name. Returns the --- Returns the args string (e.g., `"U4 off, U4 code, U1 r, U1 g, U1 b"`) or nil if no function declaration is found.
--- args string (e.g., `"U4 off, U4 code, U1 r, U1 g, U1 b"`) or nil
--- if no function declaration is found.
--- ---
--- Convention: function form is --- Convention: function form is
--- `FI_ MipsAtom ac_X(args) MipsAtomComp_Proc_(ac_X, { body })` --- `FI_ MipsAtom ac_X(args) MipsAtomComp_Proc_(ac_X, { body })`
--- We find the LAST occurrence of `"ac_X("` before `before_pos` and --- We find the LAST occurrence of `"ac_X("` before `before_pos` and extract the args from inside the parens.
--- extract the args from inside the parens. We then verify the --- We then verify the preceding context ends with `MipsAtom` (the function-decl keyword
--- preceding context ends with `MipsAtom` (the function-decl keyword
--- with possible qualifiers between). --- with possible qualifiers between).
--- ---
--- @param source string --- @param source string
@@ -169,8 +164,7 @@ local function find_function_args_for(source, name, before_pos)
local last_idx = find_last_name_open_paren(source, name, before_pos) local last_idx = find_last_name_open_paren(source, name, before_pos)
if not last_idx then return nil end if not last_idx then return nil end
-- Verify the preceding context ends with "MipsAtom" (with -- Verify the preceding context ends with "MipsAtom" (with possible qualifiers between).
-- possible qualifiers between).
local before = source:sub(1, last_idx - 1) local before = source:sub(1, last_idx - 1)
local trimmed = duffle.trim(before) local trimmed = duffle.trim(before)
if trimmed:sub(-#MIPS_ATOM) ~= MIPS_ATOM then if trimmed:sub(-#MIPS_ATOM) ~= MIPS_ATOM then
@@ -188,8 +182,7 @@ end
-- Preceding-comment-block extraction -- Preceding-comment-block extraction
-- ════════════════════════════════════════════════════════════════════════════ -- ════════════════════════════════════════════════════════════════════════════
-- Skip whitespace (space/tab/newline/CR) backward from `pos`, returning -- Skip whitespace (space/tab/newline/CR) backward from `pos`, returning the position of the first non-whitespace char.
-- the position of the first non-whitespace char.
-- @param source string -- @param source string
-- @param pos integer -- @param pos integer
-- @return integer -- @return integer
@@ -223,8 +216,7 @@ local function find_block_comment_open(source, close_pos)
return open_at return open_at
end end
-- Walk back from `open_at` over leading spaces + tabs to include the -- Walk back from `open_at` over leading spaces + tabs to include the indentation before the `/*` in the captured comment.
-- indentation before the `/*` in the captured comment.
-- @param source string -- @param source string
-- @param open_at integer -- @param open_at integer
-- @return integer -- @return integer
@@ -241,8 +233,7 @@ local function extend_left_over_indent(source, open_at)
return start return start
end end
-- Walk back from `line_end` to the start of the source line (the most -- Walk back from `line_end` to the start of the source line (the most recent `\n` or position 1).
-- recent `\n` or position 1).
-- @param source string -- @param source string
-- @param line_end integer -- @param line_end integer
-- @return integer -- @return integer
@@ -255,9 +246,8 @@ local function find_line_start(source, line_end)
end end
-- (internal) Capture one `/* ... */` block comment whose closing `*/` -- (internal) Capture one `/* ... */` block comment whose closing `*/`
-- ends at `close_end_pos`. Returns (block_text, new_scan_pos) where -- ends at `close_end_pos`. Returns (block_text, new_scan_pos) where `new_scan_pos`
-- `new_scan_pos` is where to continue scanning for more comments, or -- is where to continue scanning for more comments, or nil if no block comment was found.
-- nil if no block comment was found.
local function capture_block_comment(source, close_end_pos) local function capture_block_comment(source, close_end_pos)
local open_at = find_block_comment_open(source, close_end_pos) local open_at = find_block_comment_open(source, close_end_pos)
if not open_at then return nil end if not open_at then return nil end
@@ -276,15 +266,11 @@ local function capture_line_comment(source, line_end_pos)
return nil return nil
end end
--- Find the contiguous comment block immediately preceding `pos` in --- Find the contiguous comment block immediately preceding `pos` in `source`.
--- `source`. Returns the comment text (with the `/* */` or `//` markers --- Returns the comment text (with the `/* */` or `//` markers preserved) or an empty string if no comment is adjacent.
--- preserved) or an empty string if no comment is adjacent.
--- ---
--- Used to copy signature comments from the source declaration --- Used to copy signature comments from the source declaration (`MipsAtomComp_` / `MipsAtomComp_Proc_` / function decl)
--- (`MipsAtomComp_` / `MipsAtomComp_Proc_` / function decl) over to the --- over to the generated `mac_X` macro, so LSP/IntelliSense displays the args doc.
--- generated `mac_X` macro, so LSP/IntelliSense displays the args doc.
---
--- No regex (per the no_regex constraint).
--- ---
--- @param source string --- @param source string
--- @param pos integer --- @param pos integer
@@ -321,8 +307,7 @@ end
-- Argument-name extraction -- Argument-name extraction
-- ════════════════════════════════════════════════════════════════════════════ -- ════════════════════════════════════════════════════════════════════════════
-- Walk `trimmed` backward from `pos` over trailing whitespace / -- Walk `trimmed` backward from `pos` over trailing whitespace / asterisks / brackets, returning the position of the first
-- asterisks / brackets, returning the position of the first
-- non-trailer character (i.e. the end of the identifier). -- non-trailer character (i.e. the end of the identifier).
-- @param trimmed string -- @param trimmed string
-- @param pos integer -- @param pos integer
@@ -358,8 +343,7 @@ local function trim_ident_back(trimmed, pos)
return back return back
end end
--- Extract just the parameter NAMES from a function-args string --- Extract just the parameter NAMES from a function-args string (stripping type annotations). E.g.,
--- (stripping type annotations). E.g.,
--- `"U4 off, U4 code, U1 r, U1 g, U1 b"` -> `{"off", "code", "r", "g", "b"}` --- `"U4 off, U4 code, U1 r, U1 g, U1 b"` -> `{"off", "code", "r", "g", "b"}`
--- `"U4 *ptr"` -> `{"ptr"}` --- `"U4 *ptr"` -> `{"ptr"}`
--- `""` -> nil --- `""` -> nil
@@ -389,9 +373,8 @@ end
-- Component scanner (bare + function forms) -- Component scanner (bare + function forms)
-- ════════════════════════════════════════════════════════════════════════════ -- ════════════════════════════════════════════════════════════════════════════
-- Parse the inner content of an `AtomComp_(name, ...)` call. Returns -- Parse the inner content of an `AtomComp_(name, ...)` call.
-- (name, body_or_nil) — `body_or_nil` is non-nil iff this is the -- Returns (name, body_or_nil) — `body_or_nil` is non-nil iff this is the function-form `MipsAtomComp_Proc_(name, { body })` invocation.
-- function-form `MipsAtomComp_Proc_(name, { body })` invocation.
-- @param inner string -- the content between ( and ) of the AtomComp_ call -- @param inner string -- the content between ( and ) of the AtomComp_ call
--- @return string|nil, string|nil --- @return string|nil, string|nil
local function parse_atomcomp_inner(inner) local function parse_atomcomp_inner(inner)
@@ -414,8 +397,7 @@ local function parse_atomcomp_inner(inner)
end end
-- (internal) Try to extract a bare-form `MipsAtomComp_(ac_X)` declaration. -- (internal) Try to extract a bare-form `MipsAtomComp_(ac_X)` declaration.
-- Bare form: `MipsAtomComp_(ac_X) { body }` — body comes from the brace block -- Bare form: `MipsAtomComp_(ac_X) { body }` — body comes from the brace block AFTER the parens.
-- AFTER the parens.
-- @param source string -- @param source string
-- @param name string -- the `ac_X` ident from the parens -- @param name string -- the `ac_X` ident from the parens
--- @param ident_pos integer -- position of the `MipsAtomComp_` ident start --- @param ident_pos integer -- position of the `MipsAtomComp_` ident start
@@ -507,13 +489,11 @@ end
-- Convert `//` line comments to `/* */` block comments in a token. -- Convert `//` line comments to `/* */` block comments in a token.
-- --
-- C macros use `\` line-continuations; a `//` comment before `\` would -- C macros use `\` line-continuations; a `//` comment before `\` would consume the continuation,
-- consume the continuation, breaking the macro. We convert `//` to -- breaking the macro. We convert `//` to `/* */` so the multi-line macro structure is preserved.
-- `/* */` so the multi-line macro structure is preserved.
-- --
-- Skips `//` sequences that are inside string or character literals -- Skips `//` sequences that are inside string or character literals
-- (a rough heuristic — sufficient for component bodies which don't -- (a rough heuristic — sufficient for component bodies which don't have those constructs).
-- have those constructs).
-- --
--- @param s string --- @param s string
--- @return string --- @return string
@@ -551,10 +531,8 @@ end
-- Word-count computation (memoized recursive lookup) -- Word-count computation (memoized recursive lookup)
-- ════════════════════════════════════════════════════════════════════════════ -- ════════════════════════════════════════════════════════════════════════════
-- Strip the `mac_` prefix from a component-call ident so we can look it -- Strip the `mac_` prefix from a component-call ident so we can look it up against the components-by-name table. Returns the ident unchanged
-- up against the components-by-name table. Returns the ident unchanged -- if it doesn't start with the prefix (so a non-component ident like `mask_upper` falls through to the wc-table branch).
-- if it doesn't start with the prefix (so a non-component ident like
-- `mask_upper` falls through to the wc-table branch).
-- @param ident string|nil -- @param ident string|nil
-- @return string|nil -- @return string|nil
local function strip_mac_prefix(ident) local function strip_mac_prefix(ident)
@@ -565,9 +543,8 @@ local function strip_mac_prefix(ident)
return ident return ident
end end
-- (internal) Recursive word-count lookup. `cache` is the memoization table -- (internal) Recursive word-count lookup. `cache` is the memoization table across all calls to `compute_component_word_count`;
-- across all calls to `compute_component_word_count`; the in-progress -- the in-progress -1 sentinel detects cycles (A -> B -> A).
-- -1 sentinel detects cycles (A -> B -> A).
-- @param name string -- the component name (without `mac_`) -- @param name string -- the component name (without `mac_`)
-- @param comp_by_name table<string, Component> -- @param comp_by_name table<string, Component>
-- @param wc table<string, integer> -- @param wc table<string, integer>
@@ -604,17 +581,15 @@ local function word_count_rec(name, comp_by_name, wc, cache)
return n return n
end end
--- Compute the word count of a component body, accounting for macro --- Compute the word count of a component body, accounting for macro expansion.
--- expansion. Each comma-separated entry in the body is a "slot" that --- Each comma-separated entry in the body is a "slot" that contributes its own word count.
--- contributes its own word count. For most entries (regular MIPS --- For most entries (regular MIPS instructions) the count is 1.
--- instructions) the count is 1. For `mac_Y(...)` calls, the count is --- For `mac_Y(...)` calls, the count is the word count of `mac_Y` (recursive lookup through `components`).
--- the word count of `mac_Y` (recursive lookup through `components`).
--- For encoding macros with a known multi-word count (e.g. `mask_upper` = 2), --- For encoding macros with a known multi-word count (e.g. `mask_upper` = 2),
--- the count is taken from `word_counts`. --- the count is taken from `word_counts`.
--- ---
--- The lookup is memoized via `word_count_rec` to avoid infinite recursion --- The lookup is memoized via `word_count_rec` to avoid infinite recursion (e.g. if two components referenced each other).
--- (e.g. if two components referenced each other). This is the same --- This is the same algorithm as the original `tape_atom_annotation_pass.lua` (commit 7d20a4d).
--- algorithm as the original `tape_atom_annotation_pass.lua` (commit 7d20a4d).
--- ---
--- @param c Component --- @param c Component
--- @param components Component[] --- @param components Component[]
@@ -663,8 +638,7 @@ local function tokens_from_body(body)
return out return out
end end
--- Determine the macro signature: function-args list (function form) --- Determine the macro signature: function-args list (function form) or variadic-ignored (bare form).
--- or variadic-ignored (bare form).
--- @param args_str string|nil --- @param args_str string|nil
--- @return string --- @return string
local function signature_from_args(args_str) local function signature_from_args(args_str)
@@ -675,8 +649,8 @@ local function signature_from_args(args_str)
return "..." return "..."
end end
--- Strip the trailing `" \"` (space + backslash) line continuation --- Strip the trailing `" \"` (space + backslash) line continuation from the last body line.
--- from the last body line. The last 2 chars are always that pair. --- The last 2 chars are always that pair.
local function strip_trailing_continuation(lines) local function strip_trailing_continuation(lines)
local last = lines[#lines] local last = lines[#lines]
if last:sub(-2) == " \\" then if last:sub(-2) == " \\" then
@@ -684,9 +658,8 @@ local function strip_trailing_continuation(lines)
end end
end end
--- Emit the `#define mac_X(sig) \<newline>\t<tok1> \<newline>,\t<tok2> ...` --- Emit the `#define mac_X(sig) \<newline>\t<tok1> \<newline>,\t<tok2> ...` block.
--- block. Converts `//` line comments to `/* */` block comments in --- Converts `//` line comments to `/* */` block comments in each token so they don't break the C macro `\` line continuations.
--- each token so they don't break the C macro `\` line continuations.
local function emit_macro_body(lines, c, sig, tokens) local function emit_macro_body(lines, c, sig, tokens)
for tok_idx = 1, #tokens do for tok_idx = 1, #tokens do
tokens[tok_idx] = convert_line_comments_to_block(tokens[tok_idx]) tokens[tok_idx] = convert_line_comments_to_block(tokens[tok_idx])
@@ -699,9 +672,8 @@ local function emit_macro_body(lines, c, sig, tokens)
strip_trailing_continuation(lines) strip_trailing_continuation(lines)
end end
--- Build the list of lines for one component (signature comment, --- Build the list of lines for one component
--- `#define mac_X(...)` line with backslash-continued tokens, then --- (signature comment, `#define mac_X(...)` line with backslash-continued tokens, then `WORD_COUNT(mac_X, N)` entry).
--- `WORD_COUNT(mac_X, N)` entry).
--- @param c Component --- @param c Component
--- @param components Component[] --- @param components Component[]
--- @param wc table<string, integer> --- @param wc table<string, integer>
@@ -734,17 +706,14 @@ end
-- Per-source emit logic -- Per-source emit logic
-- ════════════════════════════════════════════════════════════════════════════ -- ════════════════════════════════════════════════════════════════════════════
-- Build the boilerplate header lines (the `#ifdef INTELLISENSE_DIRECTIVES` -- Build the boilerplate header lines (the `#ifdef INTELLISENSE_DIRECTIVES` block,
-- block, the `// Auto-generated` comment, the `// Source:` line, and the -- the `// Auto-generated` comment, the `// Source:` line, and the self-contained `WORD_COUNT` macro definition).
-- self-contained `WORD_COUNT` macro definition).
-- @param src SourceFile -- @param src SourceFile
-- @return string[] -- @return string[]
local function header_boilerplate(src) local function header_boilerplate(src)
return { return {
-- #pragma once wrapped in #ifdef INTELLISENSE_DIRECTIVES, matching -- #pragma once wrapped in #ifdef INTELLISENSE_DIRECTIVES, matching the convention in lottes_tape.h.
-- the convention in lottes_tape.h. The build does manual unity -- The build does manual unity includes (the user controls include order), so the pragma is only active for IDE/tooling.
-- includes (the user controls include order), so the pragma
-- is only active for IDE/tooling.
"#ifdef INTELLISENSE_DIRECTIVES", "#ifdef INTELLISENSE_DIRECTIVES",
"#pragma once", "#pragma once",
"#endif", "#endif",
@@ -753,8 +722,7 @@ local function header_boilerplate(src)
"// Component atoms (MipsAtomComp_(ac_*)) -> macro variants (mac_*)", "// Component atoms (MipsAtomComp_(ac_*)) -> macro variants (mac_*)",
"", "",
-- Self-contained: define WORD_COUNT if not already defined. -- Self-contained: define WORD_COUNT if not already defined.
-- We use the same definition here so the auto-generated -- We use the same definition here so the auto-generated entries below expand to compile-time constants whether
-- entries below expand to compile-time constants whether
-- the metadata file is included first or not. -- the metadata file is included first or not.
"#ifndef WORD_COUNT", "#ifndef WORD_COUNT",
"#define WORD_COUNT(name, count) enum { words_##name = (count) };", "#define WORD_COUNT(name, count) enum { words_##name = (count) };",
@@ -764,10 +732,9 @@ local function header_boilerplate(src)
end end
-- Compute the output path for one source's `.macs.h` file. -- Compute the output path for one source's `.macs.h` file.
-- The pre-rework convention uses the *directory* basename (not the -- The pre-rework convention uses the *directory* basename
-- source file basename) — e.g. `code/duffle/lottes_tape.h` produces -- (not the source file basename) — e.g. `code/duffle/lottes_tape.h` produces `code/duffle/gen/duffle.macs.h`.
-- `code/duffle/gen/duffle.macs.h`. This matches what the C codebase -- This matches what the C codebase #includes.
-- #includes.
-- @param src SourceFile -- @param src SourceFile
-- @return string -- the output directory -- @return string -- the output directory
-- @return string -- the full output path -- @return string -- the full output path
@@ -777,13 +744,10 @@ local function compute_macs_h_path(src)
return out_dir, out_path return out_dir, out_path
end end
--- Emit a per-source `.macs.h` header with the `mac_X` macros + --- Emit a per-source `.macs.h` header with the `mac_X` macros + `WORD_COUNT` entries. Writes in BINARY mode so LF line endings are
--- `WORD_COUNT` entries. Writes in BINARY mode so LF line endings are --- preserved (the git blob is LF; Windows text-mode would emit CRLF and break the byte-identical diff).
--- preserved (the git blob is LF; Windows text-mode would emit CRLF and
--- break the byte-identical diff).
--- ---
--- Honors `ctx.dry_run`: prints the intended path but does not write --- Honors `ctx.dry_run`: prints the intended path but does not write the file.
--- the file.
--- ---
--- @param ctx PassCtx --- @param ctx PassCtx
--- @param src SourceFile --- @param src SourceFile
@@ -819,8 +783,8 @@ end
-- Pass entry -- Pass entry
-- ════════════════════════════════════════════════════════════════════════════ -- ════════════════════════════════════════════════════════════════════════════
-- (internal) Extend `ctx.shared.word_counts` with this source's component -- (internal) Extend `ctx.shared.word_counts` with this source's component macros
-- macros so offsets sees them without re-reading the file. -- so offsets sees them without re-reading the file.
-- @param ctx PassCtx -- @param ctx PassCtx
-- @param components Component[] -- @param components Component[]
local function update_shared_word_counts(ctx, components) local function update_shared_word_counts(ctx, components)
+22 -37
View File
@@ -1,26 +1,19 @@
--- passes/offsets.lua — Branch-offset generator. --- passes/offsets.lua — Branch-offset generator.
--- ---
--- Scans every source for `MipsAtom_(name) { ... }` (and the raw --- Scans every source for `MipsAtom_(name) { ... }` (and the raw `MipsCode code_<name> { ... }` form) declarations,
--- `MipsCode code_<name> { ... }` form) declarations, computes the --- computes the word offset from each `atom_offset(F, T)` marker to its target `atom_label(T)` declaration,
--- word offset from each `atom_offset(F, T)` marker to its target --- and emits `<dir_basename>.offsets.h` with one `#define _atom_offset_F_T = N` per branch.
--- `atom_label(T)` declaration, and emits `<dir_basename>.offsets.h`
--- with one `#define _atom_offset_F_T = N` per branch.
--- ---
--- The offset is `target_word - branch_word - 1` (the standard MIPS --- The offset is `target_word - branch_word - 1` (the standard MIPS branch-immediate encoding: branch_offset = relative_pc_in_words - 1).
--- branch-immediate encoding: branch_offset = relative_pc_in_words - 1).
--- ---
--- **Conventions**: tabs (1/level), EmmyLua annotations, no regex, --- **Conventions**: tabs (1/level), EmmyLua annotations, no regex,
--- Lua 5.3 compatible. See --- Lua 5.3 compatible.
--- `C:\projects\Pikuma\ps1-ai\conductor\code_styleguides\lua.md`.
-- ════════════════════════════════════════════════════════════════════════════ -- ════════════════════════════════════════════════════════════════════════════
-- Module-scope requires + package.path setup -- Module-scope requires + package.path setup
-- ════════════════════════════════════════════════════════════════════════════ -- ════════════════════════════════════════════════════════════════════════════
-- Resolve `arg[0]` to an absolute-ish script directory so that -- Resolve `arg[0]` to an absolute-ish script directory so that `require("duffle")` resolves against `scripts/` regardless of CWD.
-- `require("duffle")` resolves against `scripts/` regardless of CWD.
-- Note: this boilerplate is duplicated in 6 other entry scripts; a
-- Phase-6 extraction target (`duffle.setup_package_path()`).
-- Bootstrap: see `ps1_meta.lua` for the rationale. -- Bootstrap: see `ps1_meta.lua` for the rationale.
-- Bootstrap: load `scripts/duffle_paths.lua` (sets package.path + package.cpath). -- Bootstrap: load `scripts/duffle_paths.lua` (sets package.path + package.cpath).
-- Uses `debug.getinfo` to find this file's own directory, so it works -- Uses `debug.getinfo` to find this file's own directory, so it works
@@ -145,10 +138,8 @@ end
-- Marker-call helpers -- Marker-call helpers
-- ════════════════════════════════════════════════════════════════════════════ -- ════════════════════════════════════════════════════════════════════════════
-- Extract comma-separated identifier args from a parenthesized group -- Extract comma-separated identifier args from a parenthesized group after a function-like macro call.
-- after a function-like macro call. Returns (args, after_paren) where -- Returns (args, after_paren) where `after_paren` is the position just past the closing `)`, or nil if `token` did not start with `(`.
-- `after_paren` is the position just past the closing `)`, or nil if
-- `token` did not start with `(`.
-- @param token string -- @param token string
-- @param after_ident integer -- @param after_ident integer
-- @return string[], integer|nil -- @return string[], integer|nil
@@ -177,8 +168,7 @@ local function extract_ident_args(token, after_ident)
return args, after_paren return args, after_paren
end end
-- (internal) Record a `atom_label(name)` marker — `at_pos` is the -- (internal) Record a `atom_label(name)` marker — `at_pos` is the branch-free word position within the atom body.
-- branch-free word position within the atom body.
-- @param labels table<string, integer> -- @param labels table<string, integer>
-- @param args string[] -- @param args string[]
-- @param at_pos integer -- @param at_pos integer
@@ -196,8 +186,7 @@ local function record_offset_marker(branches, args, at_pos)
end end
end end
--- Scan a single token for atom_label/atom_offset markers, walking through --- Scan a single token for atom_label/atom_offset markers, walking through balanced groups transparently (so nested calls are found).
--- balanced groups transparently (so nested calls are found).
--- @param token string --- @param token string
--- @param at_pos integer -- the branch-free word position of this token in the body --- @param at_pos integer -- the branch-free word position of this token in the body
--- @param labels table<string, integer> --- @param labels table<string, integer>
@@ -229,8 +218,7 @@ local function scan_for_atom_markers(token, at_pos, labels, branches)
end end
end end
--- Find the end position (just past the closing ')') of the first --- Find the end position (just past the closing ')') of the first atom_label/atom_offset call in `tok`. Returns 0 if no such call.
--- atom_label/atom_offset call in `tok`. Returns 0 if no such call.
--- @param tok string --- @param tok string
--- @return integer -- 0 if no marker call found; otherwise end-1 (just past ')') --- @return integer -- 0 if no marker call found; otherwise end-1 (just past ')')
local function find_marker_call_end(tok) local function find_marker_call_end(tok)
@@ -266,8 +254,7 @@ end
-- Atom scanner -- Atom scanner
-- ════════════════════════════════════════════════════════════════════════════ -- ════════════════════════════════════════════════════════════════════════════
--- Skip C qualifier keywords (`static`, `const`, etc.) and return the --- Skip C qualifier keywords (`static`, `const`, etc.) and return the position past the last qualifier.
--- position past the last qualifier.
--- @param source string --- @param source string
--- @param pos integer --- @param pos integer
--- @return integer --- @return integer
@@ -281,8 +268,7 @@ local function skip_qualifiers(source, pos)
end end
-- (internal) Try to parse the wrapped atom form: `MipsAtom_(<name>) { ... }`. -- (internal) Try to parse the wrapped atom form: `MipsAtom_(<name>) { ... }`.
-- Returns the parsed Atom (name + body + position past body), or nil if -- Returns the parsed Atom (name + body + position past body), or nil if the form didn't match.
-- the form didn't match.
-- @param source_text string -- @param source_text string
-- @param after_pos integer -- position just past `MipsAtom_` -- @param after_pos integer -- position just past `MipsAtom_`
-- @return Atom|nil -- @return Atom|nil
@@ -325,8 +311,7 @@ local function try_raw_atom(source_text, after_pos)
return { name = atom_name, body = body, after_brace = after_brace } return { name = atom_name, body = body, after_brace = after_brace }
end end
--- Find every `MipsAtom_(name) { ... }` (or raw `MipsCode code_<name> { ... }`) --- Find every `MipsAtom_(name) { ... }` (or raw `MipsCode code_<name> { ... }`) declaration in a source.
--- declaration in a source.
--- @param source_text string --- @param source_text string
--- @return Atom[] --- @return Atom[]
local function find_atoms(source_text) local function find_atoms(source_text)
@@ -369,9 +354,9 @@ end
-- ════════════════════════════════════════════════════════════════════════════ -- ════════════════════════════════════════════════════════════════════════════
-- (internal) Count words emitted by the rest of `tok` after a marker call -- (internal) Count words emitted by the rest of `tok` after a marker call
-- (the marker call itself emits 0 words, but the source pattern may bundle -- (the marker call itself emits 0 words, but the source pattern may bundle the marker with the next instruction on the same line,
-- the marker with the next instruction on the same line, separated by no -- separated by no top-level comma).
-- top-level comma). Returns the word count contributed by that rest. -- Returns the word count contributed by that rest.
-- @param tok string -- @param tok string
-- @param word_counts table -- @param word_counts table
-- @return integer -- @return integer
@@ -418,8 +403,8 @@ end
-- Offset computation + header generation -- Offset computation + header generation
-- ════════════════════════════════════════════════════════════════════════════ -- ════════════════════════════════════════════════════════════════════════════
-- Compute branch offsets as `target_word - branch_word - 1` (the -- Compute branch offsets as `target_word - branch_word - 1`
-- standard MIPS branch-immediate encoding). -- (the standard MIPS branch-immediate encoding).
-- @param labels table<string, integer> -- @param labels table<string, integer>
-- @param branches table[] -- @param branches table[]
-- @return BranchOffset[] -- @return BranchOffset[]
@@ -528,9 +513,9 @@ local function process_source(ctx, src)
return out_path return out_path
end end
--- Run the offsets pass. For each source, emits a per-module --- Run the offsets pass.
--- `<dir_basename>.offsets.h` containing `#define _atom_offset_F_T = N` --- For each source, emits a per-module `<dir_basename>.offsets.h` containing `#define _atom_offset_F_T = N` constants for every `atom_offset(F, T)` reference
--- constants for every `atom_offset(F, T)` reference in the source's atoms. --- in the source's atoms.
--- @param ctx PassCtx --- @param ctx PassCtx
--- @return PassResult --- @return PassResult
function M.run(ctx) function M.run(ctx)
+12 -22
View File
@@ -20,10 +20,8 @@
-- Module-scope requires + package.path setup -- Module-scope requires + package.path setup
-- ════════════════════════════════════════════════════════════════════════════ -- ════════════════════════════════════════════════════════════════════════════
-- Resolve `arg[0]` to an absolute-ish script directory so that -- Resolve `arg[0]` to an absolute-ish script directory so that `require("duffle")` resolves against `scripts/` regardless of CWD.
-- `require("duffle")` resolves against `scripts/` regardless of CWD. -- Note: this boilerplate is duplicated in 6 other entry scripts; a Phase-6 extraction target (`duffle.setup_package_path()`).
-- Note: this boilerplate is duplicated in 6 other entry scripts; a
-- Phase-6 extraction target (`duffle.setup_package_path()`).
-- Bootstrap: see `ps1_meta.lua` for the rationale. -- Bootstrap: see `ps1_meta.lua` for the rationale.
-- Bootstrap: load `scripts/duffle_paths.lua` (sets package.path + package.cpath). -- Bootstrap: load `scripts/duffle_paths.lua` (sets package.path + package.cpath).
-- Uses `debug.getinfo` to find this file's own directory, so it works -- Uses `debug.getinfo` to find this file's own directory, so it works
@@ -37,9 +35,8 @@ local duffle = require("duffle")
-- Constants -- Constants
-- ════════════════════════════════════════════════════════════════════════════ -- ════════════════════════════════════════════════════════════════════════════
-- Section separators used in the rendered text reports. The thin rules -- Section separators used in the rendered text reports.
-- are hand-tuned to align with the per-section content width; do not -- The thin rules are hand-tuned to align with the per-section content width; do not change without also checking the section renderers below.
-- change without also checking the section renderers below.
local RULE_THICK = "========================================================" local RULE_THICK = "========================================================"
local SECTION_HEADER_ATOMS = "── Atoms ────────────────────────────────────────────────" local SECTION_HEADER_ATOMS = "── Atoms ────────────────────────────────────────────────"
local SECTION_HEADER_ANNOTS = "── Annotations ──────────────────────────────────────────" local SECTION_HEADER_ANNOTS = "── Annotations ──────────────────────────────────────────"
@@ -147,8 +144,7 @@ local PASS_NAME = "report"
-- Per-MODULE annotation report (aggregated across all sources in a dir) -- Per-MODULE annotation report (aggregated across all sources in a dir)
-- ════════════════════════════════════════════════════════════════════════════ -- ════════════════════════════════════════════════════════════════════════════
-- Extract the basename (last path segment) of a forward- or back-slash -- Extract the basename (last path segment) of a forward- or back-slash separated path. Returns the input unchanged if no separator is found.
-- separated path. Returns the input unchanged if no separator is found.
-- @param path string -- @param path string
-- @return string -- @return string
local function source_basename(path) local function source_basename(path)
@@ -306,8 +302,7 @@ end
-- ════════════════════════════════════════════════════════════════════════════ -- ════════════════════════════════════════════════════════════════════════════
--- Render the per-project summary (`build/gen/annotation_validation.txt`). --- Render the per-project summary (`build/gen/annotation_validation.txt`).
--- Aggregates totals across all sources; lists per-source error counts --- Aggregates totals across all sources; lists per-source error counts if any source has errors.
--- if any source has errors.
--- @param all_results AnnotationResult[] --- @param all_results AnnotationResult[]
--- @return string --- @return string
local function render_project_report(all_results) local function render_project_report(all_results)
@@ -356,8 +351,7 @@ end
-- Orchestration helpers -- Orchestration helpers
-- ════════════════════════════════════════════════════════════════════════════ -- ════════════════════════════════════════════════════════════════════════════
-- Group source files by their `dir` field. Used to mirror the -- Group source files by their `dir` field. Used to mirror the per-DIRECTORY partitioning the annotation pass uses.
-- per-DIRECTORY partitioning the annotation pass uses.
-- @param sources SourceFile[] -- @param sources SourceFile[]
-- @return table<string, SourceFile[]> -- map of dir -> sources in that dir -- @return table<string, SourceFile[]> -- map of dir -> sources in that dir
local function group_sources_by_dir(sources) local function group_sources_by_dir(sources)
@@ -369,10 +363,8 @@ local function group_sources_by_dir(sources)
return by_dir return by_dir
end end
-- (internal) Validate each source in `dir_sources` via the annotation pass, -- (internal) Validate each source in `dir_sources` via the annotation pass, tagging each result with `result.source = src.path` for downstream rendering.
-- tagging each result with `result.source = src.path` for downstream rendering. -- Returns the list of module results + the flat list of all results (for the project-wide summary).
-- Returns the list of module results + the flat list of all results (for the
-- project-wide summary).
-- @param ctx PassCtx -- @param ctx PassCtx
-- @param dir_sources SourceFile[] -- @param dir_sources SourceFile[]
-- @return AnnotationResult[], AnnotationResult[] -- @return AnnotationResult[], AnnotationResult[]
@@ -416,9 +408,8 @@ end
local M = {} local M = {}
--- Run the report pass. Renders one `<dir_basename>.annotations.txt` --- Run the report pass.
--- per source-directory that has content, plus the project-wide --- Renders one `<dir_basename>.annotations.txt` per source-directory that has content, plus the project-wide `annotation_validation.txt` summary.
--- `annotation_validation.txt` summary.
--- @param ctx PassCtx --- @param ctx PassCtx
--- @return PassResult --- @return PassResult
function M.run(ctx) function M.run(ctx)
@@ -433,8 +424,7 @@ function M.run(ctx)
local all_results_for_summary = {} local all_results_for_summary = {}
for _, entry in ipairs(module_entries) do for _, entry in ipairs(module_entries) do
debug_log("entry: dir=%s basename=%s atoms_count=%d dir_sources=%d\n", debug_log("entry: dir=%s basename=%s atoms_count=%d dir_sources=%d\n", entry.dir, entry.dir_basename, entry.atoms_count, #(by_dir[entry.dir] or {}))
entry.dir, entry.dir_basename, entry.atoms_count, #(by_dir[entry.dir] or {}))
if entry.atoms_count > 0 or #(by_dir[entry.dir] or {}) > 0 then if entry.atoms_count > 0 or #(by_dir[entry.dir] or {}) > 0 then
local dir_sources = by_dir[entry.dir] or {} local dir_sources = by_dir[entry.dir] or {}
+39 -73
View File
@@ -1,57 +1,32 @@
--- passes/static_analysis.lua — Per-atom static-analysis checks. --- passes/static_analysis.lua — Per-atom static-analysis checks.
--- ---
--- The 5 checks currently shipped: --- The 5 checks currently shipped:
--- 1. **GTE pipeline-fill** — every `gte_cmdw_*` invocation must be --- 1. **GTE pipeline-fill** — every `gte_cmdw_*` invocation must be preceded by the minimum number of `nop` words
--- preceded by the minimum number of `nop` words (per --- (per `duffle.GTE_PIPELINE_LATENCY`) so the COP2 pipeline latency is fully retired before the command issues.
--- `duffle.duffle.GTE_PIPELINE_LATENCY`) so the COP2 pipeline latency is --- 2. **mac_yield uniformity** — every atom body must contain exactly one `mac_yield()` call (control transfer pattern).
--- fully retired before the command issues. --- 3. **ABI handoff** — every `atom_bind(Binds_X)` must reference a `typedef Struct_(Binds_X) { ... }` declaration.
--- 2. **mac_yield uniformity** — every atom body must contain exactly --- 4. **GPU port-store shape** — per-shape (`f3`/`f4`/`g4`/etc.) the sum of `mac_format_X_color` + `mac_gte_store_X_*` +
--- one `mac_yield()` call (control transfer pattern). --- `mac_insert_ot_tag_X` words must equal the GP0 cmd's expected packet size.
--- 3. **ABI handoff** — every `atom_bind(Binds_X)` must reference a --- 5. **per-atom cycle budget** — sum each atom body's instruction latencies (per `duffle.INSTRUCTION_LATENCY`); report total.
--- `typedef Struct_(Binds_X) { ... }` declaration.
--- 4. **GPU port-store shape** — per-shape (`f3`/`f4`/`g4`/etc.) the
--- sum of `mac_format_X_color` + `mac_gte_store_X_*` +
--- `mac_insert_ot_tag_X` words must equal the GP0 cmd's expected
--- packet size.
--- 5. **per-atom cycle budget** — sum each atom body's instruction
--- latencies (per `duffle.duffle.INSTRUCTION_LATENCY`); report total.
--- ---
--- The orchestrator (`ps1_meta.lua`) wires this module in via the --- The orchestrator (`ps1_meta.lua`) wires this module in via the
--- PASSES table: --- PASSES table:
--- `["static-analysis"] = { module = "passes.static_analysis", --- `["static-analysis"] = { module = "passes.static_analysis", kind = "validation", deps = {"word-counts", "components"},
--- kind = "validation", --- out = { { kind = "report", path_template = "<out_root>/<basename>.static_analysis.txt" } } }`
--- deps = {"word-counts", "components"},
--- out = { { kind = "report",
--- path_template = "<out_root>/<basename>.static_analysis.txt" } } }`
--- ---
--- **Conventions**: tabs (1/level), EmmyLua annotations, no regex, --- **Conventions**: tabs (1/level), EmmyLua annotations, no regex, Lua 5.3 compatible. See `lua.md` in the ps1-ai styleguides.
--- Lua 5.3 compatible. See
--- `C:\projects\Pikuma\ps1-ai\conductor\code_styleguides\lua.md`.
-- ════════════════════════════════════════════════════════════════════════════ -- ════════════════════════════════════════════════════════════════════════════
-- Module-scope requires + package.path setup -- Module-scope requires + package.path setup
-- ════════════════════════════════════════════════════════════════════════════ -- ════════════════════════════════════════════════════════════════════════════
-- `duffle.setup_package_path()` resolves `arg[0]` and prepends `scripts/`
-- (and `scripts/passes/`) to `package.path`, so `require("duffle")`
-- resolves regardless of CWD. See `duffle.lua` for the implementation.
-- Bootstrap: see `ps1_meta.lua` for the rationale.
-- Bootstrap: load `scripts/duffle_paths.lua` (sets package.path + package.cpath). -- Bootstrap: load `scripts/duffle_paths.lua` (sets package.path + package.cpath).
-- Uses `debug.getinfo` to find this file's own directory, so it works -- Uses `debug.getinfo` to find this file's own directory, so it works both standalone and when require'd from the orchestrator.
-- both standalone and when require'd from the orchestrator.
local _src = debug.getinfo(1, "S").source:sub(2) local _src = debug.getinfo(1, "S").source:sub(2)
local _dir = _src:match("(.*[/\\])") or "./" local _dir = _src:match("(.*[/\\])") or "./"
dofile(_dir .. "../duffle_paths.lua") dofile(_dir .. "../duffle_paths.lua")
local duffle = require("duffle") local duffle = require("duffle")
-- Domain tables (single source of truth in duffle.lua).
-- ════════════════════════════════════════════════════════════════════════════ -- ════════════════════════════════════════════════════════════════════════════
-- Constants -- Constants
-- ════════════════════════════════════════════════════════════════════════════ -- ════════════════════════════════════════════════════════════════════════════
@@ -137,7 +112,7 @@ local OUTPUT_EXTENSION = ".static_analysis.txt"
--- @field atom AtomBody --- @field atom AtomBody
--- @field tokens Token[] -- the tokens in the atom body, annotated --- @field tokens Token[] -- the tokens in the atom body, annotated
--- @field findings Finding[] -- findings for this atom --- @field findings Finding[] -- findings for this atom
--- @field total_cycles integer -- sum of token cycle costs (Phase 3) --- @field total_cycles integer -- sum of token cycle costs
-- ════════════════════════════════════════════════════════════════════════════ -- ════════════════════════════════════════════════════════════════════════════
-- Source walkers -- Source walkers
@@ -232,7 +207,7 @@ local function find_atom_bodies(source_text)
elseif c == 91 then elseif c == 91 then
local _, a = duffle.read_brackets(inner, inner_pos); inner_pos = a local _, a = duffle.read_brackets(inner, inner_pos); inner_pos = a
elseif c == 34 or c == 39 then elseif c == 34 or c == 39 then
inner_pos = duffle.duffle.skip_str_or_cmt(inner, inner_pos) + 1 inner_pos = duffle.skip_str_or_cmt(inner, inner_pos) + 1
else else
inner_pos = inner_pos + 1 inner_pos = inner_pos + 1
end end
@@ -381,7 +356,7 @@ local function tokenize_body(body)
elseif c == 91 then -- '[' elseif c == 91 then -- '['
local _, a = duffle.read_brackets(body, scan); scan = a local _, a = duffle.read_brackets(body, scan); scan = a
elseif c == 34 or c == 39 then -- '"' or '\'' elseif c == 34 or c == 39 then -- '"' or '\''
scan = duffle.duffle.skip_str_or_cmt(body, scan) + 1 scan = duffle.skip_str_or_cmt(body, scan) + 1
else else
scan = scan + 1 scan = scan + 1
end end
@@ -449,7 +424,7 @@ local function check_gte_pipeline_fill(atoms, findings, line_of)
check = "gte_pipeline_fill", check = "gte_pipeline_fill",
kind = "warning", kind = "warning",
msg = string.format( msg = string.format(
"%s at line %d uses `gte_cmdw_%s` but that macro is not in duffle.duffle.GTE_PIPELINE_LATENCY -- add a min_nops entry", "%s at line %d uses `gte_cmdw_%s` but that macro is not in duffle.GTE_PIPELINE_LATENCY -- add a min_nops entry",
a.name, line, variant), a.name, line, variant),
} }
ti = ti + 1 ti = ti + 1
@@ -950,7 +925,9 @@ local function check_gpu_portstore_shape(atoms, findings)
findings[#findings + 1] = { findings[#findings + 1] = {
atom = a.name, line = a.line, atom = a.name, line = a.line,
check = "gpu_portstore_shape", kind = "warning", check = "gpu_portstore_shape", kind = "warning",
msg = string.format("%s at line %d writes to R_PrimCursor via raw store_word(...) but uses no `mac_format_*_color`; the cmd byte + word count cannot be auto-validated. Consider migrating to `mac_format_X_color` + `mac_gte_store_X_post_*` + `mac_insert_ot_tag_X`.", msg = string.format("%s at line %d writes to R_PrimCursor via raw store_word(...)"
.. " but uses no `mac_format_*_color`; the cmd byte + word count cannot be auto-validated."
.. " Consider migrating to `mac_format_X_color` + `mac_gte_store_X_post_*` + `mac_insert_ot_tag_X`.",
a.name, a.line), a.name, a.line),
} }
end end
@@ -970,7 +947,7 @@ local function check_gpu_portstore_shape(atoms, findings)
end end
-- ════════════════════════════════════════════════════════════════════════════ -- ════════════════════════════════════════════════════════════════════════════
-- Check #5: per-atom cycle budget (Phase 3) -- Check #5: per-atom cycle budget
-- ════════════════════════════════════════════════════════════════════════════ -- ════════════════════════════════════════════════════════════════════════════
--- Compute the cycle cost of one token. The token is a string like --- Compute the cycle cost of one token. The token is a string like
@@ -1179,8 +1156,8 @@ local function analyze_atom_paths(atom)
end end
if n >= 1 then dfs(1, 0, {}) end if n >= 1 then dfs(1, 0, {}) end
-- cycles_full: sum of every token's cost (the previous model; useful -- cycles_full: sum of every token's cost (the legacy sum-of-all-tokens
-- for comparing against the path-aware min/max). -- value; over-counts BD-slot nops relative to the path-aware min/max).
local cycles_full = 0 local cycles_full = 0
for tok_idx = 1, n do cycles_full = cycles_full + costs[tok_idx] end for tok_idx = 1, n do cycles_full = cycles_full + costs[tok_idx] end
@@ -1210,9 +1187,9 @@ local function analyze_atom_paths(atom)
} }
end end
--- Backward-compat wrapper: returns total cycle count (the previous --- Returns total cycle count (the sum-of-all-tokens value, which over-counts
--- "best case" value, which over-counts BD-slot nops) + unknown macro --- BD-slot nops) + unknown macro list. New code should call
--- list. New code should call `analyze_atom_paths(atom)` instead. --- `analyze_atom_paths(atom)` instead.
local function count_atom_cycles(atom) local function count_atom_cycles(atom)
local tokens = tokenize_body(atom.body) local tokens = tokenize_body(atom.body)
local total = 0 local total = 0
@@ -1242,7 +1219,8 @@ local function check_per_atom_cycle_budget(atoms, findings)
findings[#findings + 1] = { findings[#findings + 1] = {
atom = a.name, line = a.line, atom = a.name, line = a.line,
check = "per_atom_cycle_budget", kind = "warning", check = "per_atom_cycle_budget", kind = "warning",
msg = string.format("%s at line %d uses macro `%s` which is not in duffle.INSTRUCTION_LATENCY; cycle count will be +%d per call (best-case). Add an entry to duffle.M.duffle.INSTRUCTION_LATENCY.", msg = string.format("%s at line %d uses macro `%s` which is not in duffle.INSTRUCTION_LATENCY; "
.. "cycle count will be +%d per call (best-case). Add an entry to duffle.INSTRUCTION_LATENCY.",
a.name, a.line, name, duffle.UNKNOWN_INSTRUCTION_CYCLES), a.name, a.line, name, duffle.UNKNOWN_INSTRUCTION_CYCLES),
} }
end end
@@ -1274,7 +1252,7 @@ local function validate(ctx, src)
check_gpu_portstore_shape(atoms, findings) check_gpu_portstore_shape(atoms, findings)
check_per_atom_cycle_budget(atoms, findings) check_per_atom_cycle_budget(atoms, findings)
-- Phase 3 cycle-budget output: attach per-path cycle data to each -- Path-aware cycle-budget output: attach per-path cycle data to each
-- atom. Best-case (no-stall) cycle count with BD-slot absorbed; the -- atom. Best-case (no-stall) cycle count with BD-slot absorbed; the
-- `cycles_full` field is the legacy sum-of-all-tokens value (kept -- `cycles_full` field is the legacy sum-of-all-tokens value (kept
-- for backward compat; over-counts BD-slot nops). -- for backward compat; over-counts BD-slot nops).
@@ -1322,7 +1300,7 @@ local function validate(ctx, src)
} }
end end
-- Phase 3: cycle-budget summary line. Per-path min/max totals. -- Path-aware cycle-budget summary line. Per-path min/max totals.
if #atoms > 0 then if #atoms > 0 then
local total_min = 0 local total_min = 0
local total_max = 0 local total_max = 0
@@ -1432,10 +1410,9 @@ local function emit_static_analysis_txt(ctx, src, result)
return out_path return out_path
end end
-- (Old per-source emit function above kept for backward compat but no -- (Old per-source emit function above; kept for backward compat but no
-- longer called from M.run; replaced by `emit_module_static_analysis_txt` -- longer called from M.run. Replaced by `emit_module_static_analysis_txt`
-- which aggregates by directory. Kept because some test harnesses may -- which aggregates by directory.)
-- still call it directly.)
--- Per-directory emit. Aggregates atoms + findings across every source --- Per-directory emit. Aggregates atoms + findings across every source
--- in `dir_sources` and writes a single report to --- in `dir_sources` and writes a single report to
@@ -1518,15 +1495,13 @@ local function emit_module_static_analysis_txt(ctx, dir, dir_sources, atoms, fin
add(string.format(" ! line %d %s", w.line, w.msg)) add(string.format(" ! line %d %s", w.line, w.msg))
end end
-- Per-atom cycle counts (Phase 3 path-aware). For each atom: -- Per-atom cycle counts (path-aware). For each atom:
-- min = shortest path through the body (earliest exit) -- min = shortest path through the body (earliest exit)
-- max = longest path through the body (full fall-through) -- max = longest path through the body (full fall-through)
-- br = number of branch instructions -- br = number of branch instructions
-- paths = number of distinct paths reached -- paths = number of distinct paths reached
-- Both min and max are best-case (no stalls); BD-slot nops are -- Both min and max are best-case (no stalls); BD-slot nops are
-- absorbed into branch costs (MIPS semantics). The previous "best -- absorbed into branch costs (MIPS semantics).
-- case" model counted every token separately, which double-counted
-- BD-slot nops; the path-aware model is the MIPS-accurate value.
add("") add("")
add("── Per-atom cycle counts (path-aware, best case, no stalls) ─") add("── Per-atom cycle counts (path-aware, best case, no stalls) ─")
if #atoms == 0 then if #atoms == 0 then
@@ -1569,13 +1544,6 @@ local function emit_module_static_analysis_txt(ctx, dir, dir_sources, atoms, fin
-- 0 atoms are skipped (they're just header files that declared -- 0 atoms are skipped (they're just header files that declared
-- no MipsAtom_ — they're already listed in the module's -- no MipsAtom_ — they're already listed in the module's
-- "Sources:" section above). -- "Sources:" section above).
--
-- TODO: per-source finding attribution. Currently we can't tell
-- which source a given error/warning came from (errors/warnings
-- only carry atom-name + line, not source-path). The per-atom
-- cycle section already shows which atoms are in which source
-- via the `(file_basename)` suffix. Adding source attribution to
-- error/warning would be a future enhancement.
for _, src in ipairs(dir_sources) do for _, src in ipairs(dir_sources) do
local src_atoms = {} local src_atoms = {}
for _, a in ipairs(atoms) do for _, a in ipairs(atoms) do
@@ -1640,10 +1608,10 @@ function M.run(ctx)
local errors = {} local errors = {}
local warnings = {} local warnings = {}
-- Phase 3.7+: aggregate per-DIRECTORY (per-module). One -- Aggregate per-DIRECTORY (per-module). One static_analysis.txt per
-- static_analysis.txt per source-directory, emitted only if the -- source-directory, emitted only if the directory contains at least
-- directory contains at least one atom. Empty-source directories -- one atom. Empty-source directories (e.g. duffle headers with no
-- (e.g. duffle headers with no atoms) produce no report. -- atoms) produce no report.
-- --
-- Group sources by `src.dir`. The first component of `dir` is the -- Group sources by `src.dir`. The first component of `dir` is the
-- module name (e.g. "code/duffle" -> "duffle", "code/gte_hello" -> -- module name (e.g. "code/duffle" -> "duffle", "code/gte_hello" ->
@@ -1689,10 +1657,8 @@ function M.run(ctx)
end end
end end
-- Skip directories with zero atoms. The previous behavior emitted -- Skip directories with zero atoms — a directory with only
-- a "<no atoms>" report per source; the new behavior emits nothing -- headers / no MipsAtom_ is "nothing to report".
-- at all (a directory with only headers / no MipsAtom_ is
-- "nothing to report").
if #all_atoms == 0 then if #all_atoms == 0 then
-- Still aggregate errors/warnings so orchestrator sees them, -- Still aggregate errors/warnings so orchestrator sees them,
-- but don't write a file. -- but don't write a file.
+37 -67
View File
@@ -1,32 +1,25 @@
--- word_count_eval.lua — Word-counting logic for the tape-atom metaprogram --- word_count_eval.lua — Word-counting logic for the tape-atom metaprogram pipeline.
--- pipeline.
--- ---
--- Three responsibilities: --- Three responsibilities:
--- 1. **Public utilities** (used by `passes/components.lua`, --- 1. **Public utilities** (used by `passes/components.lua`, `passes/offsets.lua`, `passes/annotation.lua`):
--- `passes/offsets.lua`, `passes/annotation.lua`):
--- - `M.count_token_words(token, wc)` — words emitted by one token --- - `M.count_token_words(token, wc)` — words emitted by one token
--- - `M.scan_dir(dir, suffix)` — glob walk for *.macs.h --- - `M.scan_dir(dir, suffix)` — glob walk for *.macs.h
--- - `M.count_body_words(body, wc)` — words emitted by an atom body --- - `M.count_body_words(body, wc)` — words emitted by an atom body
--- 2. **Pass entry** `M.run(ctx)` — loads metadata.h + *.macs.h into --- 2. **Pass entry** `M.run(ctx)` — loads metadata.h + *.macs.h into `ctx.shared.word_counts` for downstream passes.
--- `ctx.shared.word_counts` for downstream passes.
--- 3. **Internal helpers** for the body scanner. --- 3. **Internal helpers** for the body scanner.
--- ---
--- **Conventions**: tabs (1/level), EmmyLua annotations, no regex, --- **Conventions**: tabs (1/level), EmmyLua annotations, no regex,
--- Lua 5.3 compatible. See --- Lua 5.3 compatible.
--- `C:\projects\Pikuma\ps1-ai\conductor\code_styleguides\lua.md`.
-- ════════════════════════════════════════════════════════════════════════════ -- ════════════════════════════════════════════════════════════════════════════
-- Module-scope requires + package.path setup -- Module-scope requires + package.path setup
-- ════════════════════════════════════════════════════════════════════════════ -- ════════════════════════════════════════════════════════════════════════════
-- Resolve `arg[0]` to an absolute-ish script directory so that -- Resolve `arg[0]` to an absolute-ish script directory so that `require("duffle")` resolves against `scripts/` regardless of CWD.
-- `require("duffle")` resolves against `scripts/` regardless of CWD. -- Note: this boilerplate is duplicated in 6 other entry scripts; a Phase-6 extraction target (`duffle.setup_package_path()`).
-- Note: this boilerplate is duplicated in 6 other entry scripts; a
-- Phase-6 extraction target (`duffle.setup_package_path()`).
-- Bootstrap: see `ps1_meta.lua` for the rationale. -- Bootstrap: see `ps1_meta.lua` for the rationale.
-- Bootstrap: load `scripts/duffle_paths.lua` (sets package.path + package.cpath). -- Bootstrap: load `scripts/duffle_paths.lua` (sets package.path + package.cpath).
-- Uses `debug.getinfo` to find this file's own directory, so it works -- Uses `debug.getinfo` to find this file's own directory, so it works both standalone and when require'd from the orchestrator.
-- both standalone and when require'd from the orchestrator.
local _src = debug.getinfo(1, "S").source:sub(2) local _src = debug.getinfo(1, "S").source:sub(2)
local _dir = _src:match("(.*[/\\])") or "./" local _dir = _src:match("(.*[/\\])") or "./"
dofile(_dir .. "../duffle_paths.lua") dofile(_dir .. "../duffle_paths.lua")
@@ -36,14 +29,12 @@ local duffle = require("duffle")
-- Constants -- Constants
-- ════════════════════════════════════════════════════════════════════════════ -- ════════════════════════════════════════════════════════════════════════════
-- Windows separator chars — used to convert `dir /b /s` output (which uses -- Windows separator chars — used to convert `dir /b /s` output (which uses `\`) into POSIX paths (which our scripts expect).
-- `\`) into POSIX paths (which our scripts expect).
local PATH_SEP_BACKSLASH = "\\" local PATH_SEP_BACKSLASH = "\\"
local PATH_SEP_FORWARD = "/" local PATH_SEP_FORWARD = "/"
-- Glob command for Windows directory walk. `dir /b /s` lists all matching -- Glob command for Windows directory walk. `dir /b /s` lists all matching files recursively with bare paths (no headers);
-- files recursively with bare paths (no headers); `2>nul` discards the -- `2>nul` discards the "file not found" stderr when nothing matches.
-- "file not found" stderr when nothing matches.
local DIR_GLOB_CMD = 'dir /b /s "%s\\%s" 2>nul' local DIR_GLOB_CMD = 'dir /b /s "%s\\%s" 2>nul'
-- ════════════════════════════════════════════════════════════════════════════ -- ════════════════════════════════════════════════════════════════════════════
@@ -88,8 +79,7 @@ local M = {}
--- Count words emitted by a single comma-separated token inside an atom body. --- Count words emitted by a single comma-separated token inside an atom body.
--- For most tokens (regular MIPS instructions) this returns 1. --- For most tokens (regular MIPS instructions) this returns 1.
--- For `mac_X(...)` calls, this returns the resolved word count from `wc` --- For `mac_X(...)` calls, this returns the resolved word count from `wc` (recursively if needed). For `nop2` etc., returns wc[name].
--- (recursively if needed). For `nop2` etc., returns wc[name].
--- For unknown macros, returns 1 and (optionally) warns. --- For unknown macros, returns 1 and (optionally) warns.
--- ---
--- @param token string -- a single token from split_top_level_commas --- @param token string -- a single token from split_top_level_commas
@@ -113,30 +103,25 @@ end
-- └────────────────────────────────────────────────────────────────────┘ -- └────────────────────────────────────────────────────────────────────┘
--- Recursively scan a directory for files matching a glob suffix. --- Recursively scan a directory for files matching a glob suffix.
--- No regex per the no_regex constraint — uses plain byte matching --- No regex per the no_regex constraint — uses plain byte matching via `dir /b /s` on Windows.
--- via `dir /b /s` on Windows.
--- ---
--- The `.macs.h` files produced by the components pass always live at --- The `.macs.h` files produced by the components pass always live at `<project_root>/<module>/gen/`.
--- `<project_root>/<module>/gen/`. We can shortcut the `dir /b /s` walk by --- We can shortcut the `dir /b /s` walk by listing modules first (one `dir /b /ad`), then walking each `<module>/gen/`
--- listing modules first (one `dir /b /ad`), then walking each `<module>/gen/` --- (one `dir /b` per module, no recursion).
--- (one `dir /b` per module, no recursion). For projects with 2 modules and --- For projects with 2 modules and 0 .macs.h files, this drops the cost from ~52ms
--- 0 .macs.h files, this drops the cost from ~52ms (full recursive walk of --- (full recursive walk of the entire project tree) to ~5ms.
--- the entire project tree) to ~5ms.
--- ---
--- @param dir string -- directory to scan (absolute or relative) --- @param dir string -- directory to scan (absolute or relative)
--- @param suffix string -- file pattern, e.g. "*.macs.h" --- @param suffix string -- file pattern, e.g. "*.macs.h"
--- @return string[] --- @return string[]
-- Cache the scan_dir result per (dir, suffix) in package.loaded. Each -- Cache the scan_dir result per (dir, suffix) in package.loaded.
-- `io.popen` call on Windows is ~50-100ms of subprocess overhead, so -- Each `io.popen` call on Windows is ~50-100ms of subprocess overhead, so caching the result saves a fixed cost on every build.
-- caching the result saves a fixed cost on every build. The cache -- The cache persists for the lifetime of the Lua process (cleared when ps1_meta.lua exits).
-- persists for the lifetime of the Lua process (cleared when ps1_meta.lua -- If a build removes/creates .macs.h files mid-process, the caller can invalidate by calling `M._invalidate_scan_cache()`.
-- exits). If a build removes/creates .macs.h files mid-process, the
-- caller can invalidate by calling `M._invalidate_scan_cache()`.
local SCAN_CACHE_KEY = "__word_count_eval_scan_cache__" local SCAN_CACHE_KEY = "__word_count_eval_scan_cache__"
--- Recursively scan a directory for files matching a glob suffix. --- Recursively scan a directory for files matching a glob suffix.
--- No regex per the no_regex constraint — uses plain byte matching --- No regex per the no_regex constraint — uses plain byte matching via `dir /b /s` on Windows.
--- via `dir /b /s` on Windows.
--- ---
--- @param dir string -- directory to scan (absolute or relative) --- @param dir string -- directory to scan (absolute or relative)
--- @param suffix string -- file pattern, e.g. "*.macs.h" --- @param suffix string -- file pattern, e.g. "*.macs.h"
@@ -144,20 +129,15 @@ local SCAN_CACHE_KEY = "__word_count_eval_scan_cache__"
function M.scan_dir(dir, suffix) function M.scan_dir(dir, suffix)
local key = dir .. "\0" .. suffix local key = dir .. "\0" .. suffix
-- Check the in-process cache first. (Mostly helps when a build -- Check the in-process cache first. (Mostly helps when a build triggers multiple `M.run` calls -- e.g.
-- triggers multiple `M.run` calls -- e.g. the audit_lua_nesting -- the audit_lua_nesting script's stress tests but the cost is ~free either way.)
-- script's stress tests -- but the cost is ~free either way.)
local cache = package.loaded[SCAN_CACHE_KEY] local cache = package.loaded[SCAN_CACHE_KEY]
if cache and cache[key] then if cache and cache[key] then return cache[key] end
return cache[key]
end
local results = {} local results = {}
local pipe = io.popen(DIR_GLOB_CMD:format(dir, suffix)) local pipe = io.popen(DIR_GLOB_CMD:format(dir, suffix))
if not pipe then if not pipe then
-- Cache the empty result too (avoids re-scan if the dir is -- Cache the empty result too (avoids re-scan if the dir is genuinely empty -- e.g. a clean build before components has run yet).
-- genuinely empty -- e.g. a clean build before components
-- has run yet).
cache = cache or {} cache = cache or {}
cache[key] = results cache[key] = results
package.loaded[SCAN_CACHE_KEY] = cache package.loaded[SCAN_CACHE_KEY] = cache
@@ -177,11 +157,8 @@ function M.scan_dir(dir, suffix)
return results return results
end end
--- Invalidate the scan cache (call after creating new .macs.h files --- Invalidate the scan cache (call after creating new .macs.h files in the same Lua process — usually not needed).
--- in the same Lua process — usually not needed). function M._invalidate_scan_cache() package.loaded[SCAN_CACHE_KEY] = nil end
function M._invalidate_scan_cache()
package.loaded[SCAN_CACHE_KEY] = nil
end
-- ┌────────────────────────────────────────────────────────────────────┐ -- ┌────────────────────────────────────────────────────────────────────┐
-- │ Shared utility: count_body_words │ -- │ Shared utility: count_body_words │
@@ -189,9 +166,8 @@ end
--- Count words emitted by an entire atom body (a brace-delimited block). --- Count words emitted by an entire atom body (a brace-delimited block).
--- Splits by top-level commas; for each token, delegates to count_token_words. --- Splits by top-level commas; for each token, delegates to count_token_words.
--- Handles `atom_label(name)` / `atom_offset(tag, name)` markers (record at --- Handles `atom_label(name)` / `atom_offset(tag, name)` markers
--- current pos, do NOT advance pos; if the marker call bundles an instruction --- (record at current pos, do NOT advance pos; if the marker call bundles an instruction after it, count that instruction too).
--- after it, count that instruction too).
--- ---
--- @param body string -- brace-delimited atom body (without braces) --- @param body string -- brace-delimited atom body (without braces)
--- @param wc WordCounts -- the shared word-count table --- @param wc WordCounts -- the shared word-count table
@@ -208,10 +184,8 @@ function M.count_body_words(body, wc)
local is_marker = leading_ident == "atom_label" or leading_ident == "atom_offset" local is_marker = leading_ident == "atom_label" or leading_ident == "atom_offset"
if is_marker then if is_marker then
-- Marker call: record at current pos, do NOT advance pos. -- Marker call: record at current pos, do NOT advance pos.
-- But the source pattern may bundle the marker with the next -- But the source pattern may bundle the marker with the next instruction on a new line (no top-level comma between them).
-- instruction on a new line (no top-level comma between them). -- In that case, the rest of `tok` after the marker call is a real instruction that must still be counted.
-- In that case, the rest of `tok` after the marker call is
-- a real instruction that must still be counted.
local marker_end = M.find_marker_call_end(tok) local marker_end = M.find_marker_call_end(tok)
if marker_end > 0 and marker_end < #tok then if marker_end > 0 and marker_end < #tok then
local rest = duffle.trim(tok:sub(marker_end + 1)) local rest = duffle.trim(tok:sub(marker_end + 1))
@@ -226,8 +200,7 @@ function M.count_body_words(body, wc)
return total return total
end end
--- Find the end position (just past the closing ')') of the first --- Find the end position (just past the closing ')') of the first atom_label/atom_offset call in `tok`. Returns 0 if no such call.
--- atom_label/atom_offset call in `tok`. Returns 0 if no such call.
--- Internal helper for count_body_words. --- Internal helper for count_body_words.
--- ---
--- @param tok string --- @param tok string
@@ -255,8 +228,8 @@ function M.find_marker_call_end(tok)
return 0 return 0
end end
-- (internal) If `ident` is `atom_label`/`atom_offset` followed by `(...)`, -- (internal) If `ident` is `atom_label`/`atom_offset` followed by `(...)`, return the position just past the closing ')'.
-- return the position just past the closing ')'. Otherwise 0. -- Otherwise 0.
-- @param tok string -- @param tok string
-- @param ident string|nil -- @param ident string|nil
-- @param after_ident integer -- @param after_ident integer
@@ -273,10 +246,8 @@ end
-- │ Pass entry: M.run(ctx) — "word-counts" pass │ -- │ Pass entry: M.run(ctx) — "word-counts" pass │
-- └────────────────────────────────────────────────────────────────────┘ -- └────────────────────────────────────────────────────────────────────┘
--- Load metadata.h + scan for existing *.macs.h files into --- Load metadata.h + scan for existing *.macs.h files into ctx.shared.word_counts.
--- ctx.shared.word_counts. Loading the .macs.h files is idempotent: --- Loading the .macs.h files is idempotent: entries from later (current-build) .macs.h files override metadata.h entries of the same name.
--- entries from later (current-build) .macs.h files override
--- metadata.h entries of the same name.
--- ---
--- @param ctx PassCtx --- @param ctx PassCtx
--- @return PassResult --- @return PassResult
@@ -301,7 +272,6 @@ function M.run(ctx)
end end
ctx.shared.word_counts = wc ctx.shared.word_counts = wc
return { outputs = {}, errors = {}, warnings = {} } return { outputs = {}, errors = {}, warnings = {} }
end end
+30 -71
View File
@@ -79,9 +79,7 @@ local PASS_FLAG_DISPATCH_KEY = "__pass__"
--- @class PassOutputEntry --- @class PassOutputEntry
--- @field [string] string -- dynamic shape; key is the output kind --- @field [string] string -- dynamic shape; key is the output kind
-- (e.g. "macs_h", "offsets_h", "errors_h", -- (e.g. "macs_h", "offsets_h", "errors_h", "annotations_txt", "static_analysis_txt", "summary_txt"), value is the path
-- "annotations_txt", "static_analysis_txt",
-- "summary_txt"), value is the path
--- @class Finding --- @class Finding
--- @field line integer -- source line (or 0 for pass-level) --- @field line integer -- source line (or 0 for pass-level)
@@ -177,8 +175,7 @@ local ALL_PASS_NAMES = {
"offsets", "static-analysis", "report", "offsets", "static-analysis", "report",
} }
--- Append every pass name to args.requested_set. Used by --all and --- Append every pass name to args.requested_set. Used by --all and by the "default to --all if no pass flags were given" fallback.
--- by the "default to --all if no pass flags were given" fallback.
--- @param args ParsedArgs --- @param args ParsedArgs
local function request_all_passes(args) local function request_all_passes(args)
for _, n in ipairs(ALL_PASS_NAMES) do for _, n in ipairs(ALL_PASS_NAMES) do
@@ -186,11 +183,9 @@ local function request_all_passes(args)
end end
end end
-- Per-flag handlers. Each handler takes (args, argv, arg_idx) and -- Per-flag handlers. Each handler takes (args, argv, arg_idx) and returns the new arg_idx (so multi-arg flags like --source FILE advance it).
-- returns the new arg_idx (so multi-arg flags like --source FILE -- Returning nil + os.exit() handles termination flags (--help). This replaces the 8-way `if/elseif/elseif...` chain that nested 4 levels deep
-- advance it). Returning nil + os.exit() handles termination flags -- and made the dispatch logic hard to scan.
-- (--help). This replaces the 8-way `if/elseif/elseif...` chain
-- that nested 4 levels deep and made the dispatch logic hard to scan.
local FLAG_HANDLERS = {} local FLAG_HANDLERS = {}
-- ════════════════════════════════════════════════════════════════════════════ -- ════════════════════════════════════════════════════════════════════════════
@@ -233,50 +228,25 @@ EXAMPLE:
]]) ]])
end end
-- Per-flag handlers. Each takes (args, argv, arg_idx) and returns -- Per-flag handlers. Each takes (args, argv, arg_idx) and returns the new arg_idx (so multi-arg flags like --source FILE advance it).
-- the new arg_idx (so multi-arg flags like --source FILE advance -- Termination flags like --help call os.exit() instead.
-- it). Termination flags like --help call os.exit() instead. This -- This replaces the 8-way `if/elseif/elseif...` chain that nested 4 levels deep and made the dispatch logic hard to scan.
-- replaces the 8-way `if/elseif/elseif...` chain that nested 4
-- levels deep and made the dispatch logic hard to scan.
-- --
-- Populated AFTER print_help so the --help handler can reference it -- Populated AFTER print_help so the --help handler can reference it as an upvalue (Lua resolves locals at closure-call time,
-- as an upvalue (Lua resolves locals at closure-call time, but if the -- but if the closure is defined before the local, it falls back to _G).
-- closure is defined before the local, it falls back to _G).
FLAG_HANDLERS["--help"] = function(args) FLAG_HANDLERS["--help"] = function(args)
print_help() print_help()
os.exit(0) os.exit(0)
end end
FLAG_HANDLERS["--dry-run"] = function(args) FLAG_HANDLERS["--dry-run"] = function(args) args.dry_run = true end
args.dry_run = true FLAG_HANDLERS["--verbose"] = function(args) args.verbose = true end
end FLAG_HANDLERS["--source"] = function(args, argv, arg_idx) args.sources[#args.sources + 1] = argv[arg_idx + 1]; return arg_idx + 1 end
FLAG_HANDLERS["--metadata"] = function(args, argv, arg_idx) args.metadata = argv[arg_idx + 1]; return arg_idx + 1 end
FLAG_HANDLERS["--out-root"] = function(args, argv, arg_idx) args.out_root = argv[arg_idx + 1]; return arg_idx + 1 end
FLAG_HANDLERS["--project-root"] = function(args, argv, arg_idx) args.project_root = argv[arg_idx + 1]; return arg_idx + 1 end
FLAG_HANDLERS["--verbose"] = function(args) -- Pass-flag handler. Reads the closed-set table, expands --all, appends to requested_set. Single-statement, no nesting.
args.verbose = true
end
FLAG_HANDLERS["--source"] = function(args, argv, arg_idx)
args.sources[#args.sources + 1] = argv[arg_idx + 1]
return arg_idx + 1
end
FLAG_HANDLERS["--metadata"] = function(args, argv, arg_idx)
args.metadata = argv[arg_idx + 1]
return arg_idx + 1
end
FLAG_HANDLERS["--out-root"] = function(args, argv, arg_idx)
args.out_root = argv[arg_idx + 1]
return arg_idx + 1
end
FLAG_HANDLERS["--project-root"] = function(args, argv, arg_idx)
args.project_root = argv[arg_idx + 1]
return arg_idx + 1
end
-- Pass-flag handler. Reads the closed-set table, expands --all,
-- appends to requested_set. Single-statement, no nesting.
FLAG_HANDLERS[PASS_FLAG_DISPATCH_KEY] = function(args, a) FLAG_HANDLERS[PASS_FLAG_DISPATCH_KEY] = function(args, a)
local name = PASS_FLAG_TO_NAME[a] local name = PASS_FLAG_TO_NAME[a]
if name == ALL_PASSES_SENTINEL then if name == ALL_PASSES_SENTINEL then
@@ -318,9 +288,7 @@ local function parse_args(argv)
end end
-- Default: --all if no explicit pass flags. -- Default: --all if no explicit pass flags.
if #args.requested_set == 0 then if #args.requested_set == 0 then request_all_passes(args) end
request_all_passes(args)
end
-- Defaults: project_root = dirname(metadata). -- Defaults: project_root = dirname(metadata).
if args.metadata and not args.project_root then if args.metadata and not args.project_root then
@@ -394,8 +362,7 @@ end
-- Topological sort (Kahn's algorithm + cycle detection) -- Topological sort (Kahn's algorithm + cycle detection)
-- ════════════════════════════════════════════════════════════════════════════ -- ════════════════════════════════════════════════════════════════════════════
--- Compute the dep-closure of `requested_set`: include every pass name --- Compute the dep-closure of `requested_set`: include every pass name transitively required by the requested set.
--- transitively required by the requested set.
--- ---
--- @param passes table<string, PassDescriptor> --- @param passes table<string, PassDescriptor>
--- @param requested_set string[] --- @param requested_set string[]
@@ -431,8 +398,7 @@ local function count_entries(t)
return n return n
end end
--- Compute in-degrees for the Kahn sort: for each pass in `needed`, --- Compute in-degrees for the Kahn sort: for each pass in `needed`, the number of its deps that are also in `needed`.
--- the number of its deps that are also in `needed`.
--- ---
--- @param passes table<string, PassDescriptor> --- @param passes table<string, PassDescriptor>
--- @param needed table<string, boolean> --- @param needed table<string, boolean>
@@ -450,8 +416,7 @@ local function compute_in_degrees(passes, needed)
return in_degree return in_degree
end end
--- Seed the Kahn ready queue with passes whose in-degree is 0, sorted --- Seed the Kahn ready queue with passes whose in-degree is 0, sorted alphabetically for deterministic execution order.
--- alphabetically for deterministic execution order.
--- ---
--- @param in_degree table<string, integer> --- @param in_degree table<string, integer>
--- @return string[] --- @return string[]
@@ -464,9 +429,8 @@ local function seed_ready_queue(in_degree)
return ready return ready
end end
-- (internal) Pop the next ready pass, decrement the in-degree of every -- (internal) Pop the next ready pass, decrement the in-degree of every remaining pass that depended on it
-- remaining pass that depended on it (inserting newly-zero-degree passes -- (inserting newly-zero-degree passes back into the ready queue), and append to `order`. Keeps `ready` sorted.
-- back into the ready queue), and append to `order`. Keeps `ready` sorted.
-- @param passes table<string, PassDescriptor> -- @param passes table<string, PassDescriptor>
-- @param needed table<string, boolean> -- @param needed table<string, boolean>
-- @param in_degree table<string, integer> -- @param in_degree table<string, integer>
@@ -506,11 +470,9 @@ local function topo_sort(passes, requested_set)
process_next_ready(passes, needed, in_degree, ready, order) process_next_ready(passes, needed, in_degree, ready, order)
end end
-- Cycle detection: if order doesn't include all needed passes, -- Cycle detection: if order doesn't include all needed passes, some are stuck with in_degree > 0 (the cycle closed on itself
-- some are stuck with in_degree > 0 (the cycle closed on itself -- before Kahn could process them). Without this check, a fully-closed cycle (e.g. A -> B -> A) would silently return an emspty order list,
-- before Kahn could process them). Without this check, a fully- -- leaving the orchestrator to dispatch nothing.
-- closed cycle (e.g. A -> B -> A) would silently return an empty
-- order list, leaving the orchestrator to dispatch nothing.
if #order ~= count_entries(needed) then if #order ~= count_entries(needed) then
for name, deg in pairs(in_degree) do for name, deg in pairs(in_degree) do
if deg > 0 then if deg > 0 then
@@ -527,8 +489,7 @@ end
-- ════════════════════════════════════════════════════════════════════════════ -- ════════════════════════════════════════════════════════════════════════════
--- Render the dep graph as ASCII art. Output width capped at 78 columns. --- Render the dep graph as ASCII art. Output width capped at 78 columns.
--- Falls back to the simpler "Resolved dependency order" list only if --- Falls back to the simpler "Resolved dependency order" list only if graph width exceeds terminal width.
--- graph width exceeds terminal width.
--- ---
--- @param passes table<string, PassDescriptor> --- @param passes table<string, PassDescriptor>
--- @param requested string[] -- originally-requested passes (subset of closed) --- @param requested string[] -- originally-requested passes (subset of closed)
@@ -591,8 +552,7 @@ end
-- Main orchestrator -- Main orchestrator
-- ════════════════════════════════════════════════════════════════════════════ -- ════════════════════════════════════════════════════════════════════════════
-- (internal) Push a pass's outputs + warnings into `ctx.upstream[name]` -- (internal) Push a pass's outputs + warnings into `ctx.upstream[name]` for downstream passes to consume.
-- for downstream passes to consume.
-- @param ctx PassCtx -- @param ctx PassCtx
-- @param pass_name string -- @param pass_name string
-- @param result PassResult -- @param result PassResult
@@ -606,9 +566,8 @@ local function accumulate_pass_result(ctx, pass_name, result)
end end
end end
-- (internal) If the pass's kind is in PASS_KIND_STOP_ON_ERROR and it -- (internal) If the pass's kind is in PASS_KIND_STOP_ON_ERROR and it reported errors, write each error to stderr.
-- reported errors, write each error to stderr. Returns true if any -- Returns true if any validation errors were reported.
-- validation errors were reported.
-- @param pass_name string -- @param pass_name string
-- @param pass PassDescriptor -- @param pass PassDescriptor
-- @param result PassResult -- @param result PassResult
+50 -1
View File
@@ -11,11 +11,17 @@ $misc = join-path $PSScriptRoot 'helpers/misc.ps1'
. $misc . $misc
# TODO(Ed): Review usage of these deps # TODO(Ed): Review usage of these deps
# I orgiinally cloned them when starting to get to the C runtime usage of the course # I originally cloned them when starting to get to the C runtime usage of the course
# However, based on the heavy reliance of the PSX.Dev extension I might fallback; also # However, based on the heavy reliance of the PSX.Dev extension I might fallback; also
# The gdb server doesn't need the full repo and were only using the src/mips # The gdb server doesn't need the full repo and were only using the src/mips
# which has a standalone repo (nuggets) # which has a standalone repo (nuggets)
# armips may not be used at all but I'm not sure... # armips may not be used at all but I'm not sure...
#
# PCSX-Redux: built via MSBuild (VS2022) — automated in the build section below.
# Requires: VS2022 with C++ desktop workload + PlatformToolset=v143 retarget.
# The .vcxproj files request v145; we pass /p:PlatformToolset=v143 to MSBuild.
# NuGet packages are restored automatically on first build.
# Output: toolchain\pcsx-redux\vsprojects\x64\Debug\pcsx-redux.exe
$url_armips = 'https://github.com/Kingcom/armips.git' $url_armips = 'https://github.com/Kingcom/armips.git'
$url_pcsx_redux = 'https://github.com/grumpycoders/pcsx-redux.git' $url_pcsx_redux = 'https://github.com/grumpycoders/pcsx-redux.git'
@@ -44,6 +50,33 @@ pop-location
# $psyq_obj_parser = join-path $path_pcsx_redux_binaries 'psyq-obj-parser.exe' # $psyq_obj_parser = join-path $path_pcsx_redux_binaries 'psyq-obj-parser.exe'
# ════════════════════════════════════════════════════════════════════════════
# PCSX-Redux — built via MSBuild (VS2022)
#
# Requires: Visual Studio 2022 with the C++ desktop workload.
# The .vcxproj files target platform toolset v145, but VS2022 ships v143;
# we pass /p:PlatformToolset=v143 to retarget at build time (no file edits).
# NuGet packages (glfw, luajit.native, libFFmpeg-lite, x64sentry) are
# restored automatically by MSBuild on first build.
#
# Output: toolchain\pcsx-redux\vsprojects\x64\Debug\pcsx-redux.exe
# ════════════════════════════════════════════════════════════════════════════
# Locate MSBuild from the VS2022 install (no hardcoded path — uses vswhere).
$vswhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe"
if (-not (Test-Path $vswhere)) {
write-error "vswhere not found at '$vswhere'. Install Visual Studio 2022 with the C++ desktop workload."
exit 1
}
$msbuild_exe = & $vswhere -latest -products * -requires Microsoft.Component.MSBuild -find "MSBuild\**\Bin\MSBuild.exe" 2>$null | Select-Object -First 1
if (-not $msbuild_exe) {
write-error "MSBuild not found via vswhere. Install Visual Studio 2022 with the C++ desktop workload."
exit 1
}
$path_pcsx_sln = join-path $path_pcsx_redux 'vsprojects\pcsx-redux.sln'
& $msbuild_exe $path_pcsx_sln /p:Configuration=Release /p:Platform=x64 /p:PlatformToolset=v143 /m /v:minimal
# Locate luajit via scoop. `luajit.exe` is on PATH via scoop's shim; # Locate luajit via scoop. `luajit.exe` is on PATH via scoop's shim;
# we use `scoop prefix` to find the install root for the include dir # we use `scoop prefix` to find the install root for the include dir
# (needed to compile lpeg against luajit's headers). # (needed to compile lpeg against luajit's headers).
@@ -80,3 +113,19 @@ $lpeg_compile_args = @(
push-location $path_lpeg push-location $path_lpeg
& gcc @lpeg_compile_args & gcc @lpeg_compile_args
pop-location pop-location
# ════════════════════════════════════════════════════════════════════════════
# OpenBIOS — built from the PCSX-Redux source tree via make + mipsel-none-elf
#
# OpenBIOS is an open-source PS1 BIOS implementation (no retail BIOS dump needed).
# It builds with the MIPS cross-toolchain (`mipsel-none-elf-gcc`, on PATH via the `mips` toolchain installer)
# + `make` (on PATH via scoop).
#
# Output: toolchain\pcsx-redux\src\mips\openbios\openbios.bin
# ════════════════════════════════════════════════════════════════════════════
$path_openbios = join-path $path_pcsx_redux 'src\mips\openbios'
push-location $path_openbios
& make clean
& make
pop-location