mostly comment review (lua metaprogram)

This commit is contained in:
2026-07-11 01:47:38 -04:00
parent a0d22700db
commit 91a91b3495
9 changed files with 386 additions and 557 deletions
+94 -135
View File
@@ -138,16 +138,13 @@ end
-- Section 0: LPeg patterns (compiled once at module load)
-- ════════════════════════════════════════════════════════════════════════════
--
-- LPeg is a required dependency (PEG library, no regex). It's loaded
-- via `package.cpath` (configured by `duffle_paths.lua` to find
-- `toolchain/lpeg/lpeg.dll`). There's no hand-rolled fallback — the
-- original two-tier design added complexity for a 5-10x speedup that's
-- only relevant at the high-level scanner stage; the byte-by-byte
-- helpers in Section 1 are sufficient for the classification primitives.
-- LPeg is a required dependency (PEG library, no regex).
-- It's loaded via `package.cpath` (configured by `duffle_paths.lua` to find `toolchain/lpeg/lpeg.dll`).
-- There's no hand-rolled fallback. The original two-tier design added complexity for a 5-10x speedup that's
-- only relevant at the high-level scanner stage; the byte-by-byte helpers in Section 1 are sufficient for the classification primitives.
--
-- If the require fails, fail loud with an actionable message (per
-- lua.md §9). The build script (`update_deps.ps1`) builds lpeg.dll
-- into `toolchain/lpeg/`; if it's missing, run `update_deps.ps1`.
-- If the require fails, fail loud with an actionable message. The build script (`update_deps.ps1`) builds lpeg.dll into `toolchain/lpeg/`;
-- if it's missing, run `update_deps.ps1`.
local lpeg_ok, lpeg = pcall(require, "lpeg")
if not lpeg_ok then
io.stderr:write("[duffle] require('lpeg') failed: ", lpeg, "\n")
@@ -195,8 +192,7 @@ local lpeg_scan_to_target_pat = function(target) return (P(1) - P(target))^0 en
-- is_space(c), is_alpha(c), etc. — accept a single-char STRING (legacy)
-- is_space_byte(b), is_alpha_byte(b), etc. — accept a single-byte INTEGER
--
-- The byte-based versions are 5-10x faster in tight loops because they
-- avoid the string allocation per s:sub(pos, pos) call.
-- The byte-based versions are 5-10x faster in tight loops because they avoid the string allocation per s:sub(pos, pos) call.
-- Whitespace characters per C locale.
function M.is_space_byte(b) return b == BYTE_SPACE or b == BYTE_TAB or b == BYTE_NEWLINE or b == BYTE_CR or b == BYTE_VT or b == BYTE_FF end
@@ -301,11 +297,10 @@ function M.write_file(path, content)
f:write(content); f:close()
end
-- Cache of directories already verified to exist in this process. Each
-- ensure_dir() call may otherwise spawn a `cmd.exe mkdir` (50-100ms
-- per call on Windows) — calling it inside per-source loops added 1.5+
-- seconds to the report pass. Cache makes ensure_dir idempotent within
-- the process lifetime (safe across passes; the dir state doesn't change).
-- Cache of directories already verified to exist in this process.
-- Each ensure_dir() call may otherwise spawn a `cmd.exe mkdir` (50-100ms per call on Windows) — calling it inside per-source loops added 1.5+
-- seconds to the report pass. Cache makes ensure_dir idempotent within the process lifetime.
-- (safe across passes; the dir state doesn't change).
local _ensured_dirs = {}
function M.ensure_dir(path)
@@ -324,8 +319,7 @@ function M._reset_ensured_dirs() _ensured_dirs = {} end
-- ════════════════════════════════════════════════════════════════════════════
-- Skip a string or C-style comment starting at position `pos`.
-- Returns the position just past the construct, or `pos` unchanged if
-- no string/comment starts there. LPeg-backed.
-- Returns the position just past the construct, or `pos` unchanged if no string/comment starts there. LPeg-backed.
function M.skip_str_or_cmt(s, pos)
return lpeg.match(lpeg_str_or_cmt_pat, s, pos) or pos
end
@@ -336,21 +330,17 @@ function M.skip_ws_and_cmt(s, pos)
return lpeg.match(lpeg_ws_and_cmt_pat, s, pos) or pos
end
-- Read a C-style identifier (alpha followed by zero+ alnum) starting at
-- position `pos`. Returns the identifier string + the position just past
-- it, or nil + pos if no identifier starts here. LPeg-backed.
-- Read a C-style identifier (alpha followed by zero+ alnum) starting at position `pos`.
-- Returns the identifier string + the position just past it, or nil + pos if no identifier starts here. LPeg-backed.
function M.read_ident(s, pos)
local result = lpeg.match(lpeg_ident_pat, s, pos)
if result then return result, pos + #result end
return nil, pos
end
-- Read a balanced-delimited group (parens, braces, or brackets) starting
-- at position `pos`. Returns the inner content (between the delimiters) +
-- the position just past the closing delimiter, or nil + pos if `s[pos]`
-- isn't `open_char`.
--
-- (Hand-rolled; the depth counting makes pure LPeg awkward here.)
-- Read a balanced-delimited group (parens, braces, or brackets) starting at position `pos`.
-- Returns the inner content (between the delimiters) + the position
-- just past the closing delimiter, or nil + pos if `s[pos]` isn't `open_char`.
function M.read_balanced(s, open_char, close_char, pos)
local open_byte = open_char:byte()
if s:byte(pos) ~= open_byte then return nil, pos end
@@ -380,8 +370,8 @@ M.read_parens = function(s, pos) return M.read_balanced(s, "(", ")", pos) end
M.read_braces = function(s, pos) return M.read_balanced(s, "{", "}", pos) end
M.read_brackets = function(s, pos) return M.read_balanced(s, "[", "]", pos) end
-- Scan forward from position `start` until we find a specific single byte
-- `target`, transparently stepping over balanced parens/braces/brackets.
-- Scan forward from position `start` until we find a specific single byte `target`,
-- transparently stepping over balanced parens/braces/brackets.
-- Returns the position of `target`, or nil if not found.
function M.scan_to_char(s, target, start)
local target_byte = target:byte()
@@ -403,23 +393,18 @@ end
-- Split a brace-body into top-level comma-separated tokens. Honors nested
-- parens/braces/brackets and skips strings/comments.
--
-- FIX (2026-07-09): split at top-level NEWLINES and SEMICOLONS too, AND
-- emit a token break after a top-level comment/string. Previous behavior
-- glued the macro call after a comment into the same token, so
-- `word_count_of_token` only saw the leading ident (often nil after
-- stripping the comment), undercounting the body. See Phase 1 of the
-- branch-offset regression investigation. Pure-comment / pure-string
-- chunks (which now appear between real statements) are filtered out so
-- they contribute 0 words instead of 1.
-- FIX (2026-07-09): split at top-level NEWLINES and SEMICOLONS too, AND emit a token break after a top-level comment/string.
-- Previous behavior glued the macro call after a comment into the same token, so `word_count_of_token` only saw the
-- leading ident (often nil after stripping the comment), undercounting the body. See Phase 1 of the branch-offset regression investigation.
-- Pure-comment / pure-string chunks (which now appear between real statements) are filtered out so they contribute 0 words instead of 1.
function M.split_top_level_commas(body)
local tokens = {}
local pos = 1
local body_len = #body
local token_start = 1
-- True iff `chunk` contains any non-whitespace, non-comment, non-string
-- content (i.e., real token material). Walks through ws + comments
-- individually so a chunk like " /* trailing */ shift_lleft(...)"
-- True iff `chunk` contains any non-whitespace, non-comment, non-string content
-- (i.e., real token material). Walks through ws + comments individually so a chunk like " /* trailing */ shift_lleft(...)"
-- is correctly classified as having real content (the macro call).
local function has_real_content(chunk)
local scan = 1
@@ -447,14 +432,13 @@ function M.split_top_level_commas(body)
tokens[#tokens + 1] = chunk
elseif #tokens > 0 then
-- Pure comment/string chunk at top level (no preceding instruction content within this chunk).
-- APPEND it to the LAST token so emit-context callers (components.lua build_component_lines)
-- APPEND it to the LAST token so emit-context callers (components.lua build_component_lines)
-- can convert `// trailing comment` to `/* */` and emit it with the macro body.
-- For word counting, count_token_words only inspects the leading ident, so a trailing comment doesn't
-- affect the count.
-- For word counting, count_token_words only inspects the leading ident, so a trailing comment doesn't affect the count.
--
-- This is the second-half fix to commit 98e27c2: the first fix correctly broke top-level comments
-- This is the second-half fix to commit 98e27c2: the first fix correctly broke top-level comments
-- off from the NEXT statement (fixing macro-call word counts);
-- this fix preserves them on the PREVIOUS statement (restoring the comments in the emitted .macs.h output).
-- This fix preserves them on the PREVIOUS statement (restoring the comments in the emitted .macs.h output).
tokens[#tokens] = tokens[#tokens] .. chunk
end
end
@@ -540,7 +524,7 @@ function M.LineIndex(source)
end
-- (internal) Binary-search for the line number containing `query_pos`.
local function line_of(query_pos)
local lo, hi = 1, n
local lo, hi = 1, n
while lo <= hi do
local mid = math.floor((lo + hi) / 2)
if positions[mid] <= query_pos then
@@ -577,34 +561,23 @@ M.TAPE_ATOM_MACROS = {
-- GTE pipeline-fill latency table (static-analysis Phase 1).
--
-- For each `gte_cmdw_*` macro in code/duffle/gte.h, the minimum number
-- of consecutive COP2 "nop" words that MUST appear before any other
-- COP2 read or non-nop instruction (so the GTE pipeline latency is
-- fully retired). Latencies are sourced from the doxygen comments
-- in gte.h (e.g. `* @brief Rotate, Translate and Perspective Triple
-- (23 cycles)` with body `Two nop words fill the COP2 pipeline
-- latency`).
-- For each `gte_cmdw_*` macro in code/duffle/gte.h, the minimum number of consecutive COP2 "nop" words that MUST appear
-- before any other COP2 read or non-nop instruction (so the GTE pipeline latency is fully retired).
-- Latencies are sourced from the doxygen comments in gte.h
-- (e.g. `* @brief Rotate, Translate and Perspective Triple (23 cycles)` with body `Two nop words fill the COP2 pipeline latency`).
--
-- The check (`scripts/passes/static_analysis.lua ::
-- check_gte_pipeline_fill`) walks each atom body, counts the
-- consecutive nop words after every `gte_cmdw_*` invocation, and
-- reports a finding if the count is below this minimum. Aliases
-- are dereferenced before lookup (gté_cmdw_rtps_alias ->
-- gte_cmdw_rtps -> 2).
--
-- Values verified against PSX-SPX gte.txt (rtpt 23cy / 8cy per divide
-- => 2 nops; nclip 8cy => 2 nops; avsz3/avsz4 14cy => 2 nops; op
-- single-cycle atomic => 0 nops; mvmva 8cy matrix-vector => 2 nops).
-- The check (`scripts/passes/static_analysis.lua :: check_gte_pipeline_fill`) walks each atom body,
-- counts the consecutive nop words after every `gte_cmdw_*` invocation, and reports a finding if the count is below this minimum.
-- Aliases are dereferenced before lookup (gté_cmdw_rtps_alias -> gte_cmdw_rtps -> 2).
--
-- Values verified against PSX-SPX gte.txt (rtpt 23cy / 8cy per divide => 2 nops; nclip 8cy => 2 nops; avsz3/avsz4 14cy => 2 nops;
-- op single-cycle atomic => 0 nops; mvmva 8cy matrix-vector => 2 nops).
M.GTE_PIPELINE_LATENCY = {
-- Minimum number of consecutive `nop` words that must appear
-- IMMEDIATELY BEFORE a `gte_cmdw_<X>` invocation -- to retire
-- any preceding `lwc2` / `swc2` / pre-existing C2 state writes
-- before the GTE pipeline starts reading from V0/V1/V2 or
-- MAC0..3 / OTZ / IR0..3 at the command's issue cycle.
--
-- Values are from the doxygen comments in code/duffle/gte.h and
-- cross-checked against PSX-SPX `geometrytransformationenginegte.md`:
-- Minimum number of consecutive `nop` words that must appear IMMEDIATELY BEFORE a `gte_cmdw_<X>` invocation
-- to retire any preceding `lwc2` / `swc2` / pre-existing C2 state writes before the GTE pipeline starts reading
-- from V0/V1/V2 or MAC0..3 / OTZ / IR0..3 at the command's issue cycle.
--
-- Values are from the doxygen comments in code/duffle/gte.h and cross-checked against PSX-SPX `geometrytransformationenginegte.md`:
-- cmd cycles min pre-nops rationale
-- rtps 14 2 8c per perspective divide + 6c for IR1..4 + mac write
-- rptt 22 2 3x rtps worth of pipeline depth
@@ -612,24 +585,18 @@ M.GTE_PIPELINE_LATENCY = {
-- avsz3 14 2 14c to compute average + write OTZ
-- avsz4 16 2 avsz3 + 2c extra for avg over 4
-- mvmva 8 2 IR1..4 write + matrix work
-- op 5 0 output to MAC0 only (atomic 5c calc)
-- op 5 0 output to MAC0 only (atomic 5c calc)
--
-- The `gte_rtpt()` / `gte_nclip()` / `gte_avsz3()` wrapper macros in
-- gte.h emit the pre-cmd nops internally (asm_words(nop, nop, ...)),
-- but THOSE WRAPPERS ARE NOT USED INSIDE ATOM BODIES in this
-- codebase. Every MipsAtom_(name) body uses raw `nop2,
-- gte_cmdw_<X>, ...` form instead -- that `nop2,` is the pre-fill
-- this check validates. So values here must reflect the source-level
-- convention, NOT the wrapper-internal pre-fill (which is invisible
-- at the source level).
-- The `gte_rtpt()` / `gte_nclip()` / `gte_avsz3()` wrapper macros in gte.h emit the pre-cmd nops internally (asm_words(nop, nop, ...)),
-- but THOSE WRAPPERS ARE NOT USED INSIDE ATOM BODIES in this codebase.
-- Every MipsAtom_(name) body uses raw `nop2, gte_cmdw_<X>, ...` form instead -- that `nop2,` is the pre-fill this check validates.
-- So values here must reflect the source-level convention, NOT the wrapper-internal pre-fill (which is invisible at the source level).
--
-- Existing clean-atom bodies (cube_g4_face, floor_f3_face,
-- diag_gte) all emit `nop2,` before every `gte_cmdw_<X>` (which
-- matches values >= 2). The check passes them all.
-- Existing clean-atom bodies (cube_g4_face, floor_f3_face, diag_gte) all emit `nop2,` before every `gte_cmdw_<X>` (which matches values >= 2).
-- The check passes them all.
--
-- Aliases are listed separately because source code may use either
-- the alias or the canonical name. The check looks up the EXACT
-- macro text, so both forms must be in the table.
-- Aliases are listed separately because source code may use either the alias or the canonical name.
-- The check looks up the EXACT macro text, so both forms must be in the table.
-- Canonical macros (from code/duffle/gte.h)
["gte_cmdw_rtps"] = 2,
@@ -646,7 +613,7 @@ M.GTE_PIPELINE_LATENCY = {
["gte_cmdw_avg_sort_z4"] = 2,
-- Outer product aliases (same canonical op, 0 pre-fill nops).
-- gte_cmdw_op = canonical GTE-internal short form
-- gte_cmdw_op = canonical GTE-internal short form
-- gte_cmdw_outer_product = NOCASH / SDK-readable form
-- gte_cmdw_wedge = geometric-algebra (exterior-product) form
["gte_cmdw_outer_product"] = 0,
@@ -656,13 +623,13 @@ M.GTE_PIPELINE_LATENCY = {
-- GP0 packet sizes (total words including the 1-word tag) per GP0 cmd byte.
-- Verified against code/duffle/gp.h struct sizes + the set_poly_* macros
-- (which encode "len" = "words after tag"):
-- set_poly_f3(p) -> set_len(p, 4) -> 5 total GP0 0x20
-- set_poly_ft3(p) -> set_len(p, 7) -> 8 total GP0 0x24
-- set_poly_f4(p) -> set_len(p, 5) -> 6 total GP0 0x28
-- set_poly_f3(p) -> set_len(p, 4) -> 5 total GP0 0x20
-- set_poly_ft3(p) -> set_len(p, 7) -> 8 total GP0 0x24
-- set_poly_f4(p) -> set_len(p, 5) -> 6 total GP0 0x28
-- set_poly_ft4(p) -> set_len(p, 9) -> 10 total GP0 0x2C
-- set_poly_g3(p) -> set_len(p, 6) -> 7 total GP0 0x30
-- set_poly_g3(p) -> set_len(p, 6) -> 7 total GP0 0x30
-- set_poly_gt3(p) -> set_len(p, 9) -> 10 total GP0 0x34
-- set_poly_g4(p) -> set_len(p, 8) -> 9 total GP0 0x38
-- set_poly_g4(p) -> set_len(p, 8) -> 9 total GP0 0x38
-- set_poly_gt4(p) -> set_len(p, 12) -> 13 total GP0 0x3C
M.GP0_CMD_SIZE = {
[0x20] = 5, -- Poly_F3
@@ -676,8 +643,7 @@ M.GP0_CMD_SIZE = {
}
-- Shape suffix (after `ac_format_` / `mac_format_` prefix) -> GP0 cmd byte.
-- Lets the static-analysis check derive the cmd byte from a macro name
-- like `mac_format_g4_color` -> `g4` -> 0x38 -> 9 expected words.
-- Lets the static-analysis check derive the cmd byte from a macro name like `mac_format_g4_color` -> `g4` -> 0x38 -> 9 expected words.
M.GP0_CMD_BY_SHAPE = {
["f3"] = 0x20, ["ft3"] = 0x24,
["f4"] = 0x28, ["ft4"] = 0x2C,
@@ -685,11 +651,9 @@ M.GP0_CMD_BY_SHAPE = {
["g4"] = 0x38, ["gt4"] = 0x3C,
}
-- Per-macro prim-buffer contribution (NOT .text instruction count --
-- this is "how many 32-bit words does this macro write to the primitive
-- being built in main RAM"). Sum across `mac_format_X_color` +
-- `mac_gte_store_X_post_*` + `mac_insert_ot_tag_X` calls in an atom body
-- must equal GP0_CMD_SIZE[GP0_CMD_BY_SHAPE[shape]].
-- Per-macro prim-buffer contribution
-- (NOT .text instruction count this is "how many 32-bit words does this macro write to the primitive being built in main RAM").
-- Sum across `mac_format_X_color` + `mac_gte_store_X_post_*` + `mac_insert_ot_tag_X` calls in an atom body must equal GP0_CMD_SIZE[GP0_CMD_BY_SHAPE[shape]].
M.GP0_MACRO_CONTRIB = {
["mac_format_f3_color"] = 1,
["mac_format_g3_color"] = 3,
@@ -702,33 +666,28 @@ M.GP0_MACRO_CONTRIB = {
["mac_insert_ot_tag_g4"] = 1,
}
-- Per-macro cycle cost (best-case, no stalls). Used by the static-analysis
-- `count_atom_cycles` pass (Phase 3) to emit per-atom cycle budgets. The
-- counts cover the EXPANDED instruction sequence the macro emits (NOT just
-- the token it appears as in source). For example:
--
-- Per-macro cycle cost (best-case, no stalls). Used by the static-analysis `count_atom_cycles` pass (Phase 3) to emit per-atom cycle budgets.
-- The counts cover the EXPANDED instruction sequence the macro emits (NOT just the token it appears as in source).
-- For example:
-- mac_pack_color_word(off, cmd, r, g, b) emits:
-- load_upper_i(R_AT, (cmd << 8) | b) -- 1 cycle
-- or_i_self(R_AT, (g << 8) | r) -- 1 cycle
-- store_word(R_AT, R_PrimCursor, off) -- 1 cycle
-- = 3 cycles total
--
-- mac_yield emits a control-transfer sequence (load_word, add_ui_self,
-- jump_reg, nop) which "yields control" -- the atom body's cycle budget
-- doesn't include the yield's cost (we model it as 0; runtime cost
-- becomes part of the NEXT atom's prologue).
-- mac_yield emits a control-transfer sequence (load_word, add_ui_self, jump_reg, nop)
-- which "yields control" the atom body's cycle budget doesn't include the yield's cost (we model it as 0;
-- runtime cost becomes part of the NEXT atom's prologue).
--
-- GTE command values are the GTE instruction's intrinsic cycles (the
-- latency AFTER any pre-cmd `nop2` has retired). When the source emits
-- `nop2, gte_cmdw_X` the nops' cycles are added separately (1+1) plus
-- the gte_cmdw_X value here:
-- rtpt = 21 + 2 nops = 23 total cycles (matches PSX-SPX)
-- rtps = 12 + 2 nops = 14 total
-- nclip = 6 + 2 nops = 8 total
-- GTE command values are the GTE instruction's intrinsic cycles (the latency AFTER any pre-cmd `nop2` has retired).
-- When the source emits `nop2, gte_cmdw_X` the nops' cycles are added separately (1+1) plus the gte_cmdw_X value here:
-- rtpt = 21 + 2 nops = 23 total cycles (matches PSX-SPX)
-- rtps = 12 + 2 nops = 14 total
-- nclip = 6 + 2 nops = 8 total
-- avsz3 = 12 + 2 nops = 14 total
-- avsz4 = 14 + 2 nops = 16 total
-- mvmva = 6 + 2 nops = 8 total
-- op = 5 (no pre-cmd nops required; single-cycle atomic)
-- mvmva = 6 + 2 nops = 8 total
-- op = 5 (no pre-cmd nops required; single-cycle atomic)
M.INSTRUCTION_LATENCY = {
-- CPU ALU (single-cycle R3000A ops)
["nop"] = 1,
@@ -794,27 +753,27 @@ M.INSTRUCTION_LATENCY = {
-- COP2 commands (intrinsic cycles, EXCLUDING the 2 pre-cmd nops that
-- the source typically emits as `nop2, gte_cmdw_X`; those nops are
-- counted separately via the `nop2` entry above)
["gte_cmdw_rtpt"] = 21,
["gte_cmdw_rtps"] = 12,
["gte_cmdw_nclip"] = 6,
["gte_cmdw_avsz3"] = 12,
["gte_cmdw_avsz4"] = 14,
["gte_cmdw_mvmva"] = 6,
["gte_cmdw_op"] = 5,
["gte_cmdw_rtpt"] = 21,
["gte_cmdw_rtps"] = 12,
["gte_cmdw_nclip"] = 6,
["gte_cmdw_avsz3"] = 12,
["gte_cmdw_avsz4"] = 14,
["gte_cmdw_mvmva"] = 6,
["gte_cmdw_op"] = 5,
["gte_cmdw_outer_product"] = 5,
["gte_cmdw_wedge"] = 5,
["gte_cmdw_wedge"] = 5,
-- Long-form aliases (same cost as canonical)
["gte_cmdw_rotate_translate_perspective_single"] = 12, -- alias for rtps
["gte_cmdw_rotate_translate_perspective_triple"] = 21, -- alias for rtpt
["gte_cmdw_avg_sort_z4"] = 14, -- alias for avsz4
["gte_cmdw_avg_sort_z4"] = 14, -- alias for avsz4
-- Non-cmdw aliases from gte.h (these are `#define gte_X gte_cmdw_Y`):
["gte_avg_sort_z3"] = 12, -- alias for avsz3
["gte_avg_sort_z4"] = 14, -- alias for avsz4
["gte_rtps"] = 12, -- alias for rtps
["gte_rtpt"] = 21, -- alias for rtpt
["gte_nclip"] = 6, -- alias for nclip
["gte_avsz3"] = 12,
["gte_avsz4"] = 14,
["gte_avg_sort_z3"] = 12, -- alias for avsz3
["gte_avg_sort_z4"] = 14, -- alias for avsz4
["gte_rtps"] = 12, -- alias for rtps
["gte_rtpt"] = 21, -- alias for rtpt
["gte_nclip"] = 6, -- alias for nclip
["gte_avsz3"] = 12,
["gte_avsz4"] = 14,
-- Legacy single-cycle store helpers (gte_stotz, gte_stsxy3 are 1 cycle)
["gte_stotz"] = 1,
["gte_stsxy3"] = 1,
@@ -847,8 +806,8 @@ M.INSTRUCTION_LATENCY = {
["atom_writes"] = 0,
}
-- Default cycle cost for unknown macros. The static-analysis pass adds 1
-- cycle per unknown token and emits a "new macro; update INSTRUCTION_LATENCY"
-- Default cycle cost for unknown macros.
-- The static-analysis pass adds 1 cycle per unknown token and emits a "new macro; update INSTRUCTION_LATENCY"
-- advisory so the cycle budget stays accurate as the codebase grows.
M.UNKNOWN_INSTRUCTION_CYCLES = 1