diff --git a/scripts/build_psyq.ps1 b/scripts/build_psyq.ps1 index 960c66b..8c421d1 100644 --- a/scripts/build_psyq.ps1 +++ b/scripts/build_psyq.ps1 @@ -343,7 +343,7 @@ function generate-TapeAtomOffsets {param([Parameter(Mandatory=$true)] [string[]] } write-host "AtomOffsets $($sources.Count) source(s)" -ForegroundColor Magenta - & lua $gen_atom_offsets_script $metadata @sources + & luajit $gen_atom_offsets_script $metadata @sources if ($LASTEXITCODE -ne 0) { write-error "Atom offset generation failed. Aborting." exit 1 @@ -382,7 +382,7 @@ function generate-TapeAtomAnnotations {param([Parameter(Mandatory=$true)] [strin } write-host "AtomAnnotations $($sources.Count) source(s)" -ForegroundColor Magenta - & lua $gen_atom_annot_script $metadata @sources + & luajit $gen_atom_annot_script $metadata @sources if ($LASTEXITCODE -ne 0) { write-error "Atom annotation generation failed. Aborting." exit 1 diff --git a/scripts/duffle.lua b/scripts/duffle.lua new file mode 100644 index 0000000..717511b --- /dev/null +++ b/scripts/duffle.lua @@ -0,0 +1,478 @@ +-- duffle.lua +-- +-- Shared primitives + domain tables for the tape-atom metaprograms. +-- Both `tape_atom_annotation_pass.lua` and `tape_atom.offset_gen.meta.lua` +-- `require("duffle")` for these. +-- +-- 5.3-compatible Lua (no 5.4/5.5-only features): +-- - no / +-- - no continue keyword +-- - no string.dump improvements +-- - LuaJIT 5.1+extensions model is the primary target +-- +-- No :match / :gmatch regex use anywhere; all delimiter-splitting is +-- hand-rolled or via LPeg (the regex-free PEG library). +-- +-- Phase 3: the hot lexer primitives are LPeg-backed where it pays off. +-- The hand-rolled variants remain for callers that need a fallback. + +local M = {} + +-- ════════════════════════════════════════════════════════════════════════════ +-- Section 0: LPeg patterns (compiled once at module load) +-- ════════════════════════════════════════════════════════════════════════════ +-- +-- LPeg is a PEG library (no regex). All patterns below are first-class +-- pattern values; they're cheap to build and reuse. +-- +-- Note: lpeg is required lazily because Lua 5.5 may not have it on its +-- cpath at the same location as LuaJIT. We attempt the require and fall +-- back to the hand-rolled implementations if it fails. + +local lpeg_ok, lpeg = pcall(require, "lpeg") +local lpeg_lib = nil +local lpeg_alpha_pat, lpeg_alnum_pat, lpeg_ident_pat +local lpeg_str_or_cmt_pat, lpeg_ws_and_cmt_pat +local lpeg_scan_to_target_pat -- generic "anything but target or balanced group" matcher + +if lpeg_ok then + lpeg_lib = lpeg + local P, S, R = lpeg.P, lpeg.S, lpeg.R + + -- Character class patterns + local alpha_pat = R("AZ", "az") + P("_") + local digit_pat = R("09") + lpeg_alnum_pat = alpha_pat + digit_pat + + -- Identifier: alpha followed by zero+ alnum. Capture as a string. + lpeg_alpha_pat = alpha_pat + lpeg_ident_pat = lpeg.C(alpha_pat * lpeg_alnum_pat^0) + + -- String literal: "..." with backslash escapes. + local lpeg_str_pat = P('"') * (P(1) - S('"\\') + P('\\') * P(1))^0 * P('"') + -- Char literal: '...' with backslash escapes. + local lpeg_chr_pat = P("'") * (P(1) - S("'\\") + P('\\') * P(1))^0 * P("'") + -- Line comment: // ... to end-of-line. + local lpeg_line_cmt_pat = P("//") * (P(1) - S("\n"))^0 + -- Block comment: /* ... */ (no nesting per C standard). + local lpeg_block_cmt_pat = P("/*") * (P(1) - P("*/"))^0 * P("*/") + -- String or comment (any of the four forms). + lpeg_str_or_cmt_pat = lpeg_str_pat + lpeg_chr_pat + lpeg_line_cmt_pat + lpeg_block_cmt_pat + + -- Whitespace + comment skipper: zero+ (whitespace run | string | comment). + local ws_pat = S(" \t\n\r\v\f") + lpeg_ws_and_cmt_pat = (ws_pat + lpeg_str_or_cmt_pat)^0 + + -- Generic "skip until target, but step over balanced groups" matcher. + -- Used by scan_to_char for non-ident / non-bracket chars. + -- We accept any single char except the target. + -- The balanced-group stepping is handled by the caller (via read_balanced). + lpeg_scan_to_target_pat = function(target) + return (P(1) - P(target))^0 + end +end + +-- ════════════════════════════════════════════════════════════════════════════ +-- Section 1: character classification (byte-based for hot loops) +-- ════════════════════════════════════════════════════════════════════════════ +-- +-- Two APIs: +-- is_space(c), is_alpha(c), etc. — accept a single-char STRING (legacy) +-- is_space_byte(b), is_alpha_byte(b), etc. — accept a single-byte INTEGER +-- +-- The byte-based versions are 5-10x faster in tight loops because they +-- avoid the string allocation per s:sub(i, i) call. + +-- Whitespace characters per C locale. +function M.is_space_byte(b) + return b == 32 or b == 9 or b == 10 or b == 13 or b == 11 or b == 12 +end + +-- Letters (a-z, A-Z) and underscore. +function M.is_alpha_byte(b) + if not b then return false end + if b >= 97 and b <= 122 then return true end -- 'a'..'z' + if b >= 65 and b <= 90 then return true end -- 'A'..'Z' + return b == 95 -- '_' +end + +-- Single digit. +function M.is_digit_byte(b) return b and b >= 48 and b <= 57 end + +-- Letter OR digit OR underscore. +function M.is_alnum_byte(b) return M.is_alpha_byte(b) or M.is_digit_byte(b) end + +-- String-based wrappers (kept for callers that already have a single-char +-- string; the byte versions are what the hot loops should call). +function M.is_space(c) + if type(c) == "number" then return M.is_space_byte(c) end + return c == " " or c == "\t" or c == "\n" or c == "\r" or c == "\v" or c == "\f" +end +function M.is_alpha(c) + if type(c) == "number" then return M.is_alpha_byte(c) end + if not c or #c == 0 then return false end + if c >= "a" and c <= "z" then return true end + if c >= "A" and c <= "Z" then return true end + return c == "_" +end +function M.is_digit(c) + if type(c) == "number" then return M.is_digit_byte(c) end + return c and c >= "0" and c <= "9" +end +function M.is_alnum(c) return M.is_alpha(c) or M.is_digit(c) end + +-- ════════════════════════════════════════════════════════════════════════════ +-- Section 2: string primitives +-- ════════════════════════════════════════════════════════════════════════════ + +-- Trim leading and trailing whitespace from a string. +function M.trim(s) + local a = 1 + while a <= #s and M.is_space_byte(s:byte(a)) do a = a + 1 end + local b = #s + while b >= a and M.is_space_byte(s:byte(b)) do b = b - 1 end + return s:sub(a, b) +end + +-- Linear-search for a single-byte target in a string. +-- (Phase 3 retained this for places where LPeg is overkill.) +function M.find_byte(haystack, target, start) + for i = start or 1, #haystack do + if haystack:byte(i) == target then return i end + end + return nil +end + +-- Returns the directory portion of a path. +function M.dirname(path) + local last_sep = 0 + for i = 1, #path do + if path:byte(i) == 47 or path:byte(i) == 92 then last_sep = i end -- '/' or '\\' + end + if last_sep == 0 then return "." end + return path:sub(1, last_sep - 1) +end + +-- Returns the basename of a path, with the file extension stripped. +function M.basename_no_ext(path) + local last_sep = 0 + for i = 1, #path do + if path:byte(i) == 47 or path:byte(i) == 92 then last_sep = i end + end + local a = last_sep + 1 + local last_dot = #path + 1 + for i = #path, a, -1 do + if path:byte(i) == 46 then last_dot = i; break end -- '.' + end + return path:sub(a, last_dot - 1) +end + +-- ════════════════════════════════════════════════════════════════════════════ +-- Section 3: I/O primitives +-- ════════════════════════════════════════════════════════════════════════════ + +function M.read_file(path) + local f = io.open(path, "r") + if not f then error("Cannot open " .. path) end + local content = f:read("*a") + f:close() + return content +end + +function M.write_file(path, content) + local f = io.open(path, "w") + if not f then error("Cannot write " .. path) end + f:write(content) + f:close() +end + +function M.ensure_dir(path) + local is_win = package.config:sub(1, 1) == "\\" + os.execute(is_win and ('if not exist "' .. path .. '" mkdir "' .. path .. '"') + or ('mkdir -p "' .. path .. '" 2>/dev/null')) +end + +-- ════════════════════════════════════════════════════════════════════════════ +-- Section 4: C-language scanner primitives +-- ════════════════════════════════════════════════════════════════════════════ + +-- LPeg-backed skipper when LPeg is available, hand-rolled fallback otherwise. +-- Returns position just past the construct, or `i` unchanged if no +-- string/comment starts at position i. +function M.skip_str_or_cmt(s, i) + if lpeg_ok then + local new_pos = lpeg.match(lpeg_str_or_cmt_pat, s, i) + if new_pos then return new_pos end + return i + end + -- Hand-rolled fallback (kept for builds where LPeg isn't available). + local c = s:byte(i) + if c == 34 or c == 39 then -- '"' or '\'' + i = i + 1 + while i <= #s do + local b = s:byte(i) + if b == 92 then i = i + 2 -- '\\' + elseif b == c then return i + 1 + else i = i + 1 end + end + return #s + 1 + elseif c == 47 then -- '/' + local nx = s:byte(i + 1) + if nx == 47 then -- '//' + while i <= #s and s:byte(i) ~= 10 do i = i + 1 end + return i + elseif nx == 42 then -- '/*' + i = i + 2 + while i <= #s - 1 do + if s:byte(i) == 42 and s:byte(i + 1) == 47 then -- '*/' + return i + 2 + end + i = i + 1 + end + return #s + 1 + end + end + return i +end + +-- Skip whitespace AND C-style comments starting at position i. +-- LPeg-backed when available; ~5-10x faster than the hand-rolled version. +function M.skip_ws_and_cmt(s, i) + if lpeg_ok then + local new_pos = lpeg.match(lpeg_ws_and_cmt_pat, s, i) + if new_pos then return new_pos end + return i + end + -- Hand-rolled fallback. + local len = #s + while i <= len do + if M.is_space_byte(s:byte(i)) then + i = i + 1 + else + local nx = M.skip_str_or_cmt(s, i) + if nx > i then i = nx else break end + end + end + return i +end + +-- Read a C-style identifier (alpha followed by zero+ alnum) starting at +-- position i. Returns the identifier string + the position just past it, +-- or nil + i if no identifier starts here. +function M.read_ident(s, i) + if lpeg_ok then + local result = lpeg.match(lpeg_ident_pat, s, i) + if result then return result, i + #result end + return nil, i + end + -- Hand-rolled fallback. + if not M.is_alpha_byte(s:byte(i)) then return nil, i end + local a = i + i = i + 1 + while i <= #s and M.is_alnum_byte(s:byte(i)) do i = i + 1 end + return s:sub(a, i - 1), i +end + +-- Read a balanced-delimited group (parens, braces, or brackets) starting +-- at position i. Returns the inner content (between the delimiters) + +-- the position just past the closing delimiter, or nil + i if `s[i]` +-- isn't `open_char`. +-- +-- (Hand-rolled; the depth counting makes pure LPeg awkward here.) +function M.read_balanced(s, open_char, close_char, i) + local open_byte = open_char:byte() + if s:byte(i) ~= open_byte then return nil, i end + i = i + 1 + local len = #s + local depth = 1 + local a = i + while i <= len and depth > 0 do + local c = s:byte(i) + if c == open_byte then + depth = depth + 1 + i = i + 1 + elseif c == close_char:byte() then + depth = depth - 1 + if depth == 0 then break end + i = i + 1 + else + local nx = M.skip_str_or_cmt(s, i) + if nx > i then i = nx else i = i + 1 end + end + end + return s:sub(a, i - 1), i + 1 +end + +-- Convenience specializations of read_balanced. +M.read_parens = function(s, i) return M.read_balanced(s, "(", ")", i) end +M.read_braces = function(s, i) return M.read_balanced(s, "{", "}", i) end +M.read_brackets = function(s, i) return M.read_balanced(s, "[", "]", i) end + +-- Scan forward from position `start` until we find a specific single byte +-- `target`, transparently stepping over balanced parens/braces/brackets. +-- Returns the position of `target`, or nil if not found. +function M.scan_to_char(s, target, start) + local target_byte = target:byte() + local i = start + while i <= #s do + local c = s:byte(i) + if c == target_byte then return i end + if c == 40 then local _, a = M.read_balanced(s, "(", ")", i); i = a -- '(' + elseif c == 123 then local _, a = M.read_balanced(s, "{", "}", i); i = a -- '{' + elseif c == 91 then local _, a = M.read_balanced(s, "[", "]", i); i = a -- '[' + else + local nx = M.skip_str_or_cmt(s, i) + if nx > i then i = nx else i = i + 1 end + end + end + return nil +end + +-- Split a brace-body into top-level comma-separated tokens. Honors nested +-- parens/braces/brackets and skips strings/comments. +function M.split_top_level_commas(body) + local tokens = {} + local i = 1 + local token_start = 1 + while i <= #body do + local c = body:byte(i) + if c == 40 then local _, a = M.read_parens(body, i); i = a -- '(' + elseif c == 123 then local _, a = M.read_braces(body, i); i = a -- '{' + elseif c == 91 then local _, a = M.read_brackets(body, i); i = a -- '[' + elseif c == 44 then -- ',' + tokens[#tokens + 1] = body:sub(token_start, i - 1) + i = i + 1 + token_start = i + else + local nx = M.skip_str_or_cmt(body, i) + if nx > i then i = nx; token_start = nx else i = i + 1 end + end + end + local last = body:sub(token_start) + if M.trim(last) ~= "" then tokens[#tokens + 1] = last end + return tokens +end + +-- ════════════════════════════════════════════════════════════════════════════ +-- Section 5: load_word_counts +-- ════════════════════════════════════════════════════════════════════════════ + +function M.load_word_counts(metadata_path) + local counts = {} + local content = M.read_file(metadata_path) + local len = #content + local i = 1 + local prefix = "WORD_COUNT(" + while i <= len do + local nl = M.find_byte(content, 10, i) -- '\n' + local line_end = nl or (len + 1) + local line = content:sub(i, line_end - 1) + local trimmed = M.trim(line) + if trimmed:sub(1, #prefix) == prefix and trimmed:sub(-1) == ")" then + local inner = trimmed:sub(#prefix + 1, #trimmed - 1) + local comma = M.find_byte(inner, 44, 1) -- ',' + if comma then + counts[M.trim(inner:sub(1, comma - 1))] = + tonumber(M.trim(inner:sub(comma + 1))) + end + end + i = line_end + 1 + end + return counts +end + +-- ════════════════════════════════════════════════════════════════════════════ +-- Section 6: LineIndex (perf fix — replaces the per-call rescan line_of) +-- ════════════════════════════════════════════════════════════════════════════ + +function M.LineIndex(source) + local positions = {} + local n = 0 + for i = 1, #source do + if source:byte(i) == 10 then -- '\n' + n = n + 1 + positions[n] = i + end + end + local function line_of(pos) + local lo, hi = 1, n + while lo <= hi do + local mid = math.floor((lo + hi) / 2) + if positions[mid] <= pos then + lo = mid + 1 + else + hi = mid - 1 + end + end + return hi + 1 + end + return line_of +end + +-- ════════════════════════════════════════════════════════════════════════════ +-- Section 7: domain tables +-- ════════════════════════════════════════════════════════════════════════════ + +M.WAVE_CONTEXT_REGS = { + ["R_PrimCursor"] = { alias = "R_T7", size = 4, role = "output cursor (prim arena)" }, + ["R_FaceCursor"] = { alias = "R_T4", size = 4, role = "input cursor (face array)" }, + ["R_VertBase"] = { alias = "R_T5", size = 4, role = "base pointer (vertex array)" }, + ["R_OtBase"] = { alias = "R_T6", size = 4, role = "base pointer (ordering table)" }, +} + +M.MACRO_EXPANSION = { + ["phase_init"] = "init", + ["phase_bind"] = "bind", + ["phase_setup"] = "setup", + ["phase_work"] = "work", + ["phase_commit"] = "commit", + ["phase_terminate"] = "terminate", + ["REGION_PRIM_ARENA"] = "prim_arena", + ["REGION_FACE_ARENA"] = "face_arena", + ["REGION_VERTEX_ARENA"] = "vertex_arena", + ["REGION_OT_ARENA"] = "ot_arena", + ["REGION_HEAP_3D"] = "heap_3d_models", + ["REGION_CDROM_STREAM"] = "cdrom_stream", + ["REGION_VRAM"] = "vram_heap", + ["CADENCE_FRAME"] = "frame", + ["CADENCE_ONCE"] = "once", + ["CADENCE_ONDEMAND"] = "ondemand", +} + +M.KNOWN_PHASES = { + ["init"] = true, ["bind"] = true, ["setup"] = true, + ["work"] = true, ["commit"] = true, ["terminate"] = true, +} +M.KNOWN_REGIONS = { + ["prim_arena"] = true, ["face_arena"] = true, + ["vertex_arena"] = true, ["ot_arena"] = true, + ["heap_3d_models"] = true, ["cdrom_stream"] = true, + ["vram_heap"] = true, +} +M.KNOWN_CADENCES = { + ["frame"] = true, ["once"] = true, ["ondemand"] = true, +} + +M.TAPE_ATOM_MACROS = { + ["atom_annot"] = { kind = "work", binds = false }, + ["atom_bind"] = { kind = "bind", binds = true }, + ["atom_setup"] = { kind = "setup", binds = false }, + ["atom_commit"] = { kind = "commit", binds = false }, + ["atom_init"] = { kind = "init", binds = false }, + ["atom_terminate"] = { kind = "terminate", binds = false }, +} + +M.ATOM_PRAGMA_KINDS = { + ["resource"] = { kind = "string" }, + ["region"] = { kind = "ident", allowed = M.KNOWN_REGIONS }, + ["group"] = { kind = "ident" }, + ["cadence"] = { kind = "ident", allowed = M.KNOWN_CADENCES }, + ["async"] = { kind = "ident", allowed = { ["true"] = true, ["false"] = true } }, +} + +-- Expose the lpeg_ok flag so callers can detect the LPeg-back path. +-- True when LPeg was successfully required and the patterns above were +-- compiled at module load time. False when running in fallback mode. +M.lpeg_ok = lpeg_ok + +return M \ No newline at end of file diff --git a/scripts/tape_atom.offset_gen.meta.lua b/scripts/tape_atom.offset_gen.meta.lua index 65f59c6..f613bf6 100644 --- a/scripts/tape_atom.offset_gen.meta.lua +++ b/scripts/tape_atom.offset_gen.meta.lua @@ -20,271 +20,107 @@ -- #pragma endregion -- -- Usage: --- lua gen_atom_offsets.lua [source2 ...] +-- luajit gen_atom_offsets.lua [source2 ...] --- ============================================================ --- Character classification --- ============================================================ - -local function is_space(c) return c == " " or c == "\t" or c == "\n" or c == "\r" or c == "\v" or c == "\f" end -local function is_alpha(c) - if not c or #c == 0 then return false end - if c >= "a" and c <= "z" then return true end - if c >= "A" and c <= "Z" then return true end - return c == "_" +-- Make require("duffle") resolve to the sibling duffle.lua in this dir, +-- AND make require("lpeg") find the vendored LPeg DLL in the toolchain. +-- Both prepends are explicit (no :match / no Lua pattern — plain byte scan). +local script_path = arg[0] +local last_sep = 0 +for i = 1, #script_path do + local c = script_path:sub(i, i) + if c == "/" or c == "\\" then last_sep = i end end +local script_dir = last_sep == 0 and "./" or script_path:sub(1, last_sep) +package.path = script_dir .. "?.lua;" .. script_dir .. "?/init.lua;" .. package.path +package.cpath = "C:\\projects\\Pikuma\\ps1\\toolchain\\luajit-2.1\\lib\\lua\\5.1\\?.dll;" .. package.cpath -local function is_digit(c) return c and c >= "0" and c <= "9" end -local function is_alnum(c) return is_alpha(c) or is_digit(c) end +-- Shared primitives + domain tables live in scripts/duffle.lua. +local duffle = require("duffle") + +-- Local aliases so the rest of this file reads cleanly. These resolve +-- to the same functions in duffle.lua (5.3-compatible, no regex). +local is_space = duffle.is_space +local is_alpha = duffle.is_alpha +local is_alnum = duffle.is_alnum +local trim = duffle.trim +local find_byte = duffle.find_byte +local read_file = duffle.read_file +local write_file = duffle.write_file +local ensure_dir = duffle.ensure_dir +local dirname = duffle.dirname +local basename_no_ext = duffle.basename_no_ext +local skip_str_or_cmt = duffle.skip_str_or_cmt +local skip_ws_and_cmt = duffle.skip_ws_and_cmt +local read_ident = duffle.read_ident +local read_parens = duffle.read_parens +local read_braces = duffle.read_braces +local read_brackets = duffle.read_brackets +local scan_to_char = duffle.scan_to_char +local split_top_level_commas = duffle.split_top_level_commas +local load_word_counts = duffle.load_word_counts -- ============================================================ --- I/O +-- Offset-gen-specific helpers (not in duffle.lua) -- ============================================================ -local function read_file(path) - local f = io.open(path, "r") - if not f then error("Cannot open " .. path) end - local content = f:read("*a") - f:close() - return content -end - -local function write_file(path, content) - local f = io.open(path, "w") - if not f then error("Cannot write " .. path) end - f:write(content) - f:close() -end - --- PowerShell aliases `mkdir` to New-Item, which treats `-p` as a path, so guard the call. -local function ensure_dir(path) - local is_win = package.config:sub(1, 1) == "\\" - os.execute(is_win and ('if not exist "' .. path .. '" mkdir "' .. path .. '"') or ('mkdir -p "' .. path .. '" 2>/dev/null')) -end - --- ============================================================ --- String primitives --- ============================================================ - -local function trim(s) - local a = 1; while a <= #s and is_space(s:sub(a, a)) do a = a + 1 end - local b = #s; while b >= a and is_space(s:sub(b, b)) do b = b - 1 end - return s:sub(a, b) -end - local function starts_with(s, prefix) - if #s < #prefix then return false end - for i = 1, #prefix do - if s:sub(i, i) ~= prefix:sub(i, i) then return false end - end - return true + if #s < #prefix then return false end + for i = 1, #prefix do + if s:sub(i, i) ~= prefix:sub(i, i) then return false end + end + return true end local function ends_with(s, suffix) - if #s < #suffix then return false end - local off = #s - #suffix - for i = 1, #suffix do - if s:sub(off + i, off + i) ~= suffix:sub(i, i) then return false end - end - return true -end - -local function find_byte(haystack, target, start) - for i = start or 1, #haystack do - if haystack:sub(i, i) == target then return i end - end - return nil -end - -local function dirname(path) - local last_sep = 0 - for i = 1, #path do - local c = path:sub(i, i) - if c == "/" or c == "\\" then last_sep = i end - end - if last_sep == 0 then return "." end - return path:sub(1, last_sep - 1) -end - -local function basename_no_ext(path) - local last_sep = 0 - for i = 1, #path do - local c = path:sub(i, i) - if c == "/" or c == "\\" then last_sep = i end - end - local a = last_sep + 1 - local last_dot = #path + 1 - for i = #path, a, -1 do - if path:sub(i, i) == "." then last_dot = i; break end - end - return path:sub(a, last_dot - 1) + if #s < #suffix then return false end + local off = #s - #suffix + for i = 1, #suffix do + if s:sub(off + i, off + i) ~= suffix:sub(i, i) then return false end + end + return true end local function to_upper(s) return s:upper() end local function to_alnum_underscore(s) - local out = "" - for i = 1, #s do - local c = s:sub(i, i) - if is_alnum(c) then out = out .. c - else out = out .. "_" end - end - return out + local out = "" + for i = 1, #s do + local c = s:sub(i, i) + if is_alnum(c) then out = out .. c + else out = out .. "_" end + end + return out end local function pad_right(s, w) return s .. string.rep(" ", w - #s) end --- ============================================================ --- Lexer helpers --- ============================================================ - --- If position i starts a C string literal ("..."), char literal ('.'), --- // line comment, or /* block comment, advance past it and return the --- position just after the construct (or #s+1 if unterminated). --- Otherwise return i unchanged. -local function skip_str_or_cmt(s, i) - local c = s:sub(i, i) - if c == '"' or c == "'" then - i = i + 1 - while i <= #s do - if s:sub(i, i) == "\\" then i = i + 2 - elseif s:sub(i, i) == c then return i + 1 - else i = i + 1 end - end - return #s + 1 - elseif c == "/" then - local nx = s:sub(i+1, i+1) - if nx == "/" then - while i <= #s and s:sub(i, i) ~= "\n" do i = i + 1 end - return i - elseif nx == "*" then - i = i + 2 - while i <= #s - 1 do - if s:sub(i, i) == "*" and s:sub(i+1, i+1) == "/" then - return i + 2 - end - i = i + 1 - end - return #s + 1 - end - end - return i -end - -local function skip_ws_and_cmt(s, i) - while i <= #s do - if is_space(s:sub(i, i)) then i = i + 1 - else - local nx = skip_str_or_cmt(s, i) - if nx > i then i = nx else break end - end - end - return i -end - -local function read_ident(source, i) - if not is_alpha(source:sub(i, i)) then return nil, i end - local a = i - i = i + 1 - while i <= #source and is_alnum(source:sub(i, i)) do i = i + 1 end - return source:sub(a, i - 1), i -end - -local function read_balanced(s, open_char, close_char, i) - if s:sub(i, i) ~= open_char then return nil, i end - i = i + 1 - local len = #s - local depth = 1 - local a = i - while i <= len and depth > 0 do - local c = s:sub(i, i) - if c == open_char then - depth = depth + 1 - i = i + 1 - elseif c == close_char then - depth = depth - 1 - if depth == 0 then break end - i = i + 1 - else - local nx = skip_str_or_cmt(s, i) - if nx > i then i = nx else i = i + 1 end - end - end - return s:sub(a, i - 1), i + 1 -end - -local read_parens = function(s, i) return read_balanced(s, "(", ")", i) end -local read_braces = function(s, i) return read_balanced(s, "{", "}", i) end -local read_brackets = function(s, i) return read_balanced(s, "[", "]", i) end - -local function scan_to_char(s, target, start) - local i = start - while i <= #s do - local c = s:sub(i, i) - if c == target then return i end - if c == "(" then local _, a = read_balanced(s, "(", ")", i); i = a - elseif c == "{" then local _, a = read_balanced(s, "{", "}", i); i = a - elseif c == "[" then local _, a = read_balanced(s, "[", "]", i); i = a - else - local nx = skip_str_or_cmt(s, i) - if nx > i then i = nx else i = i + 1 end - end - end -end - -- ============================================================ -- Extract comma-separated identifier args from a parenthesized group -- after a function-like macro call. -- ============================================================ local function extract_ident_args(token, after_ident) - local arg_start = skip_ws_and_cmt(token, after_ident) - if token:sub(arg_start, arg_start) ~= "(" then return {}, nil end - local inner, after_paren = read_parens(token, arg_start) + local arg_start = skip_ws_and_cmt(token, after_ident) + if token:sub(arg_start, arg_start) ~= "(" then return {}, nil end + local inner, after_paren = read_parens(token, arg_start) - local args = {} - local n = 1 - local len = #inner - while n <= len do - n = skip_ws_and_cmt(inner, n) - if n > len then break end - local ident, after = read_ident(inner, n) - if ident and ident ~= "" then - table.insert(args, ident) - n = after - else - n = n + 1 - end - n = skip_ws_and_cmt(inner, n) - if n <= len and inner:sub(n, n) == "," then n = n + 1 end - end + local args = {} + local n = 1 + local len = #inner + while n <= len do + n = skip_ws_and_cmt(inner, n) + if n > len then break end + local ident, after = read_ident(inner, n) + if ident and ident ~= "" then + table.insert(args, ident) + n = after + else + n = n + 1 + end + n = skip_ws_and_cmt(inner, n) + if n <= len and inner:sub(n, n) == "," then n = n + 1 end + end - return args, after_paren -end - --- ============================================================ --- Load WORD_COUNT manifest --- ============================================================ - -local function load_word_counts(metadata_path) - local counts = {} - local content = read_file(metadata_path) - local len = #content - local i = 1 - local prefix = "WORD_COUNT(" - while i <= len do - local nl = find_byte(content, "\n", i) - local line_end = nl or (len + 1) - local line = content:sub(i, line_end - 1) - local trimmed = trim(line) - if starts_with(trimmed, prefix) and ends_with(trimmed, ")") then - local inner = trimmed:sub(#prefix + 1, #trimmed - 1) - local comma = find_byte(inner, ",", 1) - if comma then - counts[trim(inner:sub(1, comma - 1))] = tonumber(trim(inner:sub(comma + 1))) - end - end - i = line_end + 1 - end - return counts + return args, after_paren end -- ============================================================ @@ -292,43 +128,16 @@ end -- ============================================================ local function word_count_of_token(token, wc) - local s = trim(token) - if s == "" then return 0 end - local name, after = read_ident(s, 1) - if not name then return 1 end - if wc[name] then return wc[name] end - local j = skip_ws_and_cmt(s, after) - if s:sub(j, j) == "(" then - io.stderr:write(" warning: unknown macro '" .. name .. "', assuming 1 word\n") - end - return 1 -end - --- ============================================================ --- Split brace-body into top-level comma-separated tokens --- ============================================================ - -local function split_top_level_commas(body) - local tokens = {} - local i = 1 - local token_start = 1 - while i <= #body do - local c = body:sub(i, i) - if c == "(" then local _, a = read_parens(body, i); i = a - elseif c == "{" then local _, a = read_braces(body, i); i = a - elseif c == "[" then local _, a = read_brackets(body, i); i = a - elseif c == "," then - table.insert(tokens, body:sub(token_start, i - 1)) - i = i + 1 - token_start = i - else - local nx = skip_str_or_cmt(body, i) - if nx > i then i = nx else i = i + 1 end - end - end - local last = body:sub(token_start) - if trim(last) ~= "" then table.insert(tokens, last) end - return tokens + local s = trim(token) + if s == "" then return 0 end + local name, after = read_ident(s, 1) + if not name then return 1 end + if wc[name] then return wc[name] end + local j = skip_ws_and_cmt(s, after) + if s:sub(j, j) == "(" then + io.stderr:write(" warning: unknown macro '" .. name .. "', assuming 1 word\n") + end + return 1 end -- ============================================================ @@ -337,54 +146,95 @@ end -- ============================================================ local function scan_for_atom_markers(token, at_pos, labels, branches) - local i = 1 - local len = #token - while i <= len do - i = skip_ws_and_cmt(token, i) - if i > len then break end - local c = token:sub(i, i) - if is_alpha(c) then - local ident, after = read_ident(token, i) - if ident == "atom_label" then - local args, after_paren = extract_ident_args(token, after) - if #args >= 1 then labels[args[1]] = at_pos end - if after_paren then i = after_paren else i = after end - elseif ident == "atom_offset" then - local args, after_paren = extract_ident_args(token, after) - if #args >= 2 then table.insert(branches, {pos = at_pos, target = args[2], tag = args[1]}) end - if after_paren then i = after_paren else i = after end - else - i = after - end - else - local nx = skip_str_or_cmt(token, i) - if nx > i then i = nx else i = i + 1 end - end - end + local i = 1 + local len = #token + while i <= len do + i = skip_ws_and_cmt(token, i) + if i > len then break end + local c = token:sub(i, i) + if is_alpha(c) then + local ident, after = read_ident(token, i) + if ident == "atom_label" then + local args, after_paren = extract_ident_args(token, after) + if #args >= 1 then labels[args[1]] = at_pos end + if after_paren then i = after_paren else i = after end + elseif ident == "atom_offset" then + local args, after_paren = extract_ident_args(token, after) + if #args >= 2 then table.insert(branches, {pos = at_pos, target = args[2], tag = args[1]}) end + if after_paren then i = after_paren else i = after end + else + i = after + end + else + local nx = skip_str_or_cmt(token, i) + if nx > i then i = nx else i = i + 1 end + end + end end -- ============================================================ -- Scan atom body, count words, find markers -- ============================================================ +-- Find the end position (just past the closing ')') of the first +-- atom_label/atom_offset call in `tok`. Returns 0 if no such call. +local function find_marker_call_end(tok) + local i = 1 + local len = #tok + while i <= len do + i = skip_ws_and_cmt(tok, i) + if i > len then break end + local c = tok:sub(i, i) + if is_alpha(c) then + local ident, after = read_ident(tok, i) + if ident == "atom_label" or ident == "atom_offset" then + local j = skip_ws_and_cmt(tok, after) + if tok:sub(j, j) == "(" then + local _, end_paren = read_parens(tok, j) + return end_paren - 1 + end + return 0 + end + i = after + else + local nx = skip_str_or_cmt(tok, i) + if nx > i then i = nx else i = i + 1 end + end + end + return 0 +end + local function scan_atom_body(body, word_counts) - local pos = 0 - local labels = {} - local branches = {} - for _, tok in ipairs(split_top_level_commas(body)) do - local k = 1 - local tlen = #tok - while k <= tlen and is_space(tok:sub(k, k)) do k = k + 1 end - local leading_ident = read_ident(tok, k) - if leading_ident == "atom_label" or leading_ident == "atom_offset" then - scan_for_atom_markers(tok, pos, labels, branches) - else - local words = word_count_of_token(tok, word_counts) - scan_for_atom_markers(tok, pos, labels, branches) - pos = pos + words - end - end - return labels, branches, pos + local pos = 0 + local labels = {} + local branches = {} + for _, tok in ipairs(split_top_level_commas(body)) do + local k = 1 + local tlen = #tok + while k <= tlen and is_space(tok:sub(k, k)) do k = k + 1 end + local leading_ident = read_ident(tok, k) + if leading_ident == "atom_label" or leading_ident == "atom_offset" then + -- Marker call: record at the current pos, do NOT advance pos. + -- But the source pattern may bundle the marker with the next + -- instruction on a new line (no top-level comma between them). + -- In that case, the rest of `tok` after the marker call is a + -- real instruction that must still be counted. + scan_for_atom_markers(tok, pos, labels, branches) + local marker_end = find_marker_call_end(tok) + if marker_end > 0 and marker_end < #tok then + local rest = trim(tok:sub(marker_end + 1)) + if rest ~= "" then + local rest_words = word_count_of_token(rest, word_counts) + pos = pos + rest_words + end + end + else + local words = word_count_of_token(tok, word_counts) + scan_for_atom_markers(tok, pos, labels, branches) + pos = pos + words + end + end + return labels, branches, pos end -- ============================================================ @@ -392,81 +242,81 @@ end -- ============================================================ local function skip_qualifiers(source, i) - local keywords = { - ["static"] = true, ["const"] = true, ["volatile"] = true, - ["extern"] = true, ["register"] = true, ["auto"] = true, - ["inline"] = true, ["typedef"] = true, - ["internal"]= true, ["LP_"] = true, ["global"] = true, ["gkknown"] = true - } - while true do - i = skip_ws_and_cmt(source, i) - local ident, after = read_ident(source, i) - if not ident then return i end - if keywords[ident] then i = after else return i end - end + local keywords = { + ["static"] = true, ["const"] = true, ["volatile"] = true, + ["extern"] = true, ["register"] = true, ["auto"] = true, + ["inline"] = true, ["typedef"] = true, + ["internal"]= true, ["LP_"] = true, ["global"] = true, ["gkknown"] = true + } + while true do + i = skip_ws_and_cmt(source, i) + local ident, after = read_ident(source, i) + if not ident then return i end + if keywords[ident] then i = after else return i end + end end local function find_atoms(source_text) - local atoms = {} - local len = #source_text - local i = 1 + local atoms = {} + local len = #source_text + local i = 1 - local function try_wrapped(after_pos) - local paren_pos = skip_ws_and_cmt(source_text, after_pos) - if source_text:sub(paren_pos, paren_pos) ~= "(" then return nil end - local inner, after_paren = read_parens(source_text, paren_pos) - local n = 1 - while n <= #inner and is_space(inner:sub(n, n)) do n = n + 1 end - local ns = n - while n <= #inner and is_alnum(inner:sub(n, n)) do n = n + 1 end - local name = inner:sub(ns, n - 1) - if name == "" then return nil end - local brace_pos = scan_to_char(source_text, "{", after_paren) - if not brace_pos then return nil end - local body, after_brace = read_braces(source_text, brace_pos) - return {name = name, body = body, after_brace = after_brace} - end + local function try_wrapped(after_pos) + local paren_pos = skip_ws_and_cmt(source_text, after_pos) + if source_text:sub(paren_pos, paren_pos) ~= "(" then return nil end + local inner, after_paren = read_parens(source_text, paren_pos) + local n = 1 + while n <= #inner and is_space(inner:sub(n, n)) do n = n + 1 end + local ns = n + while n <= #inner and is_alnum(inner:sub(n, n)) do n = n + 1 end + local name = inner:sub(ns, n - 1) + if name == "" then return nil end + local brace_pos = scan_to_char(source_text, "{", after_paren) + if not brace_pos then return nil end + local body, after_brace = read_braces(source_text, brace_pos) + return {name = name, body = body, after_brace = after_brace} + end - local function try_raw(after_pos) - local next_pos = skip_ws_and_cmt(source_text, after_pos) - local next_ident, next_after = read_ident(source_text, next_pos) - if not next_ident then return nil end - if not starts_with(next_ident, "code_") then return nil end - if #next_ident <= 5 then return nil end - local atom_name = next_ident:sub(6) - local brace_pos = scan_to_char(source_text, "{", next_after) - if not brace_pos then return nil end - local body, after_brace = read_braces(source_text, brace_pos) - return {name = atom_name, body = body, after_brace = after_brace} - end + local function try_raw(after_pos) + local next_pos = skip_ws_and_cmt(source_text, after_pos) + local next_ident, next_after = read_ident(source_text, next_pos) + if not next_ident then return nil end + if not starts_with(next_ident, "code_") then return nil end + if #next_ident <= 5 then return nil end + local atom_name = next_ident:sub(6) + local brace_pos = scan_to_char(source_text, "{", next_after) + if not brace_pos then return nil end + local body, after_brace = read_braces(source_text, brace_pos) + return {name = atom_name, body = body, after_brace = after_brace} + end - while i <= len do - i = skip_ws_and_cmt(source_text, i); if i > len then break end - i = skip_qualifiers(source_text, i); if i > len then break end - local ident, after = read_ident(source_text, i) - if not ident then - i = i + 1 - elseif ident == "MipsAtom_" then - local atom = try_wrapped(after) - if atom then - table.insert(atoms, {name = atom.name, body = atom.body}) - i = atom.after_brace - else - i = i + 1 - end - elseif ident == "MipsCode" then - local atom = try_raw(after) - if atom then - table.insert(atoms, {name = atom.name, body = atom.body}) - i = atom.after_brace - else - i = after - end - else - i = after - end - end - return atoms + while i <= len do + i = skip_ws_and_cmt(source_text, i); if i > len then break end + i = skip_qualifiers(source_text, i); if i > len then break end + local ident, after = read_ident(source_text, i) + if not ident then + i = i + 1 + elseif ident == "MipsAtom_" then + local atom = try_wrapped(after) + if atom then + table.insert(atoms, {name = atom.name, body = atom.body}) + i = atom.after_brace + else + i = i + 1 + end + elseif ident == "MipsCode" then + local atom = try_raw(after) + if atom then + table.insert(atoms, {name = atom.name, body = atom.body}) + i = atom.after_brace + else + i = after + end + else + i = after + end + end + return atoms end -- ============================================================ @@ -474,15 +324,15 @@ end -- ============================================================ local function compute_offsets(labels, branches) - local results = {} - for _, br in ipairs(branches) do - local target = labels[br.target] - if not target then - error("Branch target '" .. br.target .. "' has no atom_label (at word " .. br.pos .. ")") - end - table.insert(results, {target = br.target, tag = br.tag, offset = target - br.pos - 1 }) - end - return results + local results = {} + for _, br in ipairs(branches) do + local target = labels[br.target] + if not target then + error("Branch target '" .. br.target .. "' has no atom_label (at word " .. br.pos .. ")") + end + table.insert(results, {target = br.target, tag = br.tag, offset = target - br.pos - 1 }) + end + return results end -- ============================================================ @@ -490,45 +340,45 @@ end -- ============================================================ local function generate_header(source_path, atoms_data) - local basename = basename_no_ext(source_path) - local guard = to_alnum_underscore(to_upper(basename)) .. "_OFFSETS_H" + local basename = basename_no_ext(source_path) + local guard = to_alnum_underscore(to_upper(basename)) .. "_OFFSETS_H" - local lines = {} - local function add(s) table.insert(lines, s) end + local lines = {} + local function add(s) table.insert(lines, s) end - add("// Auto-generated by tape_atom_offset_gen.meta.lua — DO NOT EDIT") - add("// Source: " .. source_path) - add("#pragma once") - add("") - add("#pragma region " .. basename) - add("") - -- add("// Dispatch macro: token-pastes _ to the enum name") - -- add("#undef atom_offset") - -- add("#define atom_offset(tag, name) atom_offset_##tag##_##name") - add("") - for _, atom in ipairs(atoms_data) do - if #atom.offsets > 0 then - add("// --- atom: " .. atom.name .. " (" .. atom.total_words .. " words) ---") - add("") - local consts = {} - for _, r in ipairs(atom.offsets) do - table.insert(consts, { - macro_name = "_atom_offset_" .. r.tag .. "_" .. r.target, - enum_name = "atom_offset_" .. r.tag .. "_" .. r.target, - value = r.offset - }) - end - for _, c in ipairs(consts) do add("#define " .. pad_right(c.macro_name, 44) .. " " .. c.value .. "") end - add("") - add("enum {") - for _, c in ipairs(consts) do add(" " .. c.enum_name .. " = " .. c.macro_name .. ",") end - add("};") - add("") - end - end - add("#pragma endregion " .. basename) - add("") - return table.concat(lines, "\n") .. "\n" + add("// Auto-generated by tape_atom_offset_gen.meta.lua — DO NOT EDIT") + add("// Source: " .. source_path) + add("#pragma once") + add("") + add("#pragma region " .. basename) + add("") + -- add("// Dispatch macro: token-pastes _ to the enum name") + -- add("#undef atom_offset") + -- add("#define atom_offset(tag, name) atom_offset_##tag##_##name") + add("") + for _, atom in ipairs(atoms_data) do + if #atom.offsets > 0 then + add("// --- atom: " .. atom.name .. " (" .. atom.total_words .. " words) ---") + add("") + local consts = {} + for _, r in ipairs(atom.offsets) do + table.insert(consts, { + macro_name = "_atom_offset_" .. r.tag .. "_" .. r.target, + enum_name = "atom_offset_" .. r.tag .. "_" .. r.target, + value = r.offset + }) + end + for _, c in ipairs(consts) do add("#define " .. pad_right(c.macro_name, 44) .. " " .. c.value .. "") end + add("") + add("enum {") + for _, c in ipairs(consts) do add(" " .. c.enum_name .. " = " .. c.macro_name .. ",") end + add("};") + add("") + end + end + add("#pragma endregion " .. basename) + add("") + return table.concat(lines, "\n") .. "\n" end -- ============================================================ @@ -536,39 +386,39 @@ end -- ============================================================ local function process_source(source_path, word_counts) - local source = read_file(source_path) - local atoms_raw = find_atoms(source) + local source = read_file(source_path) + local atoms_raw = find_atoms(source) - if #atoms_raw == 0 then - -- io.stderr:write(" note: no MipsAtom_ declarations in " .. source_path .. "\n") - return - end + if #atoms_raw == 0 then + -- io.stderr:write(" note: no MipsAtom_ declarations in " .. source_path .. "\n") + return + end - local atoms_data = {} - for _, atom in ipairs(atoms_raw) do - local labels, branches, total = scan_atom_body(atom.body, word_counts) - local offsets = compute_offsets(labels, branches) - table.insert(atoms_data, { - name = atom.name, - total_words = total, - offsets = offsets - }) - end + local atoms_data = {} + for _, atom in ipairs(atoms_raw) do + local labels, branches, total = scan_atom_body(atom.body, word_counts) + local offsets = compute_offsets(labels, branches) + table.insert(atoms_data, { + name = atom.name, + total_words = total, + offsets = offsets + }) + end - local basename = basename_no_ext(source_path) - local out_dir = dirname(source_path) .. "/gen" - ensure_dir(out_dir) - local out_path = out_dir .. "/" .. basename .. ".offsets.h" - write_file(out_path, generate_header(source_path, atoms_data)) + local basename = basename_no_ext(source_path) + local out_dir = dirname(source_path) .. "/gen" + ensure_dir(out_dir) + local out_path = out_dir .. "/" .. basename .. ".offsets.h" + write_file(out_path, generate_header(source_path, atoms_data)) - local total_branches = 0 - for _, a in ipairs(atoms_data) do total_branches = total_branches + #a.offsets end - print(" " .. basename .. ": " .. #atoms_data .. " atom(s), " .. total_branches .. " branch(es)") - for _, a in ipairs(atoms_data) do - for _, r in ipairs(a.offsets) do - print(" " .. a.name .. " -> " .. r.tag .. ":" .. r.target .. " : " .. r.offset) - end - end + local total_branches = 0 + for _, a in ipairs(atoms_data) do total_branches = total_branches + #a.offsets end + print(" " .. basename .. ": " .. #atoms_data .. " atom(s), " .. total_branches .. " branch(es)") + for _, a in ipairs(atoms_data) do + for _, r in ipairs(a.offsets) do + print(" " .. a.name .. " -> " .. r.tag .. ":" .. r.target .. " : " .. r.offset) + end + end end -- ============================================================ @@ -576,12 +426,12 @@ end -- ============================================================ local function main(args) - if #args < 2 then - print("Usage: gen_atom_offsets.lua [source2 ...]") - os.exit(1) - end - local word_counts = load_word_counts(args[1]) - for i = 2, #args do process_source(args[i], word_counts) end + if #args < 2 then + print("Usage: luajit gen_atom_offsets.lua [source2 ...]") + os.exit(1) + end + local word_counts = load_word_counts(args[1]) + for i = 2, #args do process_source(args[i], word_counts) end end -main({...}) +main({...}) \ No newline at end of file diff --git a/scripts/tape_atom_annotation_pass.lua b/scripts/tape_atom_annotation_pass.lua index bde994d..35653f1 100644 --- a/scripts/tape_atom_annotation_pass.lua +++ b/scripts/tape_atom_annotation_pass.lua @@ -25,255 +25,111 @@ -- /gen/annotation_validation.txt -- -- Usage: --- lua tape_atom_annotation_pass.lua [source2 ...] +-- luajit tape_atom_annotation_pass.lua [source2 ...] -- -- This script is meant to be run alongside tape_atom_offset_gen.lua. -- Same input files, complementary output. Both feed the build. --- ============================================================ --- Shared primitives (copied from tape_atom_offset_gen.lua so this --- script can stand alone; if you require() the original instead, --- these become the original's locals). --- ============================================================ - -local function is_space(c) return c == " " or c == "\t" or c == "\n" or c == "\r" or c == "\v" or c == "\f" end -local function is_alpha(c) - if not c or #c == 0 then return false end - if c >= "a" and c <= "z" then return true end - if c >= "A" and c <= "Z" then return true end - return c == "_" +-- Make require("duffle") resolve to the sibling duffle.lua in this dir, +-- AND make require("lpeg") find the vendored LPeg DLL in the toolchain. +-- Both prepends are explicit (no :match / no Lua pattern — plain byte scan). +local script_path = arg[0] +local last_sep = 0 +for i = 1, #script_path do + local c = script_path:sub(i, i) + if c == "/" or c == "\\" then last_sep = i end end -local function is_digit(c) return c and c >= "0" and c <= "9" end -local function is_alnum(c) return is_alpha(c) or is_digit(c) end +local script_dir = last_sep == 0 and "./" or script_path:sub(1, last_sep) +package.path = script_dir .. "?.lua;" .. script_dir .. "?/init.lua;" .. package.path +package.cpath = "C:\\projects\\Pikuma\\ps1\\toolchain\\luajit-2.1\\lib\\lua\\5.1\\?.dll;" .. package.cpath -local function trim(s) - local a = 1; while a <= #s and is_space(s:sub(a, a)) do a = a + 1 end - local b = #s; while b >= a and is_space(s:sub(b, b)) do b = b - 1 end - return s:sub(a, b) -end +-- Shared primitives + domain tables live in scripts/duffle.lua. +local duffle = require("duffle") -local function find_byte(haystack, target, start) - for i = start or 1, #haystack do - if haystack:sub(i, i) == target then return i end - end - return nil -end +-- Local aliases so the rest of this file reads cleanly. These resolve +-- to the same functions in duffle.lua (5.3-compatible, no regex). +local is_space = duffle.is_space +local is_alpha = duffle.is_alpha +local is_alnum = duffle.is_alnum +local trim = duffle.trim +local find_byte = duffle.find_byte +local read_file = duffle.read_file +local write_file = duffle.write_file +local ensure_dir = duffle.ensure_dir +local dirname = duffle.dirname +local basename_no_ext = duffle.basename_no_ext +local skip_str_or_cmt = duffle.skip_str_or_cmt +local skip_ws_and_cmt = duffle.skip_ws_and_cmt +local read_ident = duffle.read_ident +local read_parens = duffle.read_parens +local read_braces = duffle.read_braces +local scan_to_char = duffle.scan_to_char +local load_word_counts = duffle.load_word_counts -local function read_file(path) - local f = io.open(path, "r") - if not f then error("Cannot open " .. path) end - local content = f:read("*a") - f:close() - return content -end +-- Domain tables (single source of truth in duffle.lua). +local WAVE_CONTEXT_REGS = duffle.WAVE_CONTEXT_REGS +local MACRO_EXPANSION = duffle.MACRO_EXPANSION +local KNOWN_PHASES = duffle.KNOWN_PHASES +local KNOWN_REGIONS = duffle.KNOWN_REGIONS +local KNOWN_CADENCES = duffle.KNOWN_CADENCES +local TAPE_ATOM_MACROS = duffle.TAPE_ATOM_MACROS +local ATOM_PRAGMA_KINDS = duffle.ATOM_PRAGMA_KINDS -local function write_file(path, content) - local f = io.open(path, "w") - if not f then error("Cannot write " .. path) end - f:write(content) - f:close() -end - -local function dirname(path) - local last_sep = 0 - for i = 1, #path do - local c = path:sub(i, i) - if c == "/" or c == "\\" then last_sep = i end - end - if last_sep == 0 then return "." end - return path:sub(1, last_sep - 1) -end - -local function basename_no_ext(path) - local last_sep = 0 - for i = 1, #path do - local c = path:sub(i, i) - if c == "/" or c == "\\" then last_sep = i end - end - local a = last_sep + 1 - local last_dot = #path + 1 - for i = #path, a, -1 do - if path:sub(i, i) == "." then last_dot = i; break end - end - return path:sub(a, last_dot - 1) -end - -local function ensure_dir(path) - local is_win = package.config:sub(1, 1) == "\\" - os.execute(is_win and ('if not exist "' .. path .. '" mkdir "' .. path .. '"') or ('mkdir -p "' .. path .. '" 2>/dev/null')) -end +local function valid_phase(p) return KNOWN_PHASES[p] or false end +local function is_wave_context_reg(n) return WAVE_CONTEXT_REGS[n] ~= nil end -- ============================================================ --- Lexer primitives (mirror the offset-gen tool) --- ============================================================ - -local function skip_str_or_cmt(s, i) - local c = s:sub(i, i) - if c == '"' or c == "'" then - i = i + 1 - while i <= #s do - if s:sub(i, i) == "\\" then i = i + 2 - elseif s:sub(i, i) == c then return i + 1 - else i = i + 1 end - end - return #s + 1 - elseif c == "/" then - local nx = s:sub(i+1, i+1) - if nx == "/" then - while i <= #s and s:sub(i, i) ~= "\n" do i = i + 1 end - return i - elseif nx == "*" then - i = i + 2 - while i <= #s - 1 do - if s:sub(i, i) == "*" and s:sub(i+1, i+1) == "/" then - return i + 2 - end - i = i + 1 - end - return #s + 1 - end - end - return i -end - -local function skip_ws_and_cmt(s, i) - while i <= #s do - if is_space(s:sub(i, i)) then i = i + 1 - else - local nx = skip_str_or_cmt(s, i) - if nx > i then i = nx else break end - end - end - return i -end - -local function read_ident(source, i) - if not is_alpha(source:sub(i, i)) then return nil, i end - local a = i - i = i + 1 - while i <= #source and is_alnum(source:sub(i, i)) do i = i + 1 end - return source:sub(a, i - 1), i -end - -local function read_balanced(s, open_char, close_char, i) - if s:sub(i, i) ~= open_char then return nil, i end - i = i + 1 - local len = #s - local depth = 1 - local a = i - while i <= len and depth > 0 do - local c = s:sub(i, i) - if c == open_char then - depth = depth + 1 - i = i + 1 - elseif c == close_char then - depth = depth - 1 - if depth == 0 then break end - i = i + 1 - else - local nx = skip_str_or_cmt(s, i) - if nx > i then i = nx else i = i + 1 end - end - end - return s:sub(a, i - 1), i + 1 -end - -local read_parens = function(s, i) return read_balanced(s, "(", ")", i) end -local read_braces = function(s, i) return read_balanced(s, "{", "}", i) end -local read_brackets = function(s, i) return read_balanced(s, "[", "]", i) end - -local function scan_to_char(s, target, start) - local i = start - while i <= #s do - local c = s:sub(i, i) - if c == target then return i end - if c == "(" then local _, a = read_balanced(s, "(", ")", i); i = a - elseif c == "{" then local _, a = read_balanced(s, "{", "}", i); i = a - elseif c == "[" then local _, a = read_balanced(s, "[", "]", i); i = a - else - local nx = skip_str_or_cmt(s, i) - if nx > i then i = nx else i = i + 1 end - end - end -end - --- ============================================================ --- Load WORD_COUNT manifest (mirror of the offset-gen tool) --- ============================================================ - -local function load_word_counts(metadata_path) - local counts = {} - local content = read_file(metadata_path) - local len = #content - local i = 1 - local prefix = "WORD_COUNT(" - while i <= len do - local nl = find_byte(content, "\n", i) - local line_end = nl or (len + 1) - local line = content:sub(i, line_end - 1) - local trimmed = trim(line) - if trimmed:sub(1, #prefix) == prefix and trimmed:sub(-1) == ")" then - local inner = trimmed:sub(#prefix + 1, #trimmed - 1) - local comma = find_byte(inner, ",", 1) - if comma then - counts[trim(inner:sub(1, comma - 1))] = tonumber(trim(inner:sub(comma + 1))) - end - end - i = line_end + 1 - end - return counts -end - --- ============================================================ --- Wave-context register aliases +-- Hand-rolled split helpers (no :gmatch / no regex) -- --- These are the canonical names of the registers that get bound --- into the wave by the rbind atoms. The annotation pass uses this --- table to validate Binds_* struct field names — every field that --- the bind writes must map to a known wave-context register. +-- Phase 2 keeps these inline; Phase 3 may LPeg-ify them. -- ============================================================ -local WAVE_CONTEXT_REGS = { - ["R_PrimCursor"] = { alias = "R_T7", size = 4, role = "output cursor (prim arena)" }, - ["R_FaceCursor"] = { alias = "R_T4", size = 4, role = "input cursor (face array)" }, - ["R_VertBase"] = { alias = "R_T5", size = 4, role = "base pointer (vertex array)" }, - ["R_OtBase"] = { alias = "R_T6", size = 4, role = "base pointer (ordering table)" }, -} +-- Split a string at top-level commas. Used inside TAPE_ATOM_* macro +-- bodies where nested parens/braces/brackets are possible. +local function split_csv_top(s) + local tokens = {} + local i, start = 1, 1 + local depth = 0 + while i <= #s do + local c = s:sub(i, i) + if c == "(" or c == "{" or c == "[" then + depth = depth + 1 + i = i + 1 + elseif c == ")" or c == "}" or c == "]" then + depth = depth - 1 + i = i + 1 + elseif c == "," and depth == 0 then + tokens[#tokens + 1] = s:sub(start, i - 1) + i = i + 1 + start = i + else + i = i + 1 + end + end + local last = s:sub(start) + if trim(last) ~= "" then tokens[#tokens + 1] = last end + return tokens +end --- Macro expansion table: maps source-level macro names to the value they expand --- to at preprocessing time. Used so the metaprogram can validate annotations --- against the un-preprocessed source. -local MACRO_EXPANSION = { - -- phase_* tokens - ["phase_init"] = "init", - ["phase_bind"] = "bind", - ["phase_setup"] = "setup", - ["phase_work"] = "work", - ["phase_commit"] = "commit", - ["phase_terminate"] = "terminate", - -- region tokens - ["REGION_PRIM_ARENA"] = "prim_arena", - ["REGION_FACE_ARENA"] = "face_arena", - ["REGION_VERTEX_ARENA"] = "vertex_arena", - ["REGION_OT_ARENA"] = "ot_arena", - ["REGION_HEAP_3D"] = "heap_3d_models", - ["REGION_CDROM_STREAM"] = "cdrom_stream", - ["REGION_VRAM"] = "vram_heap", - -- cadence tokens - ["CADENCE_FRAME"] = "frame", - ["CADENCE_ONCE"] = "once", - ["CADENCE_ONDEMAND"] = "ondemand", -} - --- Phases + regions + cadences are defined further below (after TAPE_ATOM_MACROS). --- We expose valid_phase via upvalue once KNOWN_PHASES is defined, but for now --- it's a closure-resolved at call time. - --- Table refs initialised later; valid_phase is hooked up below KNOWN_PHASES. -local KNOWN_PHASES, KNOWN_REGIONS, KNOWN_CADENCES -- forward declarations - -local function valid_phase(p) return KNOWN_PHASES and KNOWN_PHASES[p] or false end -local function is_wave_context_reg(name) return WAVE_CONTEXT_REGS[name] ~= nil end +-- Split a string into whitespace-separated tokens. +-- The original used :gmatch("%S+"); replaced here with a hand-rolled +-- loop (faster + no regex). +local function split_ws(s) + local tokens = {} + local i, n = 1, 1 + local len = #s + while i <= len do + -- Skip whitespace. + while i <= len and is_space(s:sub(i, i)) do i = i + 1 end + if i > len then break end + local start = i + -- Take non-whitespace run. + while i <= len and not is_space(s:sub(i, i)) do i = i + 1 end + tokens[n] = s:sub(start, i - 1) + n = n + 1 + end + return tokens +end -- ============================================================ -- Parse TAPE_ATOM_ANNOT(...) calls in source @@ -289,37 +145,6 @@ local function is_wave_context_reg(name) return WAVE_CONTEXT_REGS[name] ~= nil e -- in source order. -- ============================================================ -local function split_top_level_commas(body) - local tokens = {} - local i = 1 - local token_start = 1 - while i <= #body do - local c = body:sub(i, i) - if c == "(" then local _, a = read_parens(body, i); i = a - elseif c == "{" then local _, a = read_braces(body, i); i = a - elseif c == "[" then local _, a = read_brackets(body, i); i = a - elseif c == "," then - table.insert(tokens, body:sub(token_start, i - 1)) - i = i + 1 - token_start = i - else - local nx = skip_str_or_cmt(body, i) - if nx > i then i = nx else i = i + 1 end - end - end - local last = body:sub(token_start) - if trim(last) ~= "" then table.insert(tokens, last) end - return tokens -end - -local function line_of(source, pos) - local line = 1 - for i = 1, pos do - if source:sub(i, i) == "\n" then line = line + 1 end - end - return line -end - -- Extract identifier args from a parenthesized group. Returns a list -- of {kind, value} pairs where kind is one of: -- "ident" -- a bare identifier (e.g. phase_work) @@ -327,193 +152,50 @@ end -- "atom_writes"-- an atom_writes(...) call: value is the register list -- "other" -- something we can't classify (preserved as text) local function parse_atom_annot_args(inner) - -- Split at top-level commas, respecting nested parens. - local args = {} - local tokens = split_top_level_commas(inner) - for _, tok in ipairs(tokens) do - local s = trim(tok) - if s ~= "" then - -- Detect register-list calls: atom_reads(...) / atom_writes(...) / tape_regs(...) - local regs_kind = nil - local regs_inner = nil - if s:sub(-1) == ")" then - if s:sub(1, 11) == "atom_reads(" then - regs_kind = "atom_reads" - regs_inner = s:sub(12, -2) - elseif s:sub(1, 12) == "atom_writes(" then - regs_kind = "atom_writes" - regs_inner = s:sub(13, -2) - end - end + -- Split at top-level commas, respecting nested parens. + local args = {} + local tokens = split_csv_top(inner) + for _, tok in ipairs(tokens) do + local s = trim(tok) + if s ~= "" then + -- Detect register-list calls: atom_reads(...) / atom_writes(...) / tape_regs(...) + local regs_kind = nil + local regs_inner = nil + if s:sub(-1) == ")" then + if s:sub(1, 11) == "atom_reads(" then + regs_kind = "atom_reads" + regs_inner = s:sub(12, -2) + elseif s:sub(1, 12) == "atom_writes(" then + regs_kind = "atom_writes" + regs_inner = s:sub(13, -2) + end + end - if regs_kind then - local regs = {} - for r in regs_inner:gmatch("[^,]+") do - local trimmed = trim(r) - if trimmed ~= "" then table.insert(regs, trimmed) end - end - -- Resolve any phase_* / R_* alias macros - for i, r in ipairs(regs) do - if MACRO_EXPANSION[r] then regs[i] = MACRO_EXPANSION[r] end - end - table.insert(args, {kind = regs_kind, value = regs}) - else - -- Bare identifier (e.g. phase_work) - local id, _ = read_ident(s, 1) - if id and trim(s) == id then - local v = id - if MACRO_EXPANSION[v] then v = MACRO_EXPANSION[v] end - table.insert(args, {kind = "ident", value = v}) - else - table.insert(args, {kind = "other", value = s}) - end - end - end - end - return args -end - -local TAPE_ATOM_MACROS = { - ["atom_annot"] = { kind = "work", binds = false }, - ["atom_bind"] = { kind = "bind", binds = true }, - ["atom_setup"] = { kind = "setup", binds = false }, - ["atom_commit"] = { kind = "commit", binds = false }, - ["atom_init"] = { kind = "init", binds = false }, - ["atom_terminate"] = { kind = "terminate",binds = false }, -} - --- Phase token names (must match macros in tape_atom_dsl.h) -KNOWN_PHASES = { - ["init"] = true, ["bind"] = true, ["setup"] = true, - ["work"] = true, ["commit"] = true, ["terminate"] = true, -} - --- Region token names (must match REGION_* macros in tape_atom_dsl.h) -KNOWN_REGIONS = { - ["prim_arena"] = true, ["face_arena"] = true, ["vertex_arena"] = true, - ["ot_arena"] = true, ["heap_3d_models"] = true, ["cdrom_stream"] = true, - ["vram_heap"] = true, -} - --- Cadence token names (must match CADENCE_* macros in tape_atom_dsl.h) -KNOWN_CADENCES = { - ["frame"] = true, ["once"] = true, ["ondemand"] = true, -} - --- Valid pragmas (must match atom_* macros that emit `_Pragma(...)` in tape_atom_dsl.h) -local ATOM_PRAGMA_KINDS = { - ["resource"] = { kind = "string" }, - ["region"] = { kind = "ident", allowed = KNOWN_REGIONS }, - ["group"] = { kind = "ident" }, - ["cadence"] = { kind = "ident", allowed = KNOWN_CADENCES }, - ["async"] = { kind = "ident", allowed = { ["true"] = true, ["false"] = true } }, -} - -local function find_atom_annotations(source) - local annots = {} - local len = #source - local i = 1 - while i <= len do - i = skip_ws_and_cmt(source, i); if i > len then break end - -- Match the leading identifier of a TAPE_ATOM_* macro. - local ident, after = read_ident(source, i) - if not ident then - i = i + 1 - elseif TAPE_ATOM_MACROS[ident] then - local open = skip_ws_and_cmt(source, after) - if source:sub(open, open) == "(" then - local inner, after_paren = read_parens(source, open) - local args = parse_atom_annot_args(inner) - local macro_def = TAPE_ATOM_MACROS[ident] - - if #args < 1 then - table.insert(annots, { - line = line_of(source, i), - macro = ident, - kind = macro_def.kind, - error = "missing atom name (first arg)", - }) - else - local name = args[1].value - -- atom_bind: (name, Binds_Struct, writes) - -- atom_annot: (name, phase, reads, writes) - -- atom_setup / atom_commit: (name, reads) - -- atom_init / atom_terminate: (name) - local entry = { - line = line_of(source, i), - macro = ident, - name = name, - kind = macro_def.kind, - binds = nil, - phase = nil, - reads = {}, - writes = {}, - errors = {}, - } - - -- Is this arg a register-list call (any of the recognized forms)? - local function is_regs(a) - return a and (a.kind == "atom_reads" or a.kind == "atom_writes" or a.kind == "regs") - end - - if macro_def.binds then - -- atom_bind(name, Binds_Struct, writes) - if #args >= 2 and args[2].kind == "ident" then - entry.binds = args[2].value - end - if #args >= 3 and is_regs(args[3]) then - -- A bind writes the wave-context, so atom_writes(...) is the - -- canonical form, but legacy regs(...) is also accepted. - entry.writes = args[3].value - end - elseif ident == "atom_init" or ident == "atom_terminate" then - -- (name) only, no reads/writes to extract - elseif ident == "atom_setup" then - -- atom_setup(name, reads) - if #args >= 2 and is_regs(args[2]) then - entry.reads = args[2].value - end - elseif ident == "atom_commit" then - -- atom_commit(name, reads) - if #args >= 2 and is_regs(args[2]) then - entry.reads = args[2].value - end - elseif ident == "atom_annot" then - -- atom_annot(name, phase, reads, writes) - if #args >= 2 and args[2].kind == "ident" then - entry.phase = MACRO_EXPANSION[args[2].value] or args[2].value - end - -- reads slot: atom_reads(...) is the canonical form; - -- atom_writes(...) in the reads slot is a likely bug. - if #args >= 3 and is_regs(args[3]) then - if args[3].kind == "atom_writes" then - table.insert(entry.errors, - "reads slot has atom_writes — swap order?") - end - entry.reads = args[3].value - end - -- writes slot: atom_writes(...) is canonical; - -- atom_reads(...) here is a likely bug. - if #args >= 4 and is_regs(args[4]) then - if args[4].kind == "atom_reads" then - table.insert(entry.errors, - "writes slot has atom_reads — swap order?") - end - entry.writes = args[4].value - end - end - - table.insert(annots, entry) - end - i = after_paren - else - i = open + 1 - end - else - i = after - end - end - return annots + if regs_kind then + local regs = {} + for _, r in ipairs(split_csv_top(regs_inner)) do + local trimmed = trim(r) + if trimmed ~= "" then regs[#regs + 1] = trimmed end + end + -- Resolve any phase_* / R_* alias macros + for i, r in ipairs(regs) do + if MACRO_EXPANSION[r] then regs[i] = MACRO_EXPANSION[r] end + end + args[#args + 1] = {kind = regs_kind, value = regs} + else + -- Bare identifier (e.g. phase_work) + local id, _ = read_ident(s, 1) + if id and trim(s) == id then + local v = id + if MACRO_EXPANSION[v] then v = MACRO_EXPANSION[v] end + args[#args + 1] = {kind = "ident", value = v} + else + args[#args + 1] = {kind = "other", value = s} + end + end + end + end + return args end -- ============================================================ @@ -527,93 +209,89 @@ end -- ============================================================ local function find_macro_word_annotations(source) - local out = {} - local len = #source - local i = 1 - while i <= len do - i = skip_ws_and_cmt(source, i); if i > len then break end - -- Skip preprocessor directives (lines starting with #). - -- read_ident doesn't recognize '#' so without this guard we'd - -- infinite-loop on `#ifdef` / `#pragma region` / etc. - if source:sub(i, i) == "#" then - local j = i - while j <= len and source:sub(j, j) ~= "\n" do j = j + 1 end - i = j + 1 - else - local ident, after = read_ident(source, i) - if not ident then - i = i + 1 - elseif ident == "_Pragma" then - local open = skip_ws_and_cmt(source, after) - if source:sub(open, open) == "(" then - local str, str_end = read_parens(source, open) - -- str is a parenthesized string literal; strip parens, then quotes - str = trim(str) - if str:sub(1, 1) == '"' and str:sub(-1) == '"' then - local inner = str:sub(2, -2) - -- Expect " tape_atom words=" - local space = find_byte(inner, " ", 1) - if space then - local name = inner:sub(1, space - 1) - local rest = inner:sub(space + 1) - local eq = find_byte(rest, "=", 1) - if eq then - local key = trim(rest:sub(1, eq - 1)) - local val = trim(rest:sub(eq + 1)) - if key == "tape_atom words" then - -- key is "tape_atom words"; val is "" - local n = tonumber(val) or 0 - table.insert(out, { - line = line_of(source, i), - name = name, - words = n, - }) - elseif key == "words" then - -- Tolerate "mac_X words=N" if someone writes it that way - local n = tonumber(val) or 0 - table.insert(out, { - line = line_of(source, i), - name = name, - words = n, - }) - end - end - end - end - i = str_end - else - i = open + 1 - end - elseif ident == "pragma" then - -- Directive form: `#pragma mac_X tape_atom words=N` - local rest_start = skip_ws_and_cmt(source, after) - -- Read the rest of the line (no #pragma content spans lines) - local j = rest_start - while j <= len and source:sub(j, j) ~= "\n" do j = j + 1 end - local line_text = trim(source:sub(rest_start, j - 1)) - -- tokenize - local tokens = {} - for tok in line_text:gmatch("%S+") do table.insert(tokens, tok) end - if #tokens >= 4 and tokens[2] == "tape_atom" and tokens[3] == "words=*" or - (#tokens >= 3 and tokens[2] == "words=") then - -- handled below in two forms - end - if #tokens >= 3 and tokens[2] == "tape_atom" and tokens[3]:sub(1, 6) == "words=" then - local name = tokens[1] - local n = tonumber(tokens[3]:sub(7)) or 0 - table.insert(out, { line = line_of(source, i), name = name, words = n }) - elseif #tokens >= 2 and tokens[2]:sub(1, 6) == "words=" then - local name = tokens[1] - local n = tonumber(tokens[2]:sub(7)) or 0 - table.insert(out, { line = line_of(source, i), name = name, words = n }) - end - i = j - else - i = after - end - end - end - return out + local line_of = duffle.LineIndex(source) + local out = {} + local len = #source + local i = 1 + while i <= len do + i = skip_ws_and_cmt(source, i); if i > len then break end + -- Skip preprocessor directives (lines starting with #). + -- read_ident doesn't recognize '#' so without this guard we'd + -- infinite-loop on `#ifdef` / `#pragma region` / etc. + if source:sub(i, i) == "#" then + local j = i + while j <= len and source:sub(j, j) ~= "\n" do j = j + 1 end + i = j + 1 + else + local ident, after = read_ident(source, i) + if not ident then + i = i + 1 + elseif ident == "_Pragma" then + local open = skip_ws_and_cmt(source, after) + if source:sub(open, open) == "(" then + local str, str_end = read_parens(source, open) + -- str is a parenthesized string literal; strip parens, then quotes + str = trim(str) + if str:sub(1, 1) == '"' and str:sub(-1) == '"' then + local inner = str:sub(2, -2) + -- Expect " tape_atom words=" + local space = find_byte(inner, " ", 1) + if space then + local name = inner:sub(1, space - 1) + local rest = inner:sub(space + 1) + local eq = find_byte(rest, "=", 1) + if eq then + local key = trim(rest:sub(1, eq - 1)) + local val = trim(rest:sub(eq + 1)) + if key == "tape_atom words" then + -- key is "tape_atom words"; val is "" + local n = tonumber(val) or 0 + out[#out + 1] = { + line = line_of(i), + name = name, + words = n, + } + elseif key == "words" then + -- Tolerate "mac_X words=N" if someone writes it that way + local n = tonumber(val) or 0 + out[#out + 1] = { + line = line_of(i), + name = name, + words = n, + } + end + end + end + end + i = str_end + else + i = open + 1 + end + elseif ident == "pragma" then + -- Directive form: `#pragma mac_X tape_atom words=N` + local rest_start = skip_ws_and_cmt(source, after) + -- Read the rest of the line (no #pragma content spans lines) + local j = rest_start + while j <= len and source:sub(j, j) ~= "\n" do j = j + 1 end + local line_text = trim(source:sub(rest_start, j - 1)) + -- tokenize (no :gmatch; use hand-rolled split_ws) + local tokens = split_ws(line_text) + if #tokens >= 3 and tokens[2] == "tape_atom" and tokens[3]:sub(1, 6) == "words=" then + local name = tokens[1] + local n = tonumber(tokens[3]:sub(7)) or 0 + out[#out + 1] = { line = line_of(i), name = name, words = n } + elseif #tokens >= 2 and tokens[2]:sub(1, 6) == "words=" then + local name = tokens[1] + local n = tonumber(tokens[2]:sub(7)) or 0 + out[#out + 1] = { line = line_of(i), name = name, words = n } + end + i = j + else + i = after + end + end + end + return out end -- ============================================================ @@ -646,137 +324,139 @@ end -- Map: macro name → pragma key local ATOM_ATTR_MACROS = { - ["atom_resource"] = "resource", - ["atom_region"] = "region", - ["atom_group"] = "group", - ["atom_cadence"] = "cadence", - ["atom_async"] = "async", + ["atom_resource"] = "resource", + ["atom_region"] = "region", + ["atom_group"] = "group", + ["atom_cadence"] = "cadence", + ["atom_async"] = "async", } -- Parse macro form: `atom_(atom_name, value, ...)`. -- Returns (true, entry, str_end) on success, (false) on no match. -- The entry has shape { line, name, attrs = {key = value} }. -local function try_parse_atom_attr_macro(source, i) - local ident, after = read_ident(source, i) - if not ident then return false end - local key = ATOM_ATTR_MACROS[ident] - if not key then return false end - local open = skip_ws_and_cmt(source, after) - if source:sub(open, open) ~= "(" then return false end +local function try_parse_atom_attr_macro(source, i, line_of) + local ident, after = read_ident(source, i) + if not ident then return false end + local key = ATOM_ATTR_MACROS[ident] + if not key then return false end + local open = skip_ws_and_cmt(source, after) + if source:sub(open, open) ~= "(" then return false end - -- Read parenthesized arg list. - local body, body_end = read_parens(source, open) - -- First arg = atom_name - local first, after_name = read_ident(body, 1) - if not first then return false end + -- Read parenthesized arg list. + local body, body_end = read_parens(source, open) + -- First arg = atom_name + local first, after_name = read_ident(body, 1) + if not first then return false end - -- Skip whitespace, expect ",", skip whitespace to reach value. - local j = after_name - while j <= #body and is_space(body:sub(j, j)) do j = j + 1 end - if body:sub(j, j) ~= "," then return false end - j = j + 1 - while j <= #body and is_space(body:sub(j, j)) do j = j + 1 end + -- Skip whitespace, expect ",", skip whitespace to reach value. + local j = after_name + while j <= #body and is_space(body:sub(j, j)) do j = j + 1 end + if body:sub(j, j) ~= "," then return false end + j = j + 1 + while j <= #body and is_space(body:sub(j, j)) do j = j + 1 end - -- Value parser. Accepts either: - -- - string literal: "..." (preserves inner text verbatim) - -- - bare identifier or macro name (resolved via MACRO_EXPANSION) - local value - if body:sub(j, j) == '"' then - -- find matching closing quote (handle \" escapes) - local k = j + 1 - while k <= #body do - local c = body:sub(k, k) - if c == "\\" then - k = k + 2 - elseif c == '"' then - break - else - k = k + 1 - end - end - if body:sub(k, k) ~= '"' then return false end - value = body:sub(j + 1, k - 1) - else - local id2, after_id = read_ident(body, j) - if not id2 then return false end - value = id2 - if MACRO_EXPANSION[value] then value = MACRO_EXPANSION[value] end - end + -- Value parser. Accepts either: + -- - string literal: "..." (preserves inner text verbatim) + -- - bare identifier or macro name (resolved via MACRO_EXPANSION) + local value + if body:sub(j, j) == '"' then + -- find matching closing quote (handle \" escapes) + local k = j + 1 + while k <= #body do + local c = body:sub(k, k) + if c == "\\" then + k = k + 2 + elseif c == '"' then + break + else + k = k + 1 + end + end + if body:sub(k, k) ~= '"' then return false end + value = body:sub(j + 1, k - 1) + else + local id2, after_id = read_ident(body, j) + if not id2 then return false end + value = id2 + if MACRO_EXPANSION[value] then value = MACRO_EXPANSION[value] end + end - return true, { - line = line_of(source, i), - name = first, - attrs = { [key] = value }, - }, body_end + return true, { + line = line_of(i), + name = first, + attrs = { [key] = value }, + }, body_end end local function find_atom_pragmas(source) - local out = {} - local len = #source - local i = 1 - while i <= len do - i = skip_ws_and_cmt(source, i); if i > len then break end - if source:sub(i, i) == "#" then - local j = i - while j <= len and source:sub(j, j) ~= "\n" do j = j + 1 end - i = j + 1 - else - local got, entry, next_i = try_parse_atom_attr_macro(source, i) - if got then - table.insert(out, entry) - i = next_i - else - local ident, after = read_ident(source, i) - if not ident then - i = i + 1 - elseif ident == "_Pragma" then - local open = skip_ws_and_cmt(source, after) - if source:sub(open, open) == "(" then - local str, str_end = read_parens(source, open) - str = trim(str) - if str:sub(1, 1) == '"' and str:sub(-1) == '"' then - local inner = str:sub(2, -2) - -- Expect: "atom = [= ...]" - local sp1 = find_byte(inner, " ", 1) - if sp1 and trim(inner:sub(1, sp1 - 1)) == "atom" then - local rest = trim(inner:sub(sp1 + 1)) - local sp2 = find_byte(rest, " ", 1) - if sp2 then - local name = trim(rest:sub(1, sp2 - 1)) - local attrs_str = trim(rest:sub(sp2 + 1)) - local attrs = {} - local got_any = false - for pair in attrs_str:gmatch("%S+") do - local eq = find_byte(pair, "=", 1) - if eq then - local k = trim(pair:sub(1, eq - 1)) - local v = trim(pair:sub(eq + 1)) - if MACRO_EXPANSION[v] then v = MACRO_EXPANSION[v] end - attrs[k] = v - got_any = true - end - end - if got_any then - table.insert(out, { - line = line_of(source, i), - name = name, - attrs = attrs, - }) - end - end - end - end - i = str_end - else - i = open + 1 - end - else - i = after - end - end - end - end - return out + local line_of = duffle.LineIndex(source) + local out = {} + local len = #source + local i = 1 + while i <= len do + i = skip_ws_and_cmt(source, i); if i > len then break end + if source:sub(i, i) == "#" then + local j = i + while j <= len and source:sub(j, j) ~= "\n" do j = j + 1 end + i = j + 1 + else + local got, entry, next_i = try_parse_atom_attr_macro(source, i, line_of) + if got then + out[#out + 1] = entry + i = next_i + else + local ident, after = read_ident(source, i) + if not ident then + i = i + 1 + elseif ident == "_Pragma" then + local open = skip_ws_and_cmt(source, after) + if source:sub(open, open) == "(" then + local str, str_end = read_parens(source, open) + str = trim(str) + if str:sub(1, 1) == '"' and str:sub(-1) == '"' then + local inner = str:sub(2, -2) + -- Expect: "atom = [= ...]" + local sp1 = find_byte(inner, " ", 1) + if sp1 and trim(inner:sub(1, sp1 - 1)) == "atom" then + local rest = trim(inner:sub(sp1 + 1)) + local sp2 = find_byte(rest, " ", 1) + if sp2 then + local name = trim(rest:sub(1, sp2 - 1)) + local attrs_str = trim(rest:sub(sp2 + 1)) + local attrs = {} + local got_any = false + -- tokenize (no :gmatch; use hand-rolled split_ws) + for _, pair in ipairs(split_ws(attrs_str)) do + local eq = find_byte(pair, "=", 1) + if eq then + local k = trim(pair:sub(1, eq - 1)) + local v = trim(pair:sub(eq + 1)) + if MACRO_EXPANSION[v] then v = MACRO_EXPANSION[v] end + attrs[k] = v + got_any = true + end + end + if got_any then + out[#out + 1] = { + line = line_of(i), + name = name, + attrs = attrs, + } + end + end + end + end + i = str_end + else + i = open + 1 + end + else + i = after + end + end + end + end + return out end @@ -788,77 +468,78 @@ end -- ============================================================ local function find_binds_structs(source) - local out = {} - local len = #source - local i = 1 - while i <= len do - i = skip_ws_and_cmt(source, i); if i > len then break end - -- Skip preprocessor directives (lines starting with #). - if source:sub(i, i) == "#" then - local j = i - while j <= len and source:sub(j, j) ~= "\n" do j = j + 1 end - i = j + 1 - else - local ident, after = read_ident(source, i) - if not ident then - i = i + 1 - elseif ident == "typedef" then - local j = skip_ws_and_cmt(source, after) - local id2, after2 = read_ident(source, j) - if id2 ~= "Struct_" then - i = after2 or (j + 1) - elseif id2 == "Struct_" then - local open = skip_ws_and_cmt(source, after2) - if source:sub(open, open) == "(" then - local inner, after_paren = read_parens(source, open) - local name = trim(inner) - local brace = scan_to_char(source, "{", after_paren) - if brace then - local body, after_brace = read_braces(source, brace) - local fields = {} - local byte_off = 0 - local k = 1 - while k <= #body do - k = skip_ws_and_cmt(body, k); if k > #body then break end - local tid, tafter = read_ident(body, k) - if not tid then - k = k + 1 - elseif tid == "U4" then - local fid, fafter = read_ident(body, skip_ws_and_cmt(body, tafter)) - if fid then - table.insert(fields, { name = fid, offset = byte_off }) - byte_off = byte_off + 4 - end - k = fafter or tafter + 1 - else - k = tafter + 1 - end - end - -- Only emit Binds_* structs (skip FMipsAtom512, MipsAtomBuilder, etc.) - if name:sub(1, 6) == "Binds_" then - table.insert(out, { - line = line_of(source, i), - name = name, - fields = fields, - bytes = byte_off, - }) - end - i = after_brace - else - i = open + 1 - end - else - i = open + 1 - end - else - i = after2 or (j + 1) - end - else - i = after - end - end - end - return out + local line_of = duffle.LineIndex(source) + local out = {} + local len = #source + local i = 1 + while i <= len do + i = skip_ws_and_cmt(source, i); if i > len then break end + -- Skip preprocessor directives (lines starting with #). + if source:sub(i, i) == "#" then + local j = i + while j <= len and source:sub(j, j) ~= "\n" do j = j + 1 end + i = j + 1 + else + local ident, after = read_ident(source, i) + if not ident then + i = i + 1 + elseif ident == "typedef" then + local j = skip_ws_and_cmt(source, after) + local id2, after2 = read_ident(source, j) + if id2 ~= "Struct_" then + i = after2 or (j + 1) + elseif id2 == "Struct_" then + local open = skip_ws_and_cmt(source, after2) + if source:sub(open, open) == "(" then + local inner, after_paren = read_parens(source, open) + local name = trim(inner) + local brace = scan_to_char(source, "{", after_paren) + if brace then + local body, after_brace = read_braces(source, brace) + local fields = {} + local byte_off = 0 + local k = 1 + while k <= #body do + k = skip_ws_and_cmt(body, k); if k > #body then break end + local tid, tafter = read_ident(body, k) + if not tid then + k = k + 1 + elseif tid == "U4" then + local fid, fafter = read_ident(body, skip_ws_and_cmt(body, tafter)) + if fid then + fields[#fields + 1] = { name = fid, offset = byte_off } + byte_off = byte_off + 4 + end + k = fafter or tafter + 1 + else + k = tafter + 1 + end + end + -- Only emit Binds_* structs (skip FMipsAtom512, MipsAtomBuilder, etc.) + if name:sub(1, 6) == "Binds_" then + out[#out + 1] = { + line = line_of(i), + name = name, + fields = fields, + bytes = byte_off, + } + end + i = after_brace + else + i = open + 1 + end + else + i = open + 1 + end + else + i = after2 or (j + 1) + end + else + i = after + end + end + end + return out end -- ============================================================ @@ -867,41 +548,150 @@ end -- ============================================================ local function find_atom_names(source) - local out = {} - local len = #source - local i = 1 - while i <= len do - i = skip_ws_and_cmt(source, i); if i > len then break end - local ident, after = read_ident(source, i) - if not ident then - i = i + 1 - elseif ident == "MipsAtom_" then - local open = skip_ws_and_cmt(source, after) - if source:sub(open, open) == "(" then - local inner, after_paren = read_parens(source, open) - local a = 1 - while a <= #inner and is_space(inner:sub(a, a)) do a = a + 1 end - local b = a - while b <= #inner and is_alnum(inner:sub(b, b)) do b = b + 1 end - local name = inner:sub(a, b - 1) - if name ~= "" then - table.insert(out, { line = line_of(source, i), name = name }) - end - local brace = scan_to_char(source, "{", after_paren) - if brace then - local _, after_brace = read_braces(source, brace) - i = after_brace - else - i = open + 1 - end - else - i = open + 1 - end - else - i = after - end - end - return out + local line_of = duffle.LineIndex(source) + local out = {} + local len = #source + local i = 1 + while i <= len do + i = skip_ws_and_cmt(source, i); if i > len then break end + local ident, after = read_ident(source, i) + if not ident then + i = i + 1 + elseif ident == "MipsAtom_" then + local open = skip_ws_and_cmt(source, after) + if source:sub(open, open) == "(" then + local inner, after_paren = read_parens(source, open) + local a = 1 + while a <= #inner and is_space(inner:sub(a, a)) do a = a + 1 end + local b = a + while b <= #inner and is_alnum(inner:sub(b, b)) do b = b + 1 end + local name = inner:sub(a, b - 1) + if name ~= "" then + out[#out + 1] = { line = line_of(i), name = name } + end + local brace = scan_to_char(source, "{", after_paren) + if brace then + local _, after_brace = read_braces(source, brace) + i = after_brace + else + i = open + 1 + end + else + i = open + 1 + end + else + i = after + end + end + return out +end + +local function find_atom_annotations(source) + local line_of = duffle.LineIndex(source) + local annots = {} + local len = #source + local i = 1 + while i <= len do + i = skip_ws_and_cmt(source, i); if i > len then break end + -- Match the leading identifier of a TAPE_ATOM_* macro. + local ident, after = read_ident(source, i) + if not ident then + i = i + 1 + elseif TAPE_ATOM_MACROS[ident] then + local open = skip_ws_and_cmt(source, after) + if source:sub(open, open) == "(" then + local inner, after_paren = read_parens(source, open) + local args = parse_atom_annot_args(inner) + local macro_def = TAPE_ATOM_MACROS[ident] + + if #args < 1 then + annots[#annots + 1] = { + line = line_of(i), + macro = ident, + kind = macro_def.kind, + error = "missing atom name (first arg)", + } + else + local name = args[1].value + -- atom_bind: (name, Binds_Struct, writes) + -- atom_annot: (name, phase, reads, writes) + -- atom_setup / atom_commit: (name, reads) + -- atom_init / atom_terminate: (name) + local entry = { + line = line_of(i), + macro = ident, + name = name, + kind = macro_def.kind, + binds = nil, + phase = nil, + reads = {}, + writes = {}, + errors = {}, + } + + -- Is this arg a register-list call (any of the recognized forms)? + local function is_regs(a) + return a and (a.kind == "atom_reads" or a.kind == "atom_writes" or a.kind == "regs") + end + + if macro_def.binds then + -- atom_bind(name, Binds_Struct, writes) + if #args >= 2 and args[2].kind == "ident" then + entry.binds = args[2].value + end + if #args >= 3 and is_regs(args[3]) then + -- A bind writes the wave-context, so atom_writes(...) is the + -- canonical form, but legacy regs(...) is also accepted. + entry.writes = args[3].value + end + elseif ident == "atom_init" or ident == "atom_terminate" then + -- (name) only, no reads/writes to extract + elseif ident == "atom_setup" then + -- atom_setup(name, reads) + if #args >= 2 and is_regs(args[2]) then + entry.reads = args[2].value + end + elseif ident == "atom_commit" then + -- atom_commit(name, reads) + if #args >= 2 and is_regs(args[2]) then + entry.reads = args[2].value + end + elseif ident == "atom_annot" then + -- atom_annot(name, phase, reads, writes) + if #args >= 2 and args[2].kind == "ident" then + entry.phase = MACRO_EXPANSION[args[2].value] or args[2].value + end + -- reads slot: atom_reads(...) is the canonical form; + -- atom_writes(...) in the reads slot is a likely bug. + if #args >= 3 and is_regs(args[3]) then + if args[3].kind == "atom_writes" then + entry.errors[#entry.errors + 1] = + "reads slot has atom_writes — swap order?" + end + entry.reads = args[3].value + end + -- writes slot: atom_writes(...) is canonical; + -- atom_reads(...) here is a likely bug. + if #args >= 4 and is_regs(args[4]) then + if args[4].kind == "atom_reads" then + entry.errors[#entry.errors + 1] = + "writes slot has atom_reads — swap order?" + end + entry.writes = args[4].value + end + end + + annots[#annots + 1] = entry + end + i = after_paren + else + i = open + 1 + end + else + i = after + end + end + return annots end -- ============================================================ @@ -909,217 +699,217 @@ end -- ============================================================ local function validate(source_path, word_counts) - local source = read_file(source_path) + local source = read_file(source_path) - local annots = find_atom_annotations(source) - local macros = find_macro_word_annotations(source) - local pragmas = find_atom_pragmas(source) - local binds = find_binds_structs(source) - local atoms = find_atom_names(source) + local annots = find_atom_annotations(source) + local macros = find_macro_word_annotations(source) + local pragmas = find_atom_pragmas(source) + local binds = find_binds_structs(source) + local atoms = find_atom_names(source) - -- Index for O(1) lookup - local atom_index = {} - for _, a in ipairs(atoms) do atom_index[a.name] = a end + -- Index for O(1) lookup + local atom_index = {} + for _, a in ipairs(atoms) do atom_index[a.name] = a end - local binds_index = {} - for _, b in ipairs(binds) do binds_index[b.name] = b end + local binds_index = {} + for _, b in ipairs(binds) do binds_index[b.name] = b end - local errors = {} - local warnings = {} - local info = {} + local errors = {} + local warnings = {} + local info = {} - -- 1. Every annotated atom must exist as a real MipsAtom_ declaration. - for _, a in ipairs(annots) do - if a.error then - table.insert(errors, {line = a.line, msg = a.error}) - elseif not atom_index[a.name] then - table.insert(errors, { - line = a.line, - msg = string.format("annotation for '%s' has no matching MipsAtom_(%s) { ... }", a.name, a.name) - }) - end - -- Per-entry parser errors (e.g. reads/writes slot mix-ups) - if a.errors then - for _, msg in ipairs(a.errors) do - table.insert(errors, {line = a.line, msg = string.format("'%s': %s", a.name, msg)}) - end - end - end + -- 1. Every annotated atom must exist as a real MipsAtom_ declaration. + for _, a in ipairs(annots) do + if a.error then + errors[#errors + 1] = {line = a.line, msg = a.error} + elseif not atom_index[a.name] then + errors[#errors + 1] = { + line = a.line, + msg = string.format("annotation for '%s' has no matching MipsAtom_(%s) { ... }", a.name, a.name) + } + end + -- Per-entry parser errors (e.g. reads/writes slot mix-ups) + if a.errors then + for _, msg in ipairs(a.errors) do + errors[#errors + 1] = {line = a.line, msg = string.format("'%s': %s", a.name, msg)} + end + end + end - -- 2. Every atom must have exactly one annotation (no orphans, no duplicates). - local count_per_atom = {} - for _, a in ipairs(annots) do - if a.name and not a.error then - count_per_atom[a.name] = (count_per_atom[a.name] or 0) + 1 - end - end - for _, atom in ipairs(atoms) do - local n = count_per_atom[atom.name] or 0 - if n == 0 then - table.insert(warnings, { - line = atom.line, - msg = string.format("MipsAtom_(%s) has no TAPE_ATOM_* annotation", atom.name) - }) - elseif n > 1 then - table.insert(errors, { - line = atom.line, - msg = string.format("MipsAtom_(%s) has %d annotations (expected 1)", atom.name, n) - }) - end - end + -- 2. Every atom must have exactly one annotation (no orphans, no duplicates). + local count_per_atom = {} + for _, a in ipairs(annots) do + if a.name and not a.error then + count_per_atom[a.name] = (count_per_atom[a.name] or 0) + 1 + end + end + for _, atom in ipairs(atoms) do + local n = count_per_atom[atom.name] or 0 + if n == 0 then + warnings[#warnings + 1] = { + line = atom.line, + msg = string.format("MipsAtom_(%s) has no TAPE_ATOM_* annotation", atom.name) + } + elseif n > 1 then + errors[#errors + 1] = { + line = atom.line, + msg = string.format("MipsAtom_(%s) has %d annotations (expected 1)", atom.name, n) + } + end + end - -- 3. Phase validity. - for _, a in ipairs(annots) do - if a.name and not a.error and a.phase and not valid_phase(a.phase) then - table.insert(errors, { - line = a.line, - msg = string.format("'%s' has unknown phase '%s' (expected one of init/bind/setup/work/commit/terminate)", a.name, a.phase) - }) - end - end + -- 3. Phase validity. + for _, a in ipairs(annots) do + if a.name and not a.error and a.phase and not valid_phase(a.phase) then + errors[#errors + 1] = { + line = a.line, + msg = string.format("'%s' has unknown phase '%s' (expected one of init/bind/setup/work/commit/terminate)", a.name, a.phase) + } + end + end - -- 4. BIND atoms must reference a real Binds_* struct. - for _, a in ipairs(annots) do - if a.binds then - if not binds_index[a.binds] then - table.insert(errors, { - line = a.line, - msg = string.format("'%s' binds '%s' but no Struct_(%s) { ... } declaration found", a.name, a.binds, a.binds) - }) - end - end - end + -- 4. BIND atoms must reference a real Binds_* struct. + for _, a in ipairs(annots) do + if a.binds then + if not binds_index[a.binds] then + errors[#errors + 1] = { + line = a.line, + msg = string.format("'%s' binds '%s' but no Struct_(%s) { ... } declaration found", a.name, a.binds, a.binds) + } + end + end + end - -- 5. BIND writes must be wave-context registers that match Binds_ fields. - for _, a in ipairs(annots) do - if a.binds and binds_index[a.binds] then - local bs = binds_index[a.binds] - local field_names = {} - for _, f in ipairs(bs.fields) do field_names[f.name] = true end + -- 5. BIND writes must be wave-context registers that match Binds_ fields. + for _, a in ipairs(annots) do + if a.binds and binds_index[a.binds] then + local bs = binds_index[a.binds] + local field_names = {} + for _, f in ipairs(bs.fields) do field_names[f.name] = true end - -- Binds_ field names should match wave-context registers. - -- Convention: field name == register name minus the "R_" prefix and - -- "Cursor"/"Base" suffix mapped to canonical names. - -- We accept either direct match (rare) or a simple "R_" prefix. - for _, f in ipairs(bs.fields) do - local candidate = "R_" .. f.name - if not is_wave_context_reg(candidate) then - table.insert(warnings, { - line = bs.line, - msg = string.format("%s field '%s' doesn't match a known wave-context register (candidate '%s')", a.binds, f.name, candidate) - }) - end - end + -- Binds_ field names should match wave-context registers. + -- Convention: field name == register name minus the "R_" prefix and + -- "Cursor"/"Base" suffix mapped to canonical names. + -- We accept either direct match (rare) or a simple "R_" prefix. + for _, f in ipairs(bs.fields) do + local candidate = "R_" .. f.name + if not is_wave_context_reg(candidate) then + warnings[#warnings + 1] = { + line = bs.line, + msg = string.format("%s field '%s' doesn't match a known wave-context register (candidate '%s')", a.binds, f.name, candidate) + } + end + end - -- Bind writes must all be wave-context registers. - for _, w in ipairs(a.writes) do - if not is_wave_context_reg(w) then - table.insert(warnings, { - line = a.line, - msg = string.format("%s writes '%s' which is not a known wave-context register", a.name, w) - }) - end - end - end - end + -- Bind writes must all be wave-context registers. + for _, w in ipairs(a.writes) do + if not is_wave_context_reg(w) then + warnings[#warnings + 1] = { + line = a.line, + msg = string.format("%s writes '%s' which is not a known wave-context register", a.name, w) + } + end + end + end + end - -- 6. WORK reads should be a subset of BIND writes (the wave contract). - -- We can't see tape emission sites here, but we can warn when a WORK - -- atom reads a register that no BIND atom writes. - for _, a in ipairs(annots) do - if a.kind == "work" then - for _, r in ipairs(a.reads) do - if not is_wave_context_reg(r) and r ~= "R_TapePtr" then - table.insert(warnings, { - line = a.line, - msg = string.format("work atom '%s' reads '%s' which is not a known wave-context register", a.name, r) - }) - end - end - end - end + -- 6. WORK reads should be a subset of BIND writes (the wave contract). + -- We can't see tape emission sites here, but we can warn when a WORK + -- atom reads a register that no BIND atom writes. + for _, a in ipairs(annots) do + if a.kind == "work" then + for _, r in ipairs(a.reads) do + if not is_wave_context_reg(r) and r ~= "R_TapePtr" then + warnings[#warnings + 1] = { + line = a.line, + msg = string.format("work atom '%s' reads '%s' which is not a known wave-context register", a.name, r) + } + end + end + end + end - -- 7. TAPE_WORDS(mac_X, N) ↔ WORD_COUNT(mac_X, N) drift. - for _, m in ipairs(macros) do - local declared = word_counts[m.name] - if not declared then - table.insert(errors, { - line = m.line, - msg = string.format("TAPE_WORDS(%s, %d) but '%s' is not in metadata.h", m.name, m.words, m.name) - }) - elseif declared ~= m.words then - table.insert(errors, { - line = m.line, - msg = string.format("DRIFT: TAPE_WORDS(%s, %d) but metadata.h declares WORD_COUNT(%s, %d)", m.name, m.words, m.name, declared) - }) - else - table.insert(info, { - line = m.line, - msg = string.format("OK: %s = %d words", m.name, m.words) - }) - end - end + -- 7. TAPE_WORDS(mac_X, N) ↔ WORD_COUNT(mac_X, N) drift. + for _, m in ipairs(macros) do + local declared = word_counts[m.name] + if not declared then + errors[#errors + 1] = { + line = m.line, + msg = string.format("TAPE_WORDS(%s, %d) but '%s' is not in metadata.h", m.name, m.words, m.name) + } + elseif declared ~= m.words then + errors[#errors + 1] = { + line = m.line, + msg = string.format("DRIFT: TAPE_WORDS(%s, %d) but metadata.h declares WORD_COUNT(%s, %d)", m.name, m.words, m.name, declared) + } + else + info[#info + 1] = { + line = m.line, + msg = string.format("OK: %s = %d words", m.name, m.words) + } + end + end - -- 8. atom_<...> _Pragma validation: resource/region/group/cadence/async - for _, p in ipairs(pragmas) do - -- Validate the atom name is real - if not atom_index[p.name] then - table.insert(errors, { - line = p.line, - msg = string.format("pragma references unknown atom '%s'", p.name), - }) - end + -- 8. atom_<...> _Pragma validation: resource/region/group/cadence/async + for _, p in ipairs(pragmas) do + -- Validate the atom name is real + if not atom_index[p.name] then + errors[#errors + 1] = { + line = p.line, + msg = string.format("pragma references unknown atom '%s'", p.name), + } + end - -- Validate each key - for k, v in pairs(p.attrs) do - local spec = ATOM_PRAGMA_KINDS[k] - if not spec then - table.insert(errors, { - line = p.line, - msg = string.format("'%s' has unknown pragma key '%s' (allowed: resource/region/group/cadence/async)", p.name, k), - }) - elseif spec.allowed and not spec.allowed[v] then - -- Build a human-readable allowed-set - local allowed = {} - for kk in pairs(spec.allowed) do table.insert(allowed, kk) end - table.sort(allowed) - local allowed_str = table.concat(allowed, ", ") - table.insert(errors, { - line = p.line, - msg = string.format("'%s' pragma %s=%s but '%s' is not allowed (allowed: %s)", p.name, k, v, v, allowed_str), - }) - end - end - end + -- Validate each key + for k, v in pairs(p.attrs) do + local spec = ATOM_PRAGMA_KINDS[k] + if not spec then + errors[#errors + 1] = { + line = p.line, + msg = string.format("'%s' has unknown pragma key '%s' (allowed: resource/region/group/cadence/async)", p.name, k), + } + elseif spec.allowed and not spec.allowed[v] then + -- Build a human-readable allowed-set + local allowed = {} + for kk in pairs(spec.allowed) do allowed[#allowed + 1] = kk end + table.sort(allowed) + local allowed_str = table.concat(allowed, ", ") + errors[#errors + 1] = { + line = p.line, + msg = string.format("'%s' pragma %s=%s but '%s' is not allowed (allowed: %s)", p.name, k, v, v, allowed_str), + } + end + end + end - -- 9. cad_async requires CADENCE_ONDEMAND (or no cadence — default-frame atoms can still async). - -- CADENCE_ONDEMAND requires async=true (otherwise the trigger is undefined). - for _, p in ipairs(pragmas) do - if p.attrs.cadence == "ondemand" and p.attrs.async ~= "true" then - table.insert(errors, { - line = p.line, - msg = string.format("'%s' is CADENCE_ONDEMAND but does not declare atom_async(true)", p.name), - }) - end - end + -- 9. cad_async requires CADENCE_ONDEMAND (or no cadence — default-frame atoms can still async). + -- CADENCE_ONDEMAND requires async=true (otherwise the trigger is undefined). + for _, p in ipairs(pragmas) do + if p.attrs.cadence == "ondemand" and p.attrs.async ~= "true" then + errors[#errors + 1] = { + line = p.line, + msg = string.format("'%s' is CADENCE_ONDEMAND but does not declare atom_async(true)", p.name), + } + end + end - -- 10. Information summary. - table.insert(info, { - line = 0, - msg = string.format("scanned: %d atom(s), %d annotation(s), %d pragma(s), %d macro-word-decl(s), %d binds struct(s)", - #atoms, #annots, #pragmas, #macros, #binds) - }) + -- 10. Information summary. + info[#info + 1] = { + line = 0, + msg = string.format("scanned: %d atom(s), %d annotation(s), %d pragma(s), %d macro-word-decl(s), %d binds struct(s)", + #atoms, #annots, #pragmas, #macros, #binds) + } - return { - atoms = atoms, - annots = annots, - macros = macros, - pragmas = pragmas, - binds = binds, - errors = errors, - warnings = warnings, - info = info, - } + return { + atoms = atoms, + annots = annots, + macros = macros, + pragmas = pragmas, + binds = binds, + errors = errors, + warnings = warnings, + info = info, + } end -- ============================================================ @@ -1127,122 +917,122 @@ end -- ============================================================ local function render_source_report(source_path, result) - local lines = {} - local function add(s) table.insert(lines, s) end + local lines = {} + local function add(s) lines[#lines + 1] = s end - add("========================================================") - add("ANNOTATION PASS — " .. source_path) - add("========================================================") - add("") - add(string.format("Atoms: %d Annotations: %d Pragmas: %d Binds structs: %d Macro decls: %d", - #result.atoms, #result.annots, #result.pragmas and #result.pragmas or 0, - #result.binds, #result.macros)) - add("") + add("========================================================") + add("ANNOTATION PASS — " .. source_path) + add("========================================================") + add("") + add(string.format("Atoms: %d Annotations: %d Pragmas: %d Binds structs: %d Macro decls: %d", + #result.atoms, #result.annots, #result.pragmas and #result.pragmas or 0, + #result.binds, #result.macros)) + add("") - add("── Atoms ────────────────────────────────────────────────") - for _, a in ipairs(result.atoms) do - add(string.format(" MipsAtom_(%s) line %d", a.name, a.line)) - end - add("") + add("── Atoms ────────────────────────────────────────────────") + for _, a in ipairs(result.atoms) do + add(string.format(" MipsAtom_(%s) line %d", a.name, a.line)) + end + add("") - add("── Annotations ──────────────────────────────────────────") - for _, a in ipairs(result.annots) do - if a.error then - add(string.format(" ✗ line %d %s [ERROR: %s]", a.line, a.macro or "?", a.error)) - else - local line = string.format(" %s line %d %s phase=%s", - a.kind == "work" and "●" or (a.kind == "bind" and "◆" or "○"), - a.line, a.name, a.phase or a.kind) - if a.binds then line = line .. " binds=" .. a.binds end - if #a.reads > 0 then line = line .. " reads={" .. table.concat(a.reads, ",") .. "}" end - if #a.writes > 0 then line = line .. " writes={" .. table.concat(a.writes, ",") .. "}" end - add(line) - end - end - add("") + add("── Annotations ──────────────────────────────────────────") + for _, a in ipairs(result.annots) do + if a.error then + add(string.format(" ✗ line %d %s [ERROR: %s]", a.line, a.macro or "?", a.error)) + else + local line = string.format(" %s line %d %s phase=%s", + a.kind == "work" and "●" or (a.kind == "bind" and "◆" or "○"), + a.line, a.name, a.phase or a.kind) + if a.binds then line = line .. " binds=" .. a.binds end + if #a.reads > 0 then line = line .. " reads={" .. table.concat(a.reads, ",") .. "}" end + if #a.writes > 0 then line = line .. " writes={" .. table.concat(a.writes, ",") .. "}" end + add(line) + end + end + add("") - add("── Binds_* structs ──────────────────────────────────────") - for _, b in ipairs(result.binds) do - add(string.format(" %s line %d %d bytes", b.name, b.line, b.bytes)) - for _, f in ipairs(b.fields) do - add(string.format(" +%2d: %s", f.offset, f.name)) - end - end - add("") + add("── Binds_* structs ──────────────────────────────────────") + for _, b in ipairs(result.binds) do + add(string.format(" %s line %d %d bytes", b.name, b.line, b.bytes)) + for _, f in ipairs(b.fields) do + add(string.format(" +%2d: %s", f.offset, f.name)) + end + end + add("") - add("── Macro word-count declarations ─────────────────────────") - for _, m in ipairs(result.macros) do - add(string.format(" %s line %d words=%d", m.name, m.line, m.words)) - end - add("") + add("── Macro word-count declarations ─────────────────────────") + for _, m in ipairs(result.macros) do + add(string.format(" %s line %d words=%d", m.name, m.line, m.words)) + end + add("") - add("── Atom pragmas (resource / region / group / cadence / async) ─") - if not result.pragmas or #result.pragmas == 0 then add(" (none)") end - for _, p in ipairs(result.pragmas or {}) do - local kvs = {} - for k, v in pairs(p.attrs) do table.insert(kvs, k .. "=" .. v) end - table.sort(kvs) - add(string.format(" ◇ line %d %s {%s}", p.line, p.name, table.concat(kvs, ", "))) - end - add("") + add("── Atom pragmas (resource / region / group / cadence / async) ─") + if not result.pragmas or #result.pragmas == 0 then add(" (none)") end + for _, p in ipairs(result.pragmas or {}) do + local kvs = {} + for k, v in pairs(p.attrs) do kvs[#kvs + 1] = k .. "=" .. v end + table.sort(kvs) + add(string.format(" ◇ line %d %s {%s}", p.line, p.name, table.concat(kvs, ", "))) + end + add("") - add("── Errors ──────────────────────────────────────────────") - if #result.errors == 0 then add(" (none)") end - for _, e in ipairs(result.errors) do - add(string.format(" ✗ line %d %s", e.line, e.msg)) - end - add("") + add("── Errors ──────────────────────────────────────────────") + if #result.errors == 0 then add(" (none)") end + for _, e in ipairs(result.errors) do + add(string.format(" ✗ line %d %s", e.line, e.msg)) + end + add("") - add("── Warnings ────────────────────────────────────────────") - if #result.warnings == 0 then add(" (none)") end - for _, w in ipairs(result.warnings) do - add(string.format(" ⚠ line %d %s", w.line, w.msg)) - end - add("") + add("── Warnings ────────────────────────────────────────────") + if #result.warnings == 0 then add(" (none)") end + for _, w in ipairs(result.warnings) do + add(string.format(" ⚠ line %d %s", w.line, w.msg)) + end + add("") - return table.concat(lines, "\n") .. "\n" + return table.concat(lines, "\n") .. "\n" end local function render_project_report(all_results) - local lines = {} - local function add(s) table.insert(lines, s) end + local lines = {} + local function add(s) lines[#lines + 1] = s end - local total_atoms, total_annots, total_macros, total_binds = 0, 0, 0, 0 - local total_errors, total_warnings = 0, 0 + local total_atoms, total_annots, total_macros, total_binds = 0, 0, 0, 0 + local total_errors, total_warnings = 0, 0 - for _, r in ipairs(all_results) do - total_atoms = total_atoms + #r.atoms - total_annots = total_annots + #r.annots - total_macros = total_macros + #r.macros - total_binds = total_binds + #r.binds - total_errors = total_errors + #r.errors - total_warnings = total_warnings + #r.warnings - end + for _, r in ipairs(all_results) do + total_atoms = total_atoms + #r.atoms + total_annots = total_annots + #r.annots + total_macros = total_macros + #r.macros + total_binds = total_binds + #r.binds + total_errors = total_errors + #r.errors + total_warnings = total_warnings + #r.warnings + end - add("========================================================") - add("ANNOTATION VALIDATION — project summary") - add("========================================================") - add("") - add(string.format("Atoms: %d", total_atoms)) - add(string.format("Annotations: %d", total_annots)) - add(string.format("Macros: %d", total_macros)) - add(string.format("Binds: %d", total_binds)) - add("") - add(string.format("Errors: %d", total_errors)) - add(string.format("Warnings: %d", total_warnings)) - add("") + add("========================================================") + add("ANNOTATION VALIDATION — project summary") + add("========================================================") + add("") + add(string.format("Atoms: %d", total_atoms)) + add(string.format("Annotations: %d", total_annots)) + add(string.format("Macros: %d", total_macros)) + add(string.format("Binds: %d", total_binds)) + add("") + add(string.format("Errors: %d", total_errors)) + add(string.format("Warnings: %d", total_warnings)) + add("") - if total_errors > 0 then - add("Per-source error counts:") - for _, r in ipairs(all_results) do - if #r.errors > 0 then - add(string.format(" %s : %d error(s)", r.source, #r.errors)) - end - end - add("") - end + if total_errors > 0 then + add("Per-source error counts:") + for _, r in ipairs(all_results) do + if #r.errors > 0 then + add(string.format(" %s : %d error(s)", r.source, #r.errors)) + end + end + add("") + end - return table.concat(lines, "\n") .. "\n" + return table.concat(lines, "\n") .. "\n" end -- ============================================================ @@ -1250,85 +1040,85 @@ end -- ============================================================ local function main(args) - if #args < 2 then - print("Usage: lua tape_atom_annotation_pass.lua [source2 ...]") - os.exit(1) - end - local word_counts = load_word_counts(args[1]) - local all_results = {} + if #args < 2 then + print("Usage: luajit tape_atom_annotation_pass.lua [source2 ...]") + os.exit(1) + end + local word_counts = load_word_counts(args[1]) + local all_results = {} - -- Collect every /gen directory we touch, so we can prune stale - -- empty reports left over by previous runs. A file that USED to have atoms - -- but doesn't any more (refactor / template removal) shouldn't keep its - -- report polluting the source tree. - local pruned_dirs = {} + -- Collect every /gen directory we touch, so we can prune stale + -- empty reports left over by previous runs. A file that USED to have atoms + -- but doesn't any more (refactor / template removal) shouldn't keep its + -- report polluting the source tree. + local pruned_dirs = {} - for i = 2, #args do - local source_path = args[i] - local basename = basename_no_ext(source_path) - local out_dir = dirname(source_path) .. "/gen" - local out_txt = out_dir .. "/" .. basename .. ".annotations.txt" - local out_err = out_dir .. "/" .. basename .. ".errors.h" + for i = 2, #args do + local source_path = args[i] + local basename = basename_no_ext(source_path) + local out_dir = dirname(source_path) .. "/gen" + local out_txt = out_dir .. "/" .. basename .. ".annotations.txt" + local out_err = out_dir .. "/" .. basename .. ".errors.h" - local result = validate(source_path, word_counts) - result.source = source_path + local result = validate(source_path, word_counts) + result.source = source_path - -- Decide whether this source contributes to the build at all. A source - -- without atoms (e.g. mips.h, gte.h, gcc_asm.h) has nothing for the - -- annotation DSL to validate — skip both report files. The previously - -- emitted ones, if any, get pruned below. - local has_atoms = #result.atoms > 0 + -- Decide whether this source contributes to the build at all. A source + -- without atoms (e.g. mips.h, gte.h, gcc_asm.h) has nothing for the + -- annotation DSL to validate — skip both report files. The previously + -- emitted ones, if any, get pruned below. + local has_atoms = #result.atoms > 0 - print(string.format("[pass] %s%s", source_path, - has_atoms and "" or " (no atoms - skip report)")) + print(string.format("[pass] %s%s", source_path, + has_atoms and "" or " (no atoms - skip report)")) - if has_atoms then - ensure_dir(out_dir) - pruned_dirs[out_dir] = true + if has_atoms then + ensure_dir(out_dir) + pruned_dirs[out_dir] = true - -- Per-source annotation report - write_file(out_txt, render_source_report(source_path, result)) - print(string.format(" -> %s", out_txt)) + -- Per-source annotation report + write_file(out_txt, render_source_report(source_path, result)) + print(string.format(" -> %s", out_txt)) - -- gen/.errors.h with #error lines for any structural problems. - -- The C build refuses to compile if any source produces errors via -include. - local header_lines = { - "// Auto-generated by tape_atom_annotation_pass.lua — DO NOT EDIT", - "#pragma once", - "", - } - for _, e in ipairs(result.errors) do - table.insert(header_lines, - string.format('#error "annotation: %s (line %d)"', e.msg, e.line)) - end - if #result.errors == 0 then - table.insert(header_lines, "// annotation pass OK") - end - write_file(out_err, table.concat(header_lines, "\n") .. "\n") - print(string.format(" -> %s", out_err)) + -- gen/.errors.h with #error lines for any structural problems. + -- The C build refuses to compile if any source produces errors via -include. + local header_lines = { + "// Auto-generated by tape_atom_annotation_pass.lua — DO NOT EDIT", + "#pragma once", + "", + } + for _, e in ipairs(result.errors) do + header_lines[#header_lines + 1] = + string.format('#error "annotation: %s (line %d)"', e.msg, e.line) + end + if #result.errors == 0 then + header_lines[#header_lines + 1] = "// annotation pass OK" + end + write_file(out_err, table.concat(header_lines, "\n") .. "\n") + print(string.format(" -> %s", out_err)) - table.insert(all_results, result) - else - -- No atoms: best-effort delete stale report files from a prior build. - -- We only remove the ones we know belong to this source (filename is - -- derived from the source path), never anything else in the dir. - for _, stale in ipairs({ out_txt, out_err }) do - local f = io.open(stale, "r") - if f then f:close(); os.remove(stale) end - end - end - end + all_results[#all_results + 1] = result + else + -- No atoms: best-effort delete stale report files from a prior build. + -- We only remove the ones we know belong to this source (filename is + -- derived from the source path), never anything else in the dir. + for _, stale in ipairs({ out_txt, out_err }) do + local f = io.open(stale, "r") + if f then f:close(); os.remove(stale) end + end + end + end - -- Write project-level summary. - local summary_path = dirname(args[2]) .. "/gen/annotation_validation.txt" - ensure_dir(dirname(summary_path)) - write_file(summary_path, render_project_report(all_results)) - print(string.format("[summary] %s\n", summary_path)) + -- Write project-level summary. + local summary_path = dirname(args[2]) .. "/gen/annotation_validation.txt" + ensure_dir(dirname(summary_path)) + write_file(summary_path, render_project_report(all_results)) + print(string.format("[summary] %s\n", summary_path)) - -- Exit code - local total_errors = 0 - for _, r in ipairs(all_results) do total_errors = total_errors + #r.errors end - if total_errors > 0 then os.exit(1) end + -- Exit code + local total_errors = 0 + for _, r in ipairs(all_results) do total_errors = total_errors + #r.errors end + if total_errors > 0 then os.exit(1) end end -main({...}) +main({...}) \ No newline at end of file