ed/pikuma_ps1
1
0
Fork 0
mirror of https://github.com/Ed94/pikuma_ps1.git synced 2026-07-14 05:11:25 -07:00
Code Issues Packages Projects Releases Wiki Activity

metaprogram improvements

Browse Source
This commit is contained in:
Edward R. Gonzalez
2026-07-09 01:17:35 -04:00
parent 602a0b46d8
commit 8541713d0c
4 changed files with 1712 additions and 1594 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/build_psyq.ps1
+2 -2
View File
@@ -343,7 +343,7 @@ function generate-TapeAtomOffsets {param([Parameter(Mandatory=$true)] [string[]]
}
write-host "AtomOffsets $($sources.Count) source(s)" -ForegroundColor Magenta
& lua $gen_atom_offsets_script $metadata @sources
& luajit $gen_atom_offsets_script $metadata @sources
if ($LASTEXITCODE -ne 0) {
write-error "Atom offset generation failed. Aborting."
exit 1
@@ -382,7 +382,7 @@ function generate-TapeAtomAnnotations {param([Parameter(Mandatory=$true)] [strin
}
write-host "AtomAnnotations $($sources.Count) source(s)" -ForegroundColor Magenta
& lua $gen_atom_annot_script $metadata @sources
& luajit $gen_atom_annot_script $metadata @sources
if ($LASTEXITCODE -ne 0) {
write-error "Atom annotation generation failed. Aborting."
exit 1
scripts/duffle.lua
+478
View File
@@ -0,0 +1,478 @@
-- duffle.lua
--
-- Shared primitives + domain tables for the tape-atom metaprograms.
-- Both `tape_atom_annotation_pass.lua` and `tape_atom.offset_gen.meta.lua`
-- `require("duffle")` for these.
--
-- 5.3-compatible Lua (no 5.4/5.5-only features):
-- - no <close> / <toclose>
-- - no continue keyword
-- - no string.dump improvements
-- - LuaJIT 5.1+extensions model is the primary target
--
-- No :match / :gmatch regex use anywhere; all delimiter-splitting is
-- hand-rolled or via LPeg (the regex-free PEG library).
--
-- Phase 3: the hot lexer primitives are LPeg-backed where it pays off.
-- The hand-rolled variants remain for callers that need a fallback.
local M = {}
-- ════════════════════════════════════════════════════════════════════════════
-- Section 0: LPeg patterns (compiled once at module load)
-- ════════════════════════════════════════════════════════════════════════════
--
-- LPeg is a PEG library (no regex). All patterns below are first-class
-- pattern values; they're cheap to build and reuse.
--
-- Note: lpeg is required lazily because Lua 5.5 may not have it on its
-- cpath at the same location as LuaJIT. We attempt the require and fall
-- back to the hand-rolled implementations if it fails.
local lpeg_ok, lpeg = pcall(require, "lpeg")
local lpeg_lib = nil
local lpeg_alpha_pat, lpeg_alnum_pat, lpeg_ident_pat
local lpeg_str_or_cmt_pat, lpeg_ws_and_cmt_pat
local lpeg_scan_to_target_pat -- generic "anything but target or balanced group" matcher
if lpeg_ok then
lpeg_lib = lpeg
local P, S, R = lpeg.P, lpeg.S, lpeg.R
-- Character class patterns
local alpha_pat = R("AZ", "az") + P("_")
local digit_pat = R("09")
lpeg_alnum_pat = alpha_pat + digit_pat
-- Identifier: alpha followed by zero+ alnum. Capture as a string.
lpeg_alpha_pat = alpha_pat
lpeg_ident_pat = lpeg.C(alpha_pat * lpeg_alnum_pat^0)
-- String literal: "..." with backslash escapes.
local lpeg_str_pat = P('"') * (P(1) - S('"\\') + P('\\') * P(1))^0 * P('"')
-- Char literal: '...' with backslash escapes.
local lpeg_chr_pat = P("'") * (P(1) - S("'\\") + P('\\') * P(1))^0 * P("'")
-- Line comment: // ... to end-of-line.
local lpeg_line_cmt_pat = P("//") * (P(1) - S("\n"))^0
-- Block comment: /* ... */ (no nesting per C standard).
local lpeg_block_cmt_pat = P("/*") * (P(1) - P("*/"))^0 * P("*/")
-- String or comment (any of the four forms).
lpeg_str_or_cmt_pat = lpeg_str_pat + lpeg_chr_pat + lpeg_line_cmt_pat + lpeg_block_cmt_pat
-- Whitespace + comment skipper: zero+ (whitespace run | string | comment).
local ws_pat = S(" \t\n\r\v\f")
lpeg_ws_and_cmt_pat = (ws_pat + lpeg_str_or_cmt_pat)^0
-- Generic "skip until target, but step over balanced groups" matcher.
-- Used by scan_to_char for non-ident / non-bracket chars.
-- We accept any single char except the target.
-- The balanced-group stepping is handled by the caller (via read_balanced).
lpeg_scan_to_target_pat = function(target)
return (P(1) - P(target))^0
end
end
-- ════════════════════════════════════════════════════════════════════════════
-- Section 1: character classification (byte-based for hot loops)
-- ════════════════════════════════════════════════════════════════════════════
--
-- Two APIs:
-- is_space(c), is_alpha(c), etc. — accept a single-char STRING (legacy)
-- is_space_byte(b), is_alpha_byte(b), etc. — accept a single-byte INTEGER
--
-- The byte-based versions are 5-10x faster in tight loops because they
-- avoid the string allocation per s:sub(i, i) call.
-- Whitespace characters per C locale.
function M.is_space_byte(b)
return b == 32 or b == 9 or b == 10 or b == 13 or b == 11 or b == 12
end
-- Letters (a-z, A-Z) and underscore.
function M.is_alpha_byte(b)
if not b then return false end
if b >= 97 and b <= 122 then return true end -- 'a'..'z'
if b >= 65 and b <= 90 then return true end -- 'A'..'Z'
return b == 95 -- '_'
end
-- Single digit.
function M.is_digit_byte(b) return b and b >= 48 and b <= 57 end
-- Letter OR digit OR underscore.
function M.is_alnum_byte(b) return M.is_alpha_byte(b) or M.is_digit_byte(b) end
-- String-based wrappers (kept for callers that already have a single-char
-- string; the byte versions are what the hot loops should call).
function M.is_space(c)
if type(c) == "number" then return M.is_space_byte(c) end
return c == " " or c == "\t" or c == "\n" or c == "\r" or c == "\v" or c == "\f"
end
function M.is_alpha(c)
if type(c) == "number" then return M.is_alpha_byte(c) end
if not c or #c == 0 then return false end
if c >= "a" and c <= "z" then return true end
if c >= "A" and c <= "Z" then return true end
return c == "_"
end
function M.is_digit(c)
if type(c) == "number" then return M.is_digit_byte(c) end
return c and c >= "0" and c <= "9"
end
function M.is_alnum(c) return M.is_alpha(c) or M.is_digit(c) end
-- ════════════════════════════════════════════════════════════════════════════
-- Section 2: string primitives
-- ════════════════════════════════════════════════════════════════════════════
-- Trim leading and trailing whitespace from a string.
function M.trim(s)
local a = 1
while a <= #s and M.is_space_byte(s:byte(a)) do a = a + 1 end
local b = #s
while b >= a and M.is_space_byte(s:byte(b)) do b = b - 1 end
return s:sub(a, b)
end
-- Linear-search for a single-byte target in a string.
-- (Phase 3 retained this for places where LPeg is overkill.)
function M.find_byte(haystack, target, start)
for i = start or 1, #haystack do
if haystack:byte(i) == target then return i end
end
return nil
end
-- Returns the directory portion of a path.
function M.dirname(path)
local last_sep = 0
for i = 1, #path do
if path:byte(i) == 47 or path:byte(i) == 92 then last_sep = i end -- '/' or '\\'
end
if last_sep == 0 then return "." end
return path:sub(1, last_sep - 1)
end
-- Returns the basename of a path, with the file extension stripped.
function M.basename_no_ext(path)
local last_sep = 0
for i = 1, #path do
if path:byte(i) == 47 or path:byte(i) == 92 then last_sep = i end
end
local a = last_sep + 1
local last_dot = #path + 1
for i = #path, a, -1 do
if path:byte(i) == 46 then last_dot = i; break end -- '.'
end
return path:sub(a, last_dot - 1)
end
-- ════════════════════════════════════════════════════════════════════════════
-- Section 3: I/O primitives
-- ════════════════════════════════════════════════════════════════════════════
function M.read_file(path)
local f = io.open(path, "r")
if not f then error("Cannot open " .. path) end
local content = f:read("*a")
f:close()
return content
end
function M.write_file(path, content)
local f = io.open(path, "w")
if not f then error("Cannot write " .. path) end
f:write(content)
f:close()
end
function M.ensure_dir(path)
local is_win = package.config:sub(1, 1) == "\\"
os.execute(is_win and ('if not exist "' .. path .. '" mkdir "' .. path .. '"')
or ('mkdir -p "' .. path .. '" 2>/dev/null'))
end
-- ════════════════════════════════════════════════════════════════════════════
-- Section 4: C-language scanner primitives
-- ════════════════════════════════════════════════════════════════════════════
-- LPeg-backed skipper when LPeg is available, hand-rolled fallback otherwise.
-- Returns position just past the construct, or `i` unchanged if no
-- string/comment starts at position i.
function M.skip_str_or_cmt(s, i)
if lpeg_ok then
local new_pos = lpeg.match(lpeg_str_or_cmt_pat, s, i)
if new_pos then return new_pos end
return i
end
-- Hand-rolled fallback (kept for builds where LPeg isn't available).
local c = s:byte(i)
if c == 34 or c == 39 then -- '"' or '\''
i = i + 1
while i <= #s do
local b = s:byte(i)
if b == 92 then i = i + 2 -- '\\'
elseif b == c then return i + 1
else i = i + 1 end
end
return #s + 1
elseif c == 47 then -- '/'
local nx = s:byte(i + 1)
if nx == 47 then -- '//'
while i <= #s and s:byte(i) ~= 10 do i = i + 1 end
return i
elseif nx == 42 then -- '/*'
i = i + 2
while i <= #s - 1 do
if s:byte(i) == 42 and s:byte(i + 1) == 47 then -- '*/'
return i + 2
end
i = i + 1
end
return #s + 1
end
end
return i
end
-- Skip whitespace AND C-style comments starting at position i.
-- LPeg-backed when available; ~5-10x faster than the hand-rolled version.
function M.skip_ws_and_cmt(s, i)
if lpeg_ok then
local new_pos = lpeg.match(lpeg_ws_and_cmt_pat, s, i)
if new_pos then return new_pos end
return i
end
-- Hand-rolled fallback.
local len = #s
while i <= len do
if M.is_space_byte(s:byte(i)) then
i = i + 1
else
local nx = M.skip_str_or_cmt(s, i)
if nx > i then i = nx else break end
end
end
return i
end
-- Read a C-style identifier (alpha followed by zero+ alnum) starting at
-- position i. Returns the identifier string + the position just past it,
-- or nil + i if no identifier starts here.
function M.read_ident(s, i)
if lpeg_ok then
local result = lpeg.match(lpeg_ident_pat, s, i)
if result then return result, i + #result end
return nil, i
end
-- Hand-rolled fallback.
if not M.is_alpha_byte(s:byte(i)) then return nil, i end
local a = i
i = i + 1
while i <= #s and M.is_alnum_byte(s:byte(i)) do i = i + 1 end
return s:sub(a, i - 1), i
end
-- Read a balanced-delimited group (parens, braces, or brackets) starting
-- at position i. Returns the inner content (between the delimiters) +
-- the position just past the closing delimiter, or nil + i if `s[i]`
-- isn't `open_char`.
--
-- (Hand-rolled; the depth counting makes pure LPeg awkward here.)
function M.read_balanced(s, open_char, close_char, i)
local open_byte = open_char:byte()
if s:byte(i) ~= open_byte then return nil, i end
i = i + 1
local len = #s
local depth = 1
local a = i
while i <= len and depth > 0 do
local c = s:byte(i)
if c == open_byte then
depth = depth + 1
i = i + 1
elseif c == close_char:byte() then
depth = depth - 1
if depth == 0 then break end
i = i + 1
else
local nx = M.skip_str_or_cmt(s, i)
if nx > i then i = nx else i = i + 1 end
end
end
return s:sub(a, i - 1), i + 1
end
-- Convenience specializations of read_balanced.
M.read_parens = function(s, i) return M.read_balanced(s, "(", ")", i) end
M.read_braces = function(s, i) return M.read_balanced(s, "{", "}", i) end
M.read_brackets = function(s, i) return M.read_balanced(s, "[", "]", i) end
-- Scan forward from position `start` until we find a specific single byte
-- `target`, transparently stepping over balanced parens/braces/brackets.
-- Returns the position of `target`, or nil if not found.
function M.scan_to_char(s, target, start)
local target_byte = target:byte()
local i = start
while i <= #s do
local c = s:byte(i)
if c == target_byte then return i end
if c == 40 then local _, a = M.read_balanced(s, "(", ")", i); i = a -- '('
elseif c == 123 then local _, a = M.read_balanced(s, "{", "}", i); i = a -- '{'
elseif c == 91 then local _, a = M.read_balanced(s, "[", "]", i); i = a -- '['
else
local nx = M.skip_str_or_cmt(s, i)
if nx > i then i = nx else i = i + 1 end
end
end
return nil
end
-- Split a brace-body into top-level comma-separated tokens. Honors nested
-- parens/braces/brackets and skips strings/comments.
function M.split_top_level_commas(body)
local tokens = {}
local i = 1
local token_start = 1
while i <= #body do
local c = body:byte(i)
if c == 40 then local _, a = M.read_parens(body, i); i = a -- '('
elseif c == 123 then local _, a = M.read_braces(body, i); i = a -- '{'
elseif c == 91 then local _, a = M.read_brackets(body, i); i = a -- '['
elseif c == 44 then -- ','
tokens[#tokens + 1] = body:sub(token_start, i - 1)
i = i + 1
token_start = i
else
local nx = M.skip_str_or_cmt(body, i)
if nx > i then i = nx; token_start = nx else i = i + 1 end
end
end
local last = body:sub(token_start)
if M.trim(last) ~= "" then tokens[#tokens + 1] = last end
return tokens
end
-- ════════════════════════════════════════════════════════════════════════════
-- Section 5: load_word_counts
-- ════════════════════════════════════════════════════════════════════════════
function M.load_word_counts(metadata_path)
local counts = {}
local content = M.read_file(metadata_path)
local len = #content
local i = 1
local prefix = "WORD_COUNT("
while i <= len do
local nl = M.find_byte(content, 10, i) -- '\n'
local line_end = nl or (len + 1)
local line = content:sub(i, line_end - 1)
local trimmed = M.trim(line)
if trimmed:sub(1, #prefix) == prefix and trimmed:sub(-1) == ")" then
local inner = trimmed:sub(#prefix + 1, #trimmed - 1)
local comma = M.find_byte(inner, 44, 1) -- ','
if comma then
counts[M.trim(inner:sub(1, comma - 1))] =
tonumber(M.trim(inner:sub(comma + 1)))
end
end
i = line_end + 1
end
return counts
end
-- ════════════════════════════════════════════════════════════════════════════
-- Section 6: LineIndex (perf fix — replaces the per-call rescan line_of)
-- ════════════════════════════════════════════════════════════════════════════
function M.LineIndex(source)
local positions = {}
local n = 0
for i = 1, #source do
if source:byte(i) == 10 then -- '\n'
n = n + 1
positions[n] = i
end
end
local function line_of(pos)
local lo, hi = 1, n
while lo <= hi do
local mid = math.floor((lo + hi) / 2)
if positions[mid] <= pos then
lo = mid + 1
else
hi = mid - 1
end
end
return hi + 1
end
return line_of
end
-- ════════════════════════════════════════════════════════════════════════════
-- Section 7: domain tables
-- ════════════════════════════════════════════════════════════════════════════
M.WAVE_CONTEXT_REGS = {
["R_PrimCursor"] = { alias = "R_T7", size = 4, role = "output cursor (prim arena)" },
["R_FaceCursor"] = { alias = "R_T4", size = 4, role = "input cursor (face array)" },
["R_VertBase"] = { alias = "R_T5", size = 4, role = "base pointer (vertex array)" },
["R_OtBase"] = { alias = "R_T6", size = 4, role = "base pointer (ordering table)" },
}
M.MACRO_EXPANSION = {
["phase_init"] = "init",
["phase_bind"] = "bind",
["phase_setup"] = "setup",
["phase_work"] = "work",
["phase_commit"] = "commit",
["phase_terminate"] = "terminate",
["REGION_PRIM_ARENA"] = "prim_arena",
["REGION_FACE_ARENA"] = "face_arena",
["REGION_VERTEX_ARENA"] = "vertex_arena",
["REGION_OT_ARENA"] = "ot_arena",
["REGION_HEAP_3D"] = "heap_3d_models",
["REGION_CDROM_STREAM"] = "cdrom_stream",
["REGION_VRAM"] = "vram_heap",
["CADENCE_FRAME"] = "frame",
["CADENCE_ONCE"] = "once",
["CADENCE_ONDEMAND"] = "ondemand",
}
M.KNOWN_PHASES = {
["init"] = true, ["bind"] = true, ["setup"] = true,
["work"] = true, ["commit"] = true, ["terminate"] = true,
}
M.KNOWN_REGIONS = {
["prim_arena"] = true, ["face_arena"] = true,
["vertex_arena"] = true, ["ot_arena"] = true,
["heap_3d_models"] = true, ["cdrom_stream"] = true,
["vram_heap"] = true,
}
M.KNOWN_CADENCES = {
["frame"] = true, ["once"] = true, ["ondemand"] = true,
}
M.TAPE_ATOM_MACROS = {
["atom_annot"] = { kind = "work", binds = false },
["atom_bind"] = { kind = "bind", binds = true },
["atom_setup"] = { kind = "setup", binds = false },
["atom_commit"] = { kind = "commit", binds = false },
["atom_init"] = { kind = "init", binds = false },
["atom_terminate"] = { kind = "terminate", binds = false },
}
M.ATOM_PRAGMA_KINDS = {
["resource"] = { kind = "string" },
["region"] = { kind = "ident", allowed = M.KNOWN_REGIONS },
["group"] = { kind = "ident" },
["cadence"] = { kind = "ident", allowed = M.KNOWN_CADENCES },
["async"] = { kind = "ident", allowed = { ["true"] = true, ["false"] = true } },
}
-- Expose the lpeg_ok flag so callers can detect the LPeg-back path.
-- True when LPeg was successfully required and the patterns above were
-- compiled at module load time. False when running in fallback mode.
M.lpeg_ok = lpeg_ok
return M
scripts/tape_atom.offset_gen.meta.lua
+318 -468
View File
@@ -20,271 +20,107 @@
-- #pragma endregion <basename>
--
-- Usage:
-- lua gen_atom_offsets.lua <metadata.h> <source1> [source2 ...]
-- luajit gen_atom_offsets.lua <metadata.h> <source1> [source2 ...]
-- ============================================================
-- Character classification
-- ============================================================
local function is_space(c) return c == " " or c == "\t" or c == "\n" or c == "\r" or c == "\v" or c == "\f" end
local function is_alpha(c)
if not c or #c == 0 then return false end
if c >= "a" and c <= "z" then return true end
if c >= "A" and c <= "Z" then return true end
return c == "_"
-- Make require("duffle") resolve to the sibling duffle.lua in this dir,
-- AND make require("lpeg") find the vendored LPeg DLL in the toolchain.
-- Both prepends are explicit (no :match / no Lua pattern — plain byte scan).
local script_path = arg[0]
local last_sep = 0
for i = 1, #script_path do
local c = script_path:sub(i, i)
if c == "/" or c == "\\" then last_sep = i end
end
local script_dir = last_sep == 0 and "./" or script_path:sub(1, last_sep)
package.path = script_dir .. "?.lua;" .. script_dir .. "?/init.lua;" .. package.path
package.cpath = "C:\\projects\\Pikuma\\ps1\\toolchain\\luajit-2.1\\lib\\lua\\5.1\\?.dll;" .. package.cpath
local function is_digit(c) return c and c >= "0" and c <= "9" end
local function is_alnum(c) return is_alpha(c) or is_digit(c) end
-- Shared primitives + domain tables live in scripts/duffle.lua.
local duffle = require("duffle")
-- Local aliases so the rest of this file reads cleanly. These resolve
-- to the same functions in duffle.lua (5.3-compatible, no regex).
local is_space = duffle.is_space
local is_alpha = duffle.is_alpha
local is_alnum = duffle.is_alnum
local trim = duffle.trim
local find_byte = duffle.find_byte
local read_file = duffle.read_file
local write_file = duffle.write_file
local ensure_dir = duffle.ensure_dir
local dirname = duffle.dirname
local basename_no_ext = duffle.basename_no_ext
local skip_str_or_cmt = duffle.skip_str_or_cmt
local skip_ws_and_cmt = duffle.skip_ws_and_cmt
local read_ident = duffle.read_ident
local read_parens = duffle.read_parens
local read_braces = duffle.read_braces
local read_brackets = duffle.read_brackets
local scan_to_char = duffle.scan_to_char
local split_top_level_commas = duffle.split_top_level_commas
local load_word_counts = duffle.load_word_counts
-- ============================================================
-- I/O
-- Offset-gen-specific helpers (not in duffle.lua)
-- ============================================================
local function read_file(path)
local f = io.open(path, "r")
if not f then error("Cannot open " .. path) end
local content = f:read("*a")
f:close()
return content
end
local function write_file(path, content)
local f = io.open(path, "w")
if not f then error("Cannot write " .. path) end
f:write(content)
f:close()
end
-- PowerShell aliases `mkdir` to New-Item, which treats `-p` as a path, so guard the call.
local function ensure_dir(path)
local is_win = package.config:sub(1, 1) == "\\"
os.execute(is_win and ('if not exist "' .. path .. '" mkdir "' .. path .. '"') or ('mkdir -p "' .. path .. '" 2>/dev/null'))
end
-- ============================================================
-- String primitives
-- ============================================================
local function trim(s)
local a = 1; while a <= #s and is_space(s:sub(a, a)) do a = a + 1 end
local b = #s; while b >= a and is_space(s:sub(b, b)) do b = b - 1 end
return s:sub(a, b)
end
local function starts_with(s, prefix)
if #s < #prefix then return false end
for i = 1, #prefix do
if s:sub(i, i) ~= prefix:sub(i, i) then return false end
end
return true
if #s < #prefix then return false end
for i = 1, #prefix do
if s:sub(i, i) ~= prefix:sub(i, i) then return false end
end
return true
end
local function ends_with(s, suffix)
if #s < #suffix then return false end
local off = #s - #suffix
for i = 1, #suffix do
if s:sub(off + i, off + i) ~= suffix:sub(i, i) then return false end
end
return true
end
local function find_byte(haystack, target, start)
for i = start or 1, #haystack do
if haystack:sub(i, i) == target then return i end
end
return nil
end
local function dirname(path)
local last_sep = 0
for i = 1, #path do
local c = path:sub(i, i)
if c == "/" or c == "\\" then last_sep = i end
end
if last_sep == 0 then return "." end
return path:sub(1, last_sep - 1)
end
local function basename_no_ext(path)
local last_sep = 0
for i = 1, #path do
local c = path:sub(i, i)
if c == "/" or c == "\\" then last_sep = i end
end
local a = last_sep + 1
local last_dot = #path + 1
for i = #path, a, -1 do
if path:sub(i, i) == "." then last_dot = i; break end
end
return path:sub(a, last_dot - 1)
if #s < #suffix then return false end
local off = #s - #suffix
for i = 1, #suffix do
if s:sub(off + i, off + i) ~= suffix:sub(i, i) then return false end
end
return true
end
local function to_upper(s) return s:upper() end
local function to_alnum_underscore(s)
local out = ""
for i = 1, #s do
local c = s:sub(i, i)
if is_alnum(c) then out = out .. c
else out = out .. "_" end
end
return out
local out = ""
for i = 1, #s do
local c = s:sub(i, i)
if is_alnum(c) then out = out .. c
else out = out .. "_" end
end
return out
end
local function pad_right(s, w) return s .. string.rep(" ", w - #s) end
-- ============================================================
-- Lexer helpers
-- ============================================================
-- If position i starts a C string literal ("..."), char literal ('.'),
-- // line comment, or /* block comment, advance past it and return the
-- position just after the construct (or #s+1 if unterminated).
-- Otherwise return i unchanged.
local function skip_str_or_cmt(s, i)
local c = s:sub(i, i)
if c == '"' or c == "'" then
i = i + 1
while i <= #s do
if s:sub(i, i) == "\\" then i = i + 2
elseif s:sub(i, i) == c then return i + 1
else i = i + 1 end
end
return #s + 1
elseif c == "/" then
local nx = s:sub(i+1, i+1)
if nx == "/" then
while i <= #s and s:sub(i, i) ~= "\n" do i = i + 1 end
return i
elseif nx == "*" then
i = i + 2
while i <= #s - 1 do
if s:sub(i, i) == "*" and s:sub(i+1, i+1) == "/" then
return i + 2
end
i = i + 1
end
return #s + 1
end
end
return i
end
local function skip_ws_and_cmt(s, i)
while i <= #s do
if is_space(s:sub(i, i)) then i = i + 1
else
local nx = skip_str_or_cmt(s, i)
if nx > i then i = nx else break end
end
end
return i
end
local function read_ident(source, i)
if not is_alpha(source:sub(i, i)) then return nil, i end
local a = i
i = i + 1
while i <= #source and is_alnum(source:sub(i, i)) do i = i + 1 end
return source:sub(a, i - 1), i
end
local function read_balanced(s, open_char, close_char, i)
if s:sub(i, i) ~= open_char then return nil, i end
i = i + 1
local len = #s
local depth = 1
local a = i
while i <= len and depth > 0 do
local c = s:sub(i, i)
if c == open_char then
depth = depth + 1
i = i + 1
elseif c == close_char then
depth = depth - 1
if depth == 0 then break end
i = i + 1
else
local nx = skip_str_or_cmt(s, i)
if nx > i then i = nx else i = i + 1 end
end
end
return s:sub(a, i - 1), i + 1
end
local read_parens = function(s, i) return read_balanced(s, "(", ")", i) end
local read_braces = function(s, i) return read_balanced(s, "{", "}", i) end
local read_brackets = function(s, i) return read_balanced(s, "[", "]", i) end
local function scan_to_char(s, target, start)
local i = start
while i <= #s do
local c = s:sub(i, i)
if c == target then return i end
if c == "(" then local _, a = read_balanced(s, "(", ")", i); i = a
elseif c == "{" then local _, a = read_balanced(s, "{", "}", i); i = a
elseif c == "[" then local _, a = read_balanced(s, "[", "]", i); i = a
else
local nx = skip_str_or_cmt(s, i)
if nx > i then i = nx else i = i + 1 end
end
end
end
-- ============================================================
-- Extract comma-separated identifier args from a parenthesized group
-- after a function-like macro call.
-- ============================================================
local function extract_ident_args(token, after_ident)
local arg_start = skip_ws_and_cmt(token, after_ident)
if token:sub(arg_start, arg_start) ~= "(" then return {}, nil end
local inner, after_paren = read_parens(token, arg_start)
local arg_start = skip_ws_and_cmt(token, after_ident)
if token:sub(arg_start, arg_start) ~= "(" then return {}, nil end
local inner, after_paren = read_parens(token, arg_start)
local args = {}
local n = 1
local len = #inner
while n <= len do
n = skip_ws_and_cmt(inner, n)
if n > len then break end
local ident, after = read_ident(inner, n)
if ident and ident ~= "" then
table.insert(args, ident)
n = after
else
n = n + 1
end
n = skip_ws_and_cmt(inner, n)
if n <= len and inner:sub(n, n) == "," then n = n + 1 end
end
local args = {}
local n = 1
local len = #inner
while n <= len do
n = skip_ws_and_cmt(inner, n)
if n > len then break end
local ident, after = read_ident(inner, n)
if ident and ident ~= "" then
table.insert(args, ident)
n = after
else
n = n + 1
end
n = skip_ws_and_cmt(inner, n)
if n <= len and inner:sub(n, n) == "," then n = n + 1 end
end
return args, after_paren
end
-- ============================================================
-- Load WORD_COUNT manifest
-- ============================================================
local function load_word_counts(metadata_path)
local counts = {}
local content = read_file(metadata_path)
local len = #content
local i = 1
local prefix = "WORD_COUNT("
while i <= len do
local nl = find_byte(content, "\n", i)
local line_end = nl or (len + 1)
local line = content:sub(i, line_end - 1)
local trimmed = trim(line)
if starts_with(trimmed, prefix) and ends_with(trimmed, ")") then
local inner = trimmed:sub(#prefix + 1, #trimmed - 1)
local comma = find_byte(inner, ",", 1)
if comma then
counts[trim(inner:sub(1, comma - 1))] = tonumber(trim(inner:sub(comma + 1)))
end
end
i = line_end + 1
end
return counts
return args, after_paren
end
-- ============================================================
@@ -292,43 +128,16 @@ end
-- ============================================================
local function word_count_of_token(token, wc)
local s = trim(token)
if s == "" then return 0 end
local name, after = read_ident(s, 1)
if not name then return 1 end
if wc[name] then return wc[name] end
local j = skip_ws_and_cmt(s, after)
if s:sub(j, j) == "(" then
io.stderr:write(" warning: unknown macro '" .. name .. "', assuming 1 word\n")
end
return 1
end
-- ============================================================
-- Split brace-body into top-level comma-separated tokens
-- ============================================================
local function split_top_level_commas(body)
local tokens = {}
local i = 1
local token_start = 1
while i <= #body do
local c = body:sub(i, i)
if c == "(" then local _, a = read_parens(body, i); i = a
elseif c == "{" then local _, a = read_braces(body, i); i = a
elseif c == "[" then local _, a = read_brackets(body, i); i = a
elseif c == "," then
table.insert(tokens, body:sub(token_start, i - 1))
i = i + 1
token_start = i
else
local nx = skip_str_or_cmt(body, i)
if nx > i then i = nx else i = i + 1 end
end
end
local last = body:sub(token_start)
if trim(last) ~= "" then table.insert(tokens, last) end
return tokens
local s = trim(token)
if s == "" then return 0 end
local name, after = read_ident(s, 1)
if not name then return 1 end
if wc[name] then return wc[name] end
local j = skip_ws_and_cmt(s, after)
if s:sub(j, j) == "(" then
io.stderr:write(" warning: unknown macro '" .. name .. "', assuming 1 word\n")
end
return 1
end
-- ============================================================
@@ -337,54 +146,95 @@ end
-- ============================================================
local function scan_for_atom_markers(token, at_pos, labels, branches)
local i = 1
local len = #token
while i <= len do
i = skip_ws_and_cmt(token, i)
if i > len then break end
local c = token:sub(i, i)
if is_alpha(c) then
local ident, after = read_ident(token, i)
if ident == "atom_label" then
local args, after_paren = extract_ident_args(token, after)
if #args >= 1 then labels[args[1]] = at_pos end
if after_paren then i = after_paren else i = after end
elseif ident == "atom_offset" then
local args, after_paren = extract_ident_args(token, after)
if #args >= 2 then table.insert(branches, {pos = at_pos, target = args[2], tag = args[1]}) end
if after_paren then i = after_paren else i = after end
else
i = after
end
else
local nx = skip_str_or_cmt(token, i)
if nx > i then i = nx else i = i + 1 end
end
end
local i = 1
local len = #token
while i <= len do
i = skip_ws_and_cmt(token, i)
if i > len then break end
local c = token:sub(i, i)
if is_alpha(c) then
local ident, after = read_ident(token, i)
if ident == "atom_label" then
local args, after_paren = extract_ident_args(token, after)
if #args >= 1 then labels[args[1]] = at_pos end
if after_paren then i = after_paren else i = after end
elseif ident == "atom_offset" then
local args, after_paren = extract_ident_args(token, after)
if #args >= 2 then table.insert(branches, {pos = at_pos, target = args[2], tag = args[1]}) end
if after_paren then i = after_paren else i = after end
else
i = after
end
else
local nx = skip_str_or_cmt(token, i)
if nx > i then i = nx else i = i + 1 end
end
end
end
-- ============================================================
-- Scan atom body, count words, find markers
-- ============================================================
-- Find the end position (just past the closing ')') of the first
-- atom_label/atom_offset call in `tok`. Returns 0 if no such call.
local function find_marker_call_end(tok)
local i = 1
local len = #tok
while i <= len do
i = skip_ws_and_cmt(tok, i)
if i > len then break end
local c = tok:sub(i, i)
if is_alpha(c) then
local ident, after = read_ident(tok, i)
if ident == "atom_label" or ident == "atom_offset" then
local j = skip_ws_and_cmt(tok, after)
if tok:sub(j, j) == "(" then
local _, end_paren = read_parens(tok, j)
return end_paren - 1
end
return 0
end
i = after
else
local nx = skip_str_or_cmt(tok, i)
if nx > i then i = nx else i = i + 1 end
end
end
return 0
end
local function scan_atom_body(body, word_counts)
local pos = 0
local labels = {}
local branches = {}
for _, tok in ipairs(split_top_level_commas(body)) do
local k = 1
local tlen = #tok
while k <= tlen and is_space(tok:sub(k, k)) do k = k + 1 end
local leading_ident = read_ident(tok, k)
if leading_ident == "atom_label" or leading_ident == "atom_offset" then
scan_for_atom_markers(tok, pos, labels, branches)
else
local words = word_count_of_token(tok, word_counts)
scan_for_atom_markers(tok, pos, labels, branches)
pos = pos + words
end
end
return labels, branches, pos
local pos = 0
local labels = {}
local branches = {}
for _, tok in ipairs(split_top_level_commas(body)) do
local k = 1
local tlen = #tok
while k <= tlen and is_space(tok:sub(k, k)) do k = k + 1 end
local leading_ident = read_ident(tok, k)
if leading_ident == "atom_label" or leading_ident == "atom_offset" then
-- Marker call: record at the current pos, do NOT advance pos.
-- But the source pattern may bundle the marker with the next
-- instruction on a new line (no top-level comma between them).
-- In that case, the rest of `tok` after the marker call is a
-- real instruction that must still be counted.
scan_for_atom_markers(tok, pos, labels, branches)
local marker_end = find_marker_call_end(tok)
if marker_end > 0 and marker_end < #tok then
local rest = trim(tok:sub(marker_end + 1))
if rest ~= "" then
local rest_words = word_count_of_token(rest, word_counts)
pos = pos + rest_words
end
end
else
local words = word_count_of_token(tok, word_counts)
scan_for_atom_markers(tok, pos, labels, branches)
pos = pos + words
end
end
return labels, branches, pos
end
-- ============================================================
@@ -392,81 +242,81 @@ end
-- ============================================================
local function skip_qualifiers(source, i)
local keywords = {
["static"] = true, ["const"] = true, ["volatile"] = true,
["extern"] = true, ["register"] = true, ["auto"] = true,
["inline"] = true, ["typedef"] = true,
["internal"]= true, ["LP_"] = true, ["global"] = true, ["gkknown"] = true
}
while true do
i = skip_ws_and_cmt(source, i)
local ident, after = read_ident(source, i)
if not ident then return i end
if keywords[ident] then i = after else return i end
end
local keywords = {
["static"] = true, ["const"] = true, ["volatile"] = true,
["extern"] = true, ["register"] = true, ["auto"] = true,
["inline"] = true, ["typedef"] = true,
["internal"]= true, ["LP_"] = true, ["global"] = true, ["gkknown"] = true
}
while true do
i = skip_ws_and_cmt(source, i)
local ident, after = read_ident(source, i)
if not ident then return i end
if keywords[ident] then i = after else return i end
end
end
local function find_atoms(source_text)
local atoms = {}
local len = #source_text
local i = 1
local atoms = {}
local len = #source_text
local i = 1
local function try_wrapped(after_pos)
local paren_pos = skip_ws_and_cmt(source_text, after_pos)
if source_text:sub(paren_pos, paren_pos) ~= "(" then return nil end
local inner, after_paren = read_parens(source_text, paren_pos)
local n = 1
while n <= #inner and is_space(inner:sub(n, n)) do n = n + 1 end
local ns = n
while n <= #inner and is_alnum(inner:sub(n, n)) do n = n + 1 end
local name = inner:sub(ns, n - 1)
if name == "" then return nil end
local brace_pos = scan_to_char(source_text, "{", after_paren)
if not brace_pos then return nil end
local body, after_brace = read_braces(source_text, brace_pos)
return {name = name, body = body, after_brace = after_brace}
end
local function try_wrapped(after_pos)
local paren_pos = skip_ws_and_cmt(source_text, after_pos)
if source_text:sub(paren_pos, paren_pos) ~= "(" then return nil end
local inner, after_paren = read_parens(source_text, paren_pos)
local n = 1
while n <= #inner and is_space(inner:sub(n, n)) do n = n + 1 end
local ns = n
while n <= #inner and is_alnum(inner:sub(n, n)) do n = n + 1 end
local name = inner:sub(ns, n - 1)
if name == "" then return nil end
local brace_pos = scan_to_char(source_text, "{", after_paren)
if not brace_pos then return nil end
local body, after_brace = read_braces(source_text, brace_pos)
return {name = name, body = body, after_brace = after_brace}
end
local function try_raw(after_pos)
local next_pos = skip_ws_and_cmt(source_text, after_pos)
local next_ident, next_after = read_ident(source_text, next_pos)
if not next_ident then return nil end
if not starts_with(next_ident, "code_") then return nil end
if #next_ident <= 5 then return nil end
local atom_name = next_ident:sub(6)
local brace_pos = scan_to_char(source_text, "{", next_after)
if not brace_pos then return nil end
local body, after_brace = read_braces(source_text, brace_pos)
return {name = atom_name, body = body, after_brace = after_brace}
end
local function try_raw(after_pos)
local next_pos = skip_ws_and_cmt(source_text, after_pos)
local next_ident, next_after = read_ident(source_text, next_pos)
if not next_ident then return nil end
if not starts_with(next_ident, "code_") then return nil end
if #next_ident <= 5 then return nil end
local atom_name = next_ident:sub(6)
local brace_pos = scan_to_char(source_text, "{", next_after)
if not brace_pos then return nil end
local body, after_brace = read_braces(source_text, brace_pos)
return {name = atom_name, body = body, after_brace = after_brace}
end
while i <= len do
i = skip_ws_and_cmt(source_text, i); if i > len then break end
i = skip_qualifiers(source_text, i); if i > len then break end
local ident, after = read_ident(source_text, i)
if not ident then
i = i + 1
elseif ident == "MipsAtom_" then
local atom = try_wrapped(after)
if atom then
table.insert(atoms, {name = atom.name, body = atom.body})
i = atom.after_brace
else
i = i + 1
end
elseif ident == "MipsCode" then
local atom = try_raw(after)
if atom then
table.insert(atoms, {name = atom.name, body = atom.body})
i = atom.after_brace
else
i = after
end
else
i = after
end
end
return atoms
while i <= len do
i = skip_ws_and_cmt(source_text, i); if i > len then break end
i = skip_qualifiers(source_text, i); if i > len then break end
local ident, after = read_ident(source_text, i)
if not ident then
i = i + 1
elseif ident == "MipsAtom_" then
local atom = try_wrapped(after)
if atom then
table.insert(atoms, {name = atom.name, body = atom.body})
i = atom.after_brace
else
i = i + 1
end
elseif ident == "MipsCode" then
local atom = try_raw(after)
if atom then
table.insert(atoms, {name = atom.name, body = atom.body})
i = atom.after_brace
else
i = after
end
else
i = after
end
end
return atoms
end
-- ============================================================
@@ -474,15 +324,15 @@ end
-- ============================================================
local function compute_offsets(labels, branches)
local results = {}
for _, br in ipairs(branches) do
local target = labels[br.target]
if not target then
error("Branch target '" .. br.target .. "' has no atom_label (at word " .. br.pos .. ")")
end
table.insert(results, {target = br.target, tag = br.tag, offset = target - br.pos - 1 })
end
return results
local results = {}
for _, br in ipairs(branches) do
local target = labels[br.target]
if not target then
error("Branch target '" .. br.target .. "' has no atom_label (at word " .. br.pos .. ")")
end
table.insert(results, {target = br.target, tag = br.tag, offset = target - br.pos - 1 })
end
return results
end
-- ============================================================
@@ -490,45 +340,45 @@ end
-- ============================================================
local function generate_header(source_path, atoms_data)
local basename = basename_no_ext(source_path)
local guard = to_alnum_underscore(to_upper(basename)) .. "_OFFSETS_H"
local basename = basename_no_ext(source_path)
local guard = to_alnum_underscore(to_upper(basename)) .. "_OFFSETS_H"
local lines = {}
local function add(s) table.insert(lines, s) end
local lines = {}
local function add(s) table.insert(lines, s) end
add("// Auto-generated by tape_atom_offset_gen.meta.lua — DO NOT EDIT")
add("// Source: " .. source_path)
add("#pragma once")
add("")
add("#pragma region " .. basename)
add("")
-- add("// Dispatch macro: token-pastes <tag>_<target> to the enum name")
-- add("#undef atom_offset")
-- add("#define atom_offset(tag, name) atom_offset_##tag##_##name")
add("")
for _, atom in ipairs(atoms_data) do
if #atom.offsets > 0 then
add("// --- atom: " .. atom.name .. " (" .. atom.total_words .. " words) ---")
add("")
local consts = {}
for _, r in ipairs(atom.offsets) do
table.insert(consts, {
macro_name = "_atom_offset_" .. r.tag .. "_" .. r.target,
enum_name = "atom_offset_" .. r.tag .. "_" .. r.target,
value = r.offset
})
end
for _, c in ipairs(consts) do add("#define " .. pad_right(c.macro_name, 44) .. " " .. c.value .. "") end
add("")
add("enum {")
for _, c in ipairs(consts) do add(" " .. c.enum_name .. " = " .. c.macro_name .. ",") end
add("};")
add("")
end
end
add("#pragma endregion " .. basename)
add("")
return table.concat(lines, "\n") .. "\n"
add("// Auto-generated by tape_atom_offset_gen.meta.lua — DO NOT EDIT")
add("// Source: " .. source_path)
add("#pragma once")
add("")
add("#pragma region " .. basename)
add("")
-- add("// Dispatch macro: token-pastes <tag>_<target> to the enum name")
-- add("#undef atom_offset")
-- add("#define atom_offset(tag, name) atom_offset_##tag##_##name")
add("")
for _, atom in ipairs(atoms_data) do
if #atom.offsets > 0 then
add("// --- atom: " .. atom.name .. " (" .. atom.total_words .. " words) ---")
add("")
local consts = {}
for _, r in ipairs(atom.offsets) do
table.insert(consts, {
macro_name = "_atom_offset_" .. r.tag .. "_" .. r.target,
enum_name = "atom_offset_" .. r.tag .. "_" .. r.target,
value = r.offset
})
end
for _, c in ipairs(consts) do add("#define " .. pad_right(c.macro_name, 44) .. " " .. c.value .. "") end
add("")
add("enum {")
for _, c in ipairs(consts) do add(" " .. c.enum_name .. " = " .. c.macro_name .. ",") end
add("};")
add("")
end
end
add("#pragma endregion " .. basename)
add("")
return table.concat(lines, "\n") .. "\n"
end
-- ============================================================
@@ -536,39 +386,39 @@ end
-- ============================================================
local function process_source(source_path, word_counts)
local source = read_file(source_path)
local atoms_raw = find_atoms(source)
local source = read_file(source_path)
local atoms_raw = find_atoms(source)
if #atoms_raw == 0 then
-- io.stderr:write(" note: no MipsAtom_ declarations in " .. source_path .. "\n")
return
end
if #atoms_raw == 0 then
-- io.stderr:write(" note: no MipsAtom_ declarations in " .. source_path .. "\n")
return
end
local atoms_data = {}
for _, atom in ipairs(atoms_raw) do
local labels, branches, total = scan_atom_body(atom.body, word_counts)
local offsets = compute_offsets(labels, branches)
table.insert(atoms_data, {
name = atom.name,
total_words = total,
offsets = offsets
})
end
local atoms_data = {}
for _, atom in ipairs(atoms_raw) do
local labels, branches, total = scan_atom_body(atom.body, word_counts)
local offsets = compute_offsets(labels, branches)
table.insert(atoms_data, {
name = atom.name,
total_words = total,
offsets = offsets
})
end
local basename = basename_no_ext(source_path)
local out_dir = dirname(source_path) .. "/gen"
ensure_dir(out_dir)
local out_path = out_dir .. "/" .. basename .. ".offsets.h"
write_file(out_path, generate_header(source_path, atoms_data))
local basename = basename_no_ext(source_path)
local out_dir = dirname(source_path) .. "/gen"
ensure_dir(out_dir)
local out_path = out_dir .. "/" .. basename .. ".offsets.h"
write_file(out_path, generate_header(source_path, atoms_data))
local total_branches = 0
for _, a in ipairs(atoms_data) do total_branches = total_branches + #a.offsets end
print(" " .. basename .. ": " .. #atoms_data .. " atom(s), " .. total_branches .. " branch(es)")
for _, a in ipairs(atoms_data) do
for _, r in ipairs(a.offsets) do
print(" " .. a.name .. " -> " .. r.tag .. ":" .. r.target .. " : " .. r.offset)
end
end
local total_branches = 0
for _, a in ipairs(atoms_data) do total_branches = total_branches + #a.offsets end
print(" " .. basename .. ": " .. #atoms_data .. " atom(s), " .. total_branches .. " branch(es)")
for _, a in ipairs(atoms_data) do
for _, r in ipairs(a.offsets) do
print(" " .. a.name .. " -> " .. r.tag .. ":" .. r.target .. " : " .. r.offset)
end
end
end
-- ============================================================
@@ -576,12 +426,12 @@ end
-- ============================================================
local function main(args)
if #args < 2 then
print("Usage: gen_atom_offsets.lua <metadata.h> <source1> [source2 ...]")
os.exit(1)
end
local word_counts = load_word_counts(args[1])
for i = 2, #args do process_source(args[i], word_counts) end
if #args < 2 then
print("Usage: luajit gen_atom_offsets.lua <metadata.h> <source1> [source2 ...]")
os.exit(1)
end
local word_counts = load_word_counts(args[1])
for i = 2, #args do process_source(args[i], word_counts) end
end
main({...})
main({...})
scripts/tape_atom_annotation_pass.lua
+914 -1124
View File
File diff suppressed because it is too large Load Diff