ed/pikuma_ps1
1
0
Fork 0
mirror of https://github.com/Ed94/pikuma_ps1.git synced 2026-07-13 04:41:26 -07:00
Code Issues Packages Projects Releases Wiki Activity

rework of metaprogram

Browse Source
This commit is contained in:
Edward R. Gonzalez
2026-07-09 19:30:32 -04:00
parent 1ffdda45e5
commit 59903546d7
8 changed files with 2086 additions and 2275 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/build_psyq.ps1
+36 -94
View File
@@ -316,101 +316,39 @@ function build-graphis_hello {
make-binary $elf $exe
}
# build-graphis_hello
# ps1-meta orchestrator. Replaces generate-TapeAtomOffsets +
# generate-TapeAtomAnnotations with a single invocation. Dispatches
# the 6 passes (word-counts / components / annotation / offsets /
# static-analysis / report) in dependency-topological order.
function generate-TapeAtomOffsets {param([Parameter(Mandatory=$true)] [string[]]$sources, [Parameter(Mandatory=$true)] [string]$metadata)
$gen_atom_offsets_script = join-path $path_scripts 'tape_atom.offset_gen.meta.lua'
$any_stale = $false
foreach ($src in $sources) {
$basename = [System.IO.Path]::GetFileNameWithoutExtension($src)
$dir = split-path -Path $src -Parent
$gen_dir = join-path $dir 'gen'
$out = join-path $gen_dir "$basename.offsets.h"
if (-not (test-path $out)) { $any_stale = $true; break }
$src_mtime = (get-item $src).LastWriteTimeUtc
$out_mtime = (get-item $out).LastWriteTimeUtc
$meta_mtime = (get-item $metadata).LastWriteTimeUtc
if (($src_mtime -gt $out_mtime) -or ($meta_mtime -gt $out_mtime)) {
$any_stale = $true
break
}
}
if (-not $any_stale) {
write-host "AtomOffsets all $($sources.Count) source(s) up-to-date" -ForegroundColor DarkGray
return
}
write-host "AtomOffsets $($sources.Count) source(s)" -ForegroundColor Magenta
& luajit $gen_atom_offsets_script $metadata @sources
if ($LASTEXITCODE -ne 0) {
write-error "Atom offset generation failed. Aborting."
exit 1
}
function any-stale {
param([Parameter(Mandatory=$true)][string[]]$sources,
[Parameter(Mandatory=$true)][string]$metadata,
[Parameter(Mandatory=$true)][string]$out_root)
if (-not (test-path $out_root)) { return $true }
$out_mtime = (get-item $out_root).LastWriteTimeUtc
$src_mtime = ($sources | ForEach-Object { (get-item $_).LastWriteTimeUtc } | Measure-Object -Maximum).Maximum
$meta_mtime = (get-item $metadata).LastWriteTimeUtc
return ($src_mtime -gt $out_mtime) -or ($meta_mtime -gt $out_mtime)
}
function generate-TapeAtomAnnotations {param([Parameter(Mandatory=$true)] [string[]]$sources, [Parameter(Mandatory=$true)] [string]$metadata)
# Sibling to generate-TapeAtomOffsets. Validates TAPE_ATOM_* / TAPE_WORDS
# annotations against the metadata manifest. Emits gen/<basename>.errors.h
# containing #error directives for the C build to fail on annotation drift.
$gen_atom_annot_script = join-path $path_scripts 'tape_atom_annotation_pass.lua'
$any_stale = $false
foreach ($src in $sources) {
$basename = [System.IO.Path]::GetFileNameWithoutExtension($src)
$dir = split-path -Path $src -Parent
$gen_dir = join-path $dir 'gen'
$out_txt = join-path $gen_dir "$basename.annotations.txt"
$out_err = join-path $gen_dir "$basename.errors.h"
if (-not (test-path $out_txt) -or -not (test-path $out_err)) { $any_stale = $true; break }
$src_mtime = (get-item $src).LastWriteTimeUtc
$out_txt_mtime = (get-item $out_txt).LastWriteTimeUtc
$out_err_mtime = (get-item $out_err).LastWriteTimeUtc
$out_mtime = if ($out_txt_mtime -gt $out_err_mtime) { $out_txt_mtime } else { $out_err_mtime }
$meta_mtime = (get-item $metadata).LastWriteTimeUtc
if (($src_mtime -gt $out_mtime) -or ($meta_mtime -gt $out_mtime)) {
$any_stale = $true
break
}
}
if (-not $any_stale) {
write-host "AtomAnnotations all $($sources.Count) source(s) up-to-date" -ForegroundColor DarkGray
return
}
write-host "AtomAnnotations $($sources.Count) source(s)" -ForegroundColor Magenta
& luajit $gen_atom_annot_script $metadata @sources
if ($LASTEXITCODE -ne 0) {
write-error "Atom annotation generation failed. Aborting."
exit 1
}
# If any source produced annotation errors, surface them now and halt the
# build. The errors.h files are also #include'd via -include below, so
# the C build would fail at preprocessing time anyway — failing here gives
# a more readable error in the build log.
$err_count = 0
foreach ($src in $sources) {
$basename = [System.IO.Path]::GetFileNameWithoutExtension($src)
$dir = split-path -Path $src -Parent
$gen_dir = join-path $dir 'gen'
$ann_txt = join-path $gen_dir "$basename.annotations.txt"
$err_h = join-path $gen_dir "$basename.errors.h"
if ((test-path $ann_txt) -and (test-path $err_h)) {
$txt = get-content $ann_txt -raw
if ($txt -match 'Errors:\s+([1-9]\d*)') {
$err_count += [int]$Matches[1]
write-warning "Annotation errors in $src — see $ann_txt"
}
}
}
if ($err_count -gt 0) {
write-error "Annotation pass failed: $err_count error(s) across $($sources.Count) source(s). Aborting."
exit 1
}
function ps1-meta {
param(
[Parameter(Mandatory=$true)][string[]]$sources,
[Parameter(Mandatory=$true)][string]$metadata,
[string]$out_root = (join-path $path_build 'gen'),
[string[]]$passes = @('--all')
)
$script = join-path $path_scripts 'ps1_meta.lua'
write-host "ps1-meta $($sources.Count) source(s), passes=$($passes -join ',')" `
-ForegroundColor Magenta
$arg_list = @($passes) + @('--metadata', $metadata) + @('--out-root', $out_root)
foreach ($s in $sources) { $arg_list += @('--source', $s) }
& luajit $script @arg_list
if ($LASTEXITCODE -ne 0) {
write-error "ps1-meta failed (exit $LASTEXITCODE). Aborting."
exit $LASTEXITCODE
}
}
function build-gte_hello {
@@ -423,8 +361,12 @@ function build-gte_hello {
$source_dirs = @($path_duffle, $path_module)
$atom_sources = Get-SourceFiles -paths $source_dirs -extensions @('.h', '.c')
generate-TapeAtomAnnotations -sources $atom_sources -metadata $path_atom_metadata
generate-TapeAtomOffsets -sources $atom_sources -metadata $path_atom_metadata
if (any-stale -sources $atom_sources -metadata $path_atom_metadata -out_root (join-path $path_build 'gen')) {
ps1-meta -sources $atom_sources -metadata $path_atom_metadata -out_root (join-path $path_build 'gen')
} else {
write-host "ps1-meta all $($atom_sources.Count) source(s) up-to-date" `
-ForegroundColor DarkGray
}
$assemble_args = @()
$assemble_args += $f_debug
scripts/duffle.lua
+22 -2
View File
@@ -370,8 +370,28 @@ function M.split_top_level_commas(body)
local function emit(end_pos)
if end_pos >= token_start then
local chunk = body:sub(token_start, end_pos)
if M.trim(chunk) ~= "" and has_real_content(chunk) then
tokens[#tokens + 1] = chunk
if M.trim(chunk) ~= "" then
if has_real_content(chunk) then
tokens[#tokens + 1] = chunk
elseif #tokens > 0 then
-- Pure comment/string chunk at top level (no
-- preceding instruction content within this chunk).
-- APPEND it to the LAST token so emit-context
-- callers (components.lua build_component_lines)
-- can convert `// trailing comment` to `/* */`
-- and emit it with the macro body. For word
-- counting, count_token_words only inspects the
-- leading ident, so a trailing comment doesn't
-- affect the count.
--
-- This is the second-half fix to commit 98e27c2:
-- the first fix correctly broke top-level comments
-- off from the NEXT statement (fixing macro-call
-- word counts); this fix preserves them on the
-- PREVIOUS statement (restoring the comments in
-- the emitted .macs.h output).
tokens[#tokens] = tokens[#tokens] .. chunk
end
end
token_start = end_pos + 1
end
scripts/passes/annotation.lua
+861 -6
View File
@@ -6,13 +6,850 @@
-- - atom_resource / atom_region / atom_group / atom_cadence / atom_async
-- - Binds_* struct declarations
-- Writes:
-- - build/gen/<basename>.errors.h (with #error directives on findings)
-- - build/gen/<basename>.annotations.txt (human-readable summary)
-- - <ctx.out_root>/<basename>.errors.h (with #error directives on findings)
-- - <ctx.out_root>/<basename>.annotations.txt (human-readable summary)
-- Ported from scripts/tape_atom_annotation_pass.lua:78-545 + 1081-1407
-- (validation only — NOT rendering, which goes to passes/report.lua).
--
-- Coding standard: tabs (1/level), EmmyLua annotations, no regex.
-- ════════════════════════════════════════════════════════════════════════════
-- Module-scope requires + package.path setup
-- ════════════════════════════════════════════════════════════════════════════
local script_path = arg and arg[0] or "?"
local last_sep = 0
for i = 1, #script_path do
local c = script_path:sub(i, i)
if c == "/" or c == "\\" then last_sep = i end
end
local script_dir = last_sep == 0 and "./" or script_path:sub(1, last_sep)
package.path = script_dir .. "../?.lua;" .. script_dir .. "../?/init.lua;" .. script_dir .. "?.lua;" .. package.path
local duffle = require("duffle")
local is_space = duffle.is_space
local is_alpha = duffle.is_alpha
local is_alnum = duffle.is_alnum
local trim = duffle.trim
local find_byte = duffle.find_byte
local read_file = duffle.read_file
local write_file = duffle.write_file
local ensure_dir = duffle.ensure_dir
local dirname = duffle.dirname
local basename_no_ext = duffle.basename_no_ext
local skip_str_or_cmt = duffle.skip_str_or_cmt
local skip_ws_and_cmt = duffle.skip_ws_and_cmt
local read_ident = duffle.read_ident
local read_parens = duffle.read_parens
local read_braces = duffle.read_braces
local scan_to_char = duffle.scan_to_char
local split_top_level_commas = duffle.split_top_level_commas
-- Domain tables (single source of truth in duffle.lua).
local WAVE_CONTEXT_REGS = duffle.WAVE_CONTEXT_REGS
local MACRO_EXPANSION = duffle.MACRO_EXPANSION
local KNOWN_PHASES = duffle.KNOWN_PHASES
local TAPE_ATOM_MACROS = duffle.TAPE_ATOM_MACROS
local ATOM_PRAGMA_KINDS = duffle.ATOM_PRAGMA_KINDS
local function valid_phase(p) return KNOWN_PHASES[p] or false end
local function is_wave_context_reg(n) return WAVE_CONTEXT_REGS[n] ~= nil end
-- ════════════════════════════════════════════════════════════════════════════
-- Hand-rolled split helpers (no :gmatch / no regex)
-- ════════════════════════════════════════════════════════════════════════════
--- Split a string at top-level commas. Used inside TAPE_ATOM_* macro
--- bodies where nested parens/braces/brackets are possible.
local function split_csv_top(s)
local tokens = {}
local i, start = 1, 1
local depth = 0
while i <= #s do
local c = s:sub(i, i)
if c == "(" or c == "{" or c == "[" then
depth = depth + 1
i = i + 1
elseif c == ")" or c == "}" or c == "]" then
depth = depth - 1
i = i + 1
elseif c == "," and depth == 0 then
tokens[#tokens + 1] = s:sub(start, i - 1)
i = i + 1
start = i
else
i = i + 1
end
end
local last = s:sub(start)
if trim(last) ~= "" then tokens[#tokens + 1] = last end
return tokens
end
--- Split a string into whitespace-separated tokens.
--- Hand-rolled (no :gmatch / no regex).
local function split_ws(s)
local tokens = {}
local i, n = 1, 1
local len = #s
while i <= len do
-- Skip whitespace.
while i <= len and is_space(s:sub(i, i)) do i = i + 1 end
if i > len then break end
local start = i
-- Take non-whitespace run.
while i <= len and not is_space(s:sub(i, i)) do i = i + 1 end
tokens[n] = s:sub(start, i - 1)
n = n + 1
end
return tokens
end
-- ════════════════════════════════════════════════════════════════════════════
-- Parse TAPE_ATOM_ANNOT(...) calls
-- ════════════════════════════════════════════════════════════════════════════
--- Extract identifier args from a parenthesized group. Returns a list
--- of {kind, value} pairs where kind is one of:
--- "ident" -- a bare identifier (e.g. phase_work)
--- "atom_reads" -- an atom_reads(...) call: value is the register list
--- "atom_writes" -- an atom_writes(...) call: value is the register list
--- "other" -- something we can't classify (preserved as text)
local function parse_atom_annot_args(inner)
local args = {}
local tokens = split_csv_top(inner)
for _, tok in ipairs(tokens) do
local s = trim(tok)
if s ~= "" then
-- Detect register-list calls: atom_reads(...) / atom_writes(...)
local regs_kind = nil
local regs_inner = nil
if s:sub(-1) == ")" then
if s:sub(1, 11) == "atom_reads(" then
regs_kind = "atom_reads"
regs_inner = s:sub(12, -2)
elseif s:sub(1, 12) == "atom_writes(" then
regs_kind = "atom_writes"
regs_inner = s:sub(13, -2)
end
end
if regs_kind then
local regs = {}
for _, r in ipairs(split_csv_top(regs_inner)) do
local trimmed = trim(r)
if trimmed ~= "" then regs[#regs + 1] = trimmed end
end
-- Resolve any phase_* / R_* alias macros
for i, r in ipairs(regs) do
if MACRO_EXPANSION[r] then regs[i] = MACRO_EXPANSION[r] end
end
args[#args + 1] = {kind = regs_kind, value = regs}
else
-- Bare identifier (e.g. phase_work)
local id = read_ident(s, 1)
if id and trim(s) == id then
local v = id
if MACRO_EXPANSION[v] then v = MACRO_EXPANSION[v] end
args[#args + 1] = {kind = "ident", value = v}
else
args[#args + 1] = {kind = "other", value = s}
end
end
end
end
return args
end
-- ════════════════════════════════════════════════════════════════════════════
-- Parse TAPE_WORDS(mac_X, N) pragma directives
-- ════════════════════════════════════════════════════════════════════════════
--- Find every TAPE_WORDS(mac_X, N) pragma in source.
--- Accepts both forms:
--- _Pragma("mac_X tape_atom words=N") (operator form)
--- #pragma mac_X tape_atom words=N (directive form)
local function find_macro_word_annotations(source)
local line_of = duffle.LineIndex(source)
local out = {}
local len = #source
local i = 1
while i <= len do
i = skip_ws_and_cmt(source, i); if i > len then break end
-- Skip preprocessor directives (lines starting with #).
if source:sub(i, i) == "#" then
local j = i
while j <= len and source:sub(j, j) ~= "\n" do j = j + 1 end
i = j + 1
else
local ident, after = read_ident(source, i)
if not ident then
i = i + 1
elseif ident == "_Pragma" then
local open = skip_ws_and_cmt(source, after)
if source:sub(open, open) == "(" then
local str, str_end = read_parens(source, open)
str = trim(str)
if str:sub(1, 1) == '"' and str:sub(-1) == '"' then
local inner = str:sub(2, -2)
local space = find_byte(inner, " ", 1)
if space then
local name = inner:sub(1, space - 1)
local rest = inner:sub(space + 1)
local eq = find_byte(rest, "=", 1)
if eq then
local key = trim(rest:sub(1, eq - 1))
local val = trim(rest:sub(eq + 1))
if key == "tape_atom words" or key == "words" then
local n = tonumber(val) or 0
out[#out + 1] = {
line = line_of(i),
name = name,
words = n,
}
end
end
end
end
i = str_end
else
i = open + 1
end
elseif ident == "pragma" then
-- Directive form: `#pragma mac_X tape_atom words=N`
local rest_start = skip_ws_and_cmt(source, after)
local j = rest_start
while j <= len and source:sub(j, j) ~= "\n" do j = j + 1 end
local line_text = trim(source:sub(rest_start, j - 1))
local tokens = split_ws(line_text)
if #tokens >= 3 and tokens[2] == "tape_atom" and tokens[3]:sub(1, 6) == "words=" then
local name = tokens[1]
local n = tonumber(tokens[3]:sub(7)) or 0
out[#out + 1] = { line = line_of(i), name = name, words = n }
elseif #tokens >= 2 and tokens[2]:sub(1, 6) == "words=" then
local name = tokens[1]
local n = tonumber(tokens[2]:sub(7)) or 0
out[#out + 1] = { line = line_of(i), name = name, words = n }
end
i = j
else
i = after
end
end
end
return out
end
-- ════════════════════════════════════════════════════════════════════════════
-- Parse `atom_<...>` Pragma / _Pragma annotations
-- ════════════════════════════════════════════════════════════════════════════
local ATOM_ATTR_MACROS = {
["atom_resource"] = "resource",
["atom_region"] = "region",
["atom_group"] = "group",
["atom_cadence"] = "cadence",
["atom_async"] = "async",
}
--- Parse macro form: `atom_<key>(atom_name, value, ...)`.
--- Returns (true, entry, str_end) on success, (false) on no match.
local function try_parse_atom_attr_macro(source, i, line_of)
local ident, after = read_ident(source, i)
if not ident then return false end
local key = ATOM_ATTR_MACROS[ident]
if not key then return false end
local open = skip_ws_and_cmt(source, after)
if source:sub(open, open) ~= "(" then return false end
local body, body_end = read_parens(source, open)
local first, after_name = read_ident(body, 1)
if not first then return false end
local j = after_name
while j <= #body and is_space(body:sub(j, j)) do j = j + 1 end
if body:sub(j, j) ~= "," then return false end
j = j + 1
while j <= #body and is_space(body:sub(j, j)) do j = j + 1 end
local value
if body:sub(j, j) == '"' then
local k = j + 1
while k <= #body do
local c = body:sub(k, k)
if c == "\\" then
k = k + 2
elseif c == '"' then
break
else
k = k + 1
end
end
if body:sub(k, k) ~= '"' then return false end
value = body:sub(j + 1, k - 1)
else
local id2, after_id = read_ident(body, j)
if not id2 then return false end
value = id2
if MACRO_EXPANSION[value] then value = MACRO_EXPANSION[value] end
end
return true, {
line = line_of(i),
name = first,
attrs = { [key] = value },
}, body_end
end
local function find_atom_pragmas(source)
local line_of = duffle.LineIndex(source)
local out = {}
local len = #source
local i = 1
while i <= len do
i = skip_ws_and_cmt(source, i); if i > len then break end
if source:sub(i, i) == "#" then
local j = i
while j <= len and source:sub(j, j) ~= "\n" do j = j + 1 end
i = j + 1
else
local got, entry, next_i = try_parse_atom_attr_macro(source, i, line_of)
if got then
out[#out + 1] = entry
i = next_i
else
local ident, after = read_ident(source, i)
if not ident then
i = i + 1
elseif ident == "_Pragma" then
local open = skip_ws_and_cmt(source, after)
if source:sub(open, open) == "(" then
local str, str_end = read_parens(source, open)
str = trim(str)
if str:sub(1, 1) == '"' and str:sub(-1) == '"' then
local inner = str:sub(2, -2)
local sp1 = find_byte(inner, " ", 1)
if sp1 and trim(inner:sub(1, sp1 - 1)) == "atom" then
local rest = trim(inner:sub(sp1 + 1))
local sp2 = find_byte(rest, " ", 1)
if sp2 then
local name = trim(rest:sub(1, sp2 - 1))
local attrs_str = trim(rest:sub(sp2 + 1))
local attrs = {}
local got_any = false
for _, pair in ipairs(split_ws(attrs_str)) do
local eq = find_byte(pair, "=", 1)
if eq then
local k = trim(pair:sub(1, eq - 1))
local v = trim(pair:sub(eq + 1))
if MACRO_EXPANSION[v] then v = MACRO_EXPANSION[v] end
attrs[k] = v
got_any = true
end
end
if got_any then
out[#out + 1] = {
line = line_of(i),
name = name,
attrs = attrs,
}
end
end
end
end
i = str_end
else
i = open + 1
end
else
i = after
end
end
end
end
return out
end
-- ════════════════════════════════════════════════════════════════════════════
-- Parse `typedef Struct_(Binds_X) { ... };` declarations
-- ════════════════════════════════════════════════════════════════════════════
--- Find every Binds_* struct declaration.
--- Returns a list of {line, name, fields = {{name, byte_offset}, ...}}.
local function find_binds_structs(source)
local line_of = duffle.LineIndex(source)
local out = {}
local len = #source
local i = 1
while i <= len do
i = skip_ws_and_cmt(source, i); if i > len then break end
if source:sub(i, i) == "#" then
local j = i
while j <= len and source:sub(j, j) ~= "\n" do j = j + 1 end
i = j + 1
else
local ident, after = read_ident(source, i)
if not ident then
i = i + 1
elseif ident == "typedef" then
local j = skip_ws_and_cmt(source, after)
local id2, after2 = read_ident(source, j)
if id2 ~= "Struct_" then
i = after2 or (j + 1)
elseif id2 == "Struct_" then
local open = skip_ws_and_cmt(source, after2)
if source:sub(open, open) == "(" then
local inner, after_paren = read_parens(source, open)
local name = trim(inner)
local brace = scan_to_char(source, "{", after_paren)
if brace then
local body, after_brace = read_braces(source, brace)
local fields = {}
local byte_off = 0
local k = 1
while k <= #body do
k = skip_ws_and_cmt(body, k); if k > #body then break end
local tid, tafter = read_ident(body, k)
if not tid then
k = k + 1
elseif tid == "U4" then
local fid, fafter = read_ident(body, skip_ws_and_cmt(body, tafter))
if fid then
fields[#fields + 1] = { name = fid, offset = byte_off }
byte_off = byte_off + 4
end
k = fafter or tafter + 1
else
k = tafter + 1
end
end
-- Only emit Binds_* structs.
if name:sub(1, 6) == "Binds_" then
out[#out + 1] = {
line = line_of(i),
name = name,
fields = fields,
bytes = byte_off,
}
end
i = after_brace
else
i = open + 1
end
else
i = open + 1
end
else
i = after2 or (j + 1)
end
else
i = after
end
end
end
return out
end
-- ════════════════════════════════════════════════════════════════════════════
-- Find every MipsAtom_(name) { ... } declaration in source
-- ════════════════════════════════════════════════════════════════════════════
local function find_atom_names(source)
local line_of = duffle.LineIndex(source)
local out = {}
local len = #source
local i = 1
while i <= len do
i = skip_ws_and_cmt(source, i); if i > len then break end
local ident, after = read_ident(source, i)
if not ident then
i = i + 1
elseif ident == "MipsAtom_" then
local open = skip_ws_and_cmt(source, after)
if source:sub(open, open) == "(" then
local inner, after_paren = read_parens(source, open)
local a = 1
while a <= #inner and is_space(inner:sub(a, a)) do a = a + 1 end
local b = a
while b <= #inner and is_alnum(inner:sub(b, b)) do b = b + 1 end
local name = inner:sub(a, b - 1)
if name ~= "" then
out[#out + 1] = { line = line_of(i), name = name }
end
local brace = scan_to_char(source, "{", after_paren)
if brace then
local _, after_brace = read_braces(source, brace)
i = after_brace
else
i = open + 1
end
else
i = open + 1
end
else
i = after
end
end
return out
end
-- ════════════════════════════════════════════════════════════════════════════
-- Find atom annotations (atom_annot / atom_init / atom_setup / atom_commit
-- / atom_bind / atom_terminate)
-- ════════════════════════════════════════════════════════════════════════════
local function find_atom_annotations(source)
local line_of = duffle.LineIndex(source)
local annots = {}
local len = #source
local i = 1
while i <= len do
i = skip_ws_and_cmt(source, i); if i > len then break end
-- Skip preprocessor directives (lines starting with #).
-- Without this guard, `#define atom_init(name) ...` macro
-- definitions get misinterpreted as annotation calls with
-- the literal placeholder "name" as the atom name.
if source:sub(i, i) == "#" then
local j = i
while j <= len and source:sub(j, j) ~= "\n" do j = j + 1 end
i = j + 1
else
local ident, after = read_ident(source, i)
if not ident then
i = i + 1
elseif TAPE_ATOM_MACROS[ident] then
local open = skip_ws_and_cmt(source, after)
if source:sub(open, open) == "(" then
local inner, after_paren = read_parens(source, open)
local args = parse_atom_annot_args(inner)
local macro_def = TAPE_ATOM_MACROS[ident]
if #args < 1 then
annots[#annots + 1] = {
line = line_of(i),
macro = ident,
kind = macro_def.kind,
error = "missing atom name (first arg)",
}
else
local name = args[1].value
local entry = {
line = line_of(i),
macro = ident,
name = name,
kind = macro_def.kind,
binds = nil,
phase = nil,
reads = {},
writes = {},
errors = {},
}
local function is_regs(a)
return a and (a.kind == "atom_reads" or a.kind == "atom_writes" or a.kind == "regs")
end
if macro_def.binds then
if #args >= 2 and args[2].kind == "ident" then
entry.binds = args[2].value
end
if #args >= 3 and is_regs(args[3]) then
entry.writes = args[3].value
end
elseif ident == "atom_init" or ident == "atom_terminate" then
-- (name) only
elseif ident == "atom_setup" then
if #args >= 2 and is_regs(args[2]) then
entry.reads = args[2].value
end
elseif ident == "atom_commit" then
if #args >= 2 and is_regs(args[2]) then
entry.reads = args[2].value
end
elseif ident == "atom_annot" then
if #args >= 2 and args[2].kind == "ident" then
entry.phase = MACRO_EXPANSION[args[2].value] or args[2].value
end
if #args >= 3 and is_regs(args[3]) then
if args[3].kind == "atom_writes" then
entry.errors[#entry.errors + 1] =
"reads slot has atom_writes — swap order?"
end
entry.reads = args[3].value
end
if #args >= 4 and is_regs(args[4]) then
if args[4].kind == "atom_reads" then
entry.errors[#entry.errors + 1] =
"writes slot has atom_reads — swap order?"
end
entry.writes = args[4].value
end
end
annots[#annots + 1] = entry
end
i = after_paren
else
i = open + 1
end
else
i = after
end
end
end
return annots
end
-- ════════════════════════════════════════════════════════════════════════════
-- Validation (ported from tape_atom_annotation_pass.lua:1193-1405)
-- ════════════════════════════════════════════════════════════════════════════
local function validate(ctx, src)
local source = src.text
local annots = find_atom_annotations(source)
local macros = find_macro_word_annotations(source)
local pragmas = find_atom_pragmas(source)
local binds = find_binds_structs(source)
local atoms = find_atom_names(source)
local atom_index = {}
for _, a in ipairs(atoms) do atom_index[a.name] = a end
local binds_index = {}
for _, b in ipairs(binds) do binds_index[b.name] = b end
local errors = {}
local warnings = {}
local info = {}
-- 1. Every annotated atom must exist as a real MipsAtom_ declaration.
for _, a in ipairs(annots) do
if a.error then
errors[#errors + 1] = {line = a.line, msg = a.error}
elseif not atom_index[a.name] then
errors[#errors + 1] = {
line = a.line,
msg = string.format("annotation for '%s' has no matching MipsAtom_(%s) { ... }", a.name, a.name),
}
end
if a.errors then
for _, msg in ipairs(a.errors) do
errors[#errors + 1] = {line = a.line, msg = string.format("'%s': %s", a.name, msg)}
end
end
end
-- 2. Every atom must have exactly one annotation (no orphans, no duplicates).
local count_per_atom = {}
for _, a in ipairs(annots) do
if a.name and not a.error then
count_per_atom[a.name] = (count_per_atom[a.name] or 0) + 1
end
end
for _, atom in ipairs(atoms) do
local n = count_per_atom[atom.name] or 0
if n == 0 then
warnings[#warnings + 1] = {
line = atom.line,
msg = string.format("MipsAtom_(%s) has no TAPE_ATOM_* annotation", atom.name),
}
elseif n > 1 then
errors[#errors + 1] = {
line = atom.line,
msg = string.format("MipsAtom_(%s) has %d annotations (expected 1)", atom.name, n),
}
end
end
-- 3. Phase validity.
for _, a in ipairs(annots) do
if a.name and not a.error and a.phase and not valid_phase(a.phase) then
errors[#errors + 1] = {
line = a.line,
msg = string.format("'%s' has unknown phase '%s' (expected one of init/bind/setup/work/commit/terminate)", a.name, a.phase),
}
end
end
-- 4. BIND atoms must reference a real Binds_* struct.
for _, a in ipairs(annots) do
if a.binds then
if not binds_index[a.binds] then
errors[#errors + 1] = {
line = a.line,
msg = string.format("'%s' binds '%s' but no Struct_(%s) { ... } declaration found", a.name, a.binds, a.binds),
}
end
end
end
-- 5. BIND writes must be wave-context registers that match Binds_ fields.
for _, a in ipairs(annots) do
if a.binds and binds_index[a.binds] then
local bs = binds_index[a.binds]
local field_names = {}
for _, f in ipairs(bs.fields) do field_names[f.name] = true end
for _, f in ipairs(bs.fields) do
local candidate = "R_" .. f.name
if not is_wave_context_reg(candidate) then
warnings[#warnings + 1] = {
line = bs.line,
msg = string.format("%s field '%s' doesn't match a known wave-context register (candidate '%s')", a.binds, f.name, candidate),
}
end
end
for _, w in ipairs(a.writes) do
if not is_wave_context_reg(w) then
warnings[#warnings + 1] = {
line = a.line,
msg = string.format("%s writes '%s' which is not a known wave-context register", a.name, w),
}
end
end
end
end
-- 6. WORK reads should be a subset of BIND writes (the wave contract).
for _, a in ipairs(annots) do
if a.kind == "work" then
for _, r in ipairs(a.reads) do
if not is_wave_context_reg(r) and r ~= "R_TapePtr" then
warnings[#warnings + 1] = {
line = a.line,
msg = string.format("work atom '%s' reads '%s' which is not a known wave-context register", a.name, r),
}
end
end
end
end
-- 7. TAPE_WORDS(mac_X, N) ↔ WORD_COUNT(mac_X, N) drift.
for _, m in ipairs(macros) do
local declared = ctx.shared.word_counts[m.name]
if not declared then
errors[#errors + 1] = {
line = m.line,
msg = string.format("TAPE_WORDS(%s, %d) but '%s' is not in metadata.h", m.name, m.words, m.name),
}
elseif declared ~= m.words then
errors[#errors + 1] = {
line = m.line,
msg = string.format("DRIFT: TAPE_WORDS(%s, %d) but metadata.h declares WORD_COUNT(%s, %d)", m.name, m.words, m.name, declared),
}
else
info[#info + 1] = {
line = m.line,
msg = string.format("OK: %s = %d words", m.name, m.words),
}
end
end
-- 8. atom_<...> _Pragma validation: resource/region/group/cadence/async
for _, p in ipairs(pragmas) do
if not atom_index[p.name] then
errors[#errors + 1] = {
line = p.line,
msg = string.format("pragma references unknown atom '%s'", p.name),
}
end
for k, v in pairs(p.attrs) do
local spec = ATOM_PRAGMA_KINDS[k]
if not spec then
errors[#errors + 1] = {
line = p.line,
msg = string.format("'%s' has unknown pragma key '%s' (allowed: resource/region/group/cadence/async)", p.name, k),
}
elseif spec.allowed and not spec.allowed[v] then
local allowed = {}
for kk in pairs(spec.allowed) do allowed[#allowed + 1] = kk end
table.sort(allowed)
local allowed_str = table.concat(allowed, ", ")
errors[#errors + 1] = {
line = p.line,
msg = string.format("'%s' pragma %s=%s but '%s' is not allowed (allowed: %s)", p.name, k, v, v, allowed_str),
}
end
end
end
-- 9. CADENCE_ONDEMAND requires async=true.
for _, p in ipairs(pragmas) do
if p.attrs.cadence == "ondemand" and p.attrs.async ~= "true" then
errors[#errors + 1] = {
line = p.line,
msg = string.format("'%s' is CADENCE_ONDEMAND but does not declare atom_async(true)", p.name),
}
end
end
-- 10. Information summary.
info[#info + 1] = {
line = 0,
msg = string.format("scanned: %d atom(s), %d annotation(s), %d pragma(s), %d macro-word-decl(s), %d binds struct(s)",
#atoms, #annots, #pragmas, #macros, #binds),
}
return {
atoms = atoms,
annots = annots,
macros = macros,
pragmas = pragmas,
binds = binds,
errors = errors,
warnings = warnings,
info = info,
}
end
-- ════════════════════════════════════════════════════════════════════════════
-- Per-source output: errors.h + annotations.txt
-- ════════════════════════════════════════════════════════════════════════════
--- Render <basename>.errors.h with #error directives for any structural issues.
local function emit_errors_h(ctx, src, result)
local out_path = ctx.out_root .. "/" .. src.basename .. ".errors.h"
local lines = {
"// Auto-generated by ps1_meta.lua (passes/annotation.lua) — DO NOT EDIT",
"#pragma once",
"",
}
for _, e in ipairs(result.errors) do
lines[#lines + 1] = string.format('#error "annotation: %s (line %d)"', e.msg, e.line)
end
if #result.errors == 0 then
lines[#lines + 1] = "// annotation pass OK"
end
if ctx.dry_run then return nil end
ensure_dir(ctx.out_root)
write_file(out_path, table.concat(lines, "\n") .. "\n")
return out_path
end
--- Render <basename>.annotations.txt human-readable summary.
--- Implementation lives in passes/report.lua (extracted to keep this
--- file focused on validation). We delegate via a callback set on ctx.
local function emit_annotations_txt(ctx, src, result)
-- The annotation pass emits a structured result; the report pass
-- renders it. To avoid a circular dep, the M.run below packs the
-- result into ctx.upstream.annotation for the report pass to
-- consume via ctx.flags._annot_results.
ctx.flags = ctx.flags or {}
ctx.flags._annot_results = ctx.flags._annot_results or {}
ctx.flags._annot_results[#ctx.flags._annot_results + 1] = {
source = src,
result = result,
}
return nil -- annotations.txt is written by report.lua
end
-- ════════════════════════════════════════════════════════════════════════════
-- M.run — orchestrator entry
-- ════════════════════════════════════════════════════════════════════════════
--- @class M
local M = {}
@@ -20,10 +857,28 @@ local M = {}
--- @param ctx PassCtx
--- @return PassResult
function M.run(ctx)
error("passes.annotation.run: implement by porting " ..
"find_atom_names, find_atom_annotations, find_macro_word_annotations, " ..
"find_atom_pragmas, find_binds_structs, validate from " ..
"tape_atom_annotation_pass.lua:78-1407")
local outputs = {}
local errors = {}
local warnings = {}
for _, src in ipairs(ctx.sources) do
local result = validate(ctx, src)
local err_path = emit_errors_h(ctx, src, result)
if err_path then
table.insert(outputs, { errors_h = err_path })
end
-- Stash result for report pass.
emit_annotations_txt(ctx, src, result)
for _, e in ipairs(result.errors) do
errors[#errors + 1] = { line = e.line, msg = e.msg }
end
for _, w in ipairs(result.warnings) do
warnings[#warnings + 1] = { line = w.line, msg = w.msg }
end
end
return { outputs = outputs, errors = errors, warnings = warnings }
end
return M
scripts/passes/components.lua
+602 -20
View File
@@ -1,39 +1,621 @@
-- passes/components.lua
--
-- Generate <module>/gen/<basename>.macs.h from MipsAtomComp_ declarations
-- in source files. Ported from tape_atom_annotation_pass.lua:773-1079
-- in source files. Ported from tape_atom_annotation_pass.lua:604-1079
-- (find_component_atoms, preceding_comment_block, extract_arg_names,
-- convert_line_comments_to_block, compute_component_word_count,
-- emit_component_macros_h).
--
-- Coding standard: tabs (1/level), EmmyLua annotations, no regex.
--- @class Component
--- @field name string
--- @field body string
--- @field args string|nil
--- @field line integer
--- @field comment string|nil
--- @class M
-- ════════════════════════════════════════════════════════════════════════════
-- Module-scope requires + package.path setup
-- ════════════════════════════════════════════════════════════════════════════
local script_path = arg and arg[0] or "?"
local last_sep = 0
for i = 1, #script_path do
local c = script_path:sub(i, i)
if c == "/" or c == "\\" then last_sep = i end
end
local script_dir = last_sep == 0 and "./" or script_path:sub(1, last_sep)
package.path = script_dir .. "../?.lua;" .. script_dir .. "../?/init.lua;" .. script_dir .. "?.lua;" .. package.path
local duffle = require("duffle")
local trim = duffle.trim
local read_ident = duffle.read_ident
local is_space = duffle.is_space
local is_alpha = duffle.is_alpha
local is_alnum = duffle.is_alnum
local skip_ws_and_cmt = duffle.skip_ws_and_cmt
local skip_str_or_cmt = duffle.skip_str_or_cmt
local split_top_level_commas = duffle.split_top_level_commas
local ensure_dir = duffle.ensure_dir
local basename_no_ext = duffle.basename_no_ext
local dirname = duffle.dirname
local read_parens = duffle.read_parens
local read_braces = duffle.read_braces
local scan_to_char = duffle.scan_to_char
local word_count_eval = require("word_count_eval")
local count_body_words = word_count_eval.count_body_words
local M = {}
-- ════════════════════════════════════════════════════════════════════════════
-- Local helpers
-- ════════════════════════════════════════════════════════════════════════════
-- Write content to disk in binary mode so LF line endings are preserved on
-- Windows (text mode would convert LF -> CRLF, breaking byte-identical diffs
-- against git-tracked gen/*.macs.h files which are stored as LF).
local function write_file_lf(path, content)
local f = io.open(path, "wb")
if not f then error("Cannot write " .. path) end
f:write(content)
f:close()
end
-- Convert a (possibly relative) path to an absolute Windows path. The
-- pre-rework output's "// Source:" comment line used the absolute path
-- (e.g. "C:\projects\Pikuma\ps1\code\duffle\lottes_tape.h"); if we want
-- byte-identical output, we must normalize relative -> absolute before
-- emitting that comment.
local function to_absolute_path(path)
if #path >= 2 and path:sub(2, 2) == ":" then
-- Already absolute; normalize slashes for consistency.
return (path:gsub("/", "\\"))
end
local p = io.popen("cd")
if not p then return path end
local cwd = p:read("*l")
p:close()
if not cwd then return path end
-- Normalize forward slashes to backslashes (Windows convention) on
-- both the cwd AND the relative path tail, so the join is uniform.
cwd = cwd:gsub("/", "\\")
local tail = (path:gsub("/", "\\"))
return cwd .. "\\" .. tail
end
-- ════════════════════════════════════════════════════════════════════════════
-- Ported helpers (verbatim from tape_atom_annotation_pass.lua:604-1079)
-- ════════════════════════════════════════════════════════════════════════════
-- ============================================================
-- Find the args of the function declaration that immediately precedes
-- a MipsAtomComp_Proc_ invocation of the given name. Returns the
-- args string (e.g., "U4 off, U4 code, U1 r, U1 g, U1 b") or nil
-- if no function declaration is found.
--
-- Convention: function form is
-- FI_ MipsAtom ac_X(args) MipsAtomComp_Proc_(ac_X, { body })
-- We find the LAST occurrence of "ac_X(" before before_pos and
-- extract the args from inside the parens.
--
-- No regex (per the no_regex constraint). Uses string.find with
-- plain mode (4th arg = true) to find the name + open paren.
-- ============================================================
--- @param source string
--- @param name string
--- @param before_pos integer
--- @return string|nil
local function find_function_args_for(source, name, before_pos)
local search = source:sub(1, before_pos)
local name_paren = name .. "("
local last_idx = nil
local p = 1
while true do
local s = search:find(name_paren, p, true) -- plain (no regex)
if not s then break end
last_idx = s
p = s + #name_paren
end
if not last_idx then return nil end
-- Verify the preceding context ends with "MipsAtom" (with
-- possible qualifiers between). Check the last word is
-- "MipsAtom" (or the trimmed before ends with that token).
local before = search:sub(1, last_idx - 1)
local trimmed = duffle.trim(before)
if trimmed:sub(-#"MipsAtom") ~= "MipsAtom" then
-- Preceding context is not a function declaration.
-- This shouldn't happen with the convention, but guard anyway.
return nil
end
local open_paren = last_idx + #name -- position of "("
local inner = read_parens(source, open_paren)
if not inner then return nil end
return inner
end
-- ============================================================
-- Find the contiguous comment block immediately preceding `pos` in
-- `source`. Returns the comment text (with the `/* */` or `//` markers
-- preserved) or an empty string if no comment is adjacent.
-- Used to copy signature comments from the source declaration
-- (`MipsAtomComp_` / `MipsAtomComp_Proc_` / function decl) over to the generated
-- `mac_X` macro, so LSP/IntelliSense displays the args doc.
-- No regex (per the no_regex constraint).
-- ============================================================
--- @param source string
--- @param pos integer
--- @return string
local function preceding_comment_block(source, pos)
local i = pos
local pieces = {}
while true do
-- Skip whitespace
local j = i - 1
while j > 0 do
local c = source:sub(j, j)
if c == " " or c == "\t" or c == "\n" or c == "\r" then
j = j - 1
else
break
end
end
if j == 0 then break end
-- Check for /* ... */ ending at j
if j >= 2 and source:sub(j-1, j) == "*/" then
local s = source:sub(1, j - 1)
local last_open = nil
for k = #s - 1, 1, -1 do
if s:sub(k, k+1) == "/*" then
last_open = k
break
end
end
if last_open then
-- Include the leading whitespace+indentation before /*
local block_start = last_open
while block_start > 1 do
local c = source:sub(block_start - 1, block_start - 1)
if c == " " or c == "\t" then
block_start = block_start - 1
else
break
end
end
table.insert(pieces, 1, source:sub(block_start, j))
i = block_start
else
break
end
-- Check for // comment ending at j (j is at end of line, j-1 is \n)
elseif j >= 1 and (source:sub(j, j) == "\n" or source:sub(j, j) == "\r") then
-- Walk back to the start of the line
local line_start = j
while line_start > 1 and source:sub(line_start-1, line_start-1) ~= "\n" do
line_start = line_start - 1
end
local line = source:sub(line_start, j)
if line:sub(1, 2) == "//" then
table.insert(pieces, 1, line)
i = line_start - 1
else
break
end
else
break
end
end
if #pieces == 0 then return "" end
return table.concat(pieces, "\n")
end
-- ============================================================
-- Extract just the parameter NAMES from a function-args string
-- (stripping type annotations). E.g.,
-- "U4 off, U4 code, U1 r, U1 g, U1 b" -> {"off", "code", "r", "g", "b"}
-- "U4 *ptr" -> {"ptr"}
-- "" -> nil
-- No regex — uses duffle.is_alnum + plain string ops.
-- ============================================================
--- @param args_str string|nil
--- @return string[]|nil
local function extract_arg_names(args_str)
if not args_str or args_str == "" then return nil end
local names = {}
local tokens = duffle.split_top_level_commas(args_str)
for _, tok in ipairs(tokens) do
local trimmed = duffle.trim(tok)
if trimmed ~= "" then
-- Walk backwards from end of trimmed arg, skipping
-- trailing whitespace / asterisks / brackets.
local i = #trimmed
while i > 0 do
local c = trimmed:sub(i, i)
if c == " " or c == "\t" or c == "*" or c == "]" or c == "[" then
i = i - 1
else
break
end
end
-- Now find the end of the last identifier (the param name).
local j = i
while j > 0 do
local c = trimmed:sub(j, j)
if duffle.is_alnum(c) or c == "_" then
j = j - 1
else
break
end
end
local name = trimmed:sub(j + 1, i)
if name ~= "" then
names[#names + 1] = name
end
end
end
if #names == 0 then return nil end
return names
end
-- ============================================================
-- Find every MipsAtomComp_(ac_<X>) { body } declaration in source.
-- Supports BOTH the bare form and the function form:
-- Bare: MipsAtomComp_(ac_X) { body }
-- Function: MipsAtomComp_Proc_(ac_X, { body }) (with a preceding
-- "FI_ MipsAtom ac_X(args)" function declaration)
-- Returns: {line, name, body, args} where args is the function-args
-- string (or nil for the bare form).
-- ============================================================
-- WORD_COUNT entry in gen/<dir_basename>.components.h)
-- ============================================================
--- @param source string
--- @return Component[]
local function find_component_atoms(source)
local line_of = duffle.LineIndex(source)
local out = {}
local len = #source
local i = 1
while i <= len do
i = skip_ws_and_cmt(source, i); if i > len then break end
local ident, after = read_ident(source, i)
if not ident then
i = i + 1
elseif ident == "MipsAtomComp_Proc_"
or ident == "MipsAtomComp_" then
local open = skip_ws_and_cmt(source, after)
if source:sub(open, open) == "(" then
local inner, after_paren = read_parens(source, open)
-- Parse args: 1 arg = bare form, 2 args = function form
local tokens = duffle.split_top_level_commas(inner)
local name, body = nil, nil
if #tokens == 1 then
name = duffle.trim(tokens[1])
elseif #tokens == 2 then
name = duffle.trim(tokens[1])
local body_raw = duffle.trim(tokens[2])
-- Strip leading { and trailing } if present
if #body_raw >= 2
and body_raw:sub(1, 1) == "{"
and body_raw:sub(-1) == "}" then
body = duffle.trim(body_raw:sub(2, -2))
else
body = body_raw
end
end
if name and name:sub(1, 3) == "ac_" then
-- Find the function args (preceding function decl).
-- For the bare form this returns nil (no function).
local args = find_function_args_for(source, name, open)
-- Capture the preceding comment block (signature doc).
-- Walk back from `i` (position of the identifier start)
-- so the walk-back goes through whitespace+comment and
-- stops AT the comment (not at the identifier chars).
local comment = preceding_comment_block(source, i)
if body == nil then
-- Bare form: body is the brace block AFTER the parens.
local brace = scan_to_char(source, "{", after_paren)
if brace then
local body_content, after_brace = read_braces(source, brace)
out[#out + 1] = {
line = line_of(i),
name = name:sub(4), -- strip "ac_" prefix
body = body_content,
args = args,
comment = comment,
}
i = after_brace
else
i = open + 1
end
else
-- Function form: body is the second arg (already extracted).
out[#out + 1] = {
line = line_of(i),
name = name:sub(4), -- strip "ac_" prefix
body = body,
args = args,
comment = comment,
}
i = after_paren
end
else
i = open + 1
end
else
i = open + 1
end
else
i = after
end
end
return out
end
-- ============================================================
-- Emit a per-directory generated header with mac_X(...) macros
-- derived from MipsAtomComp_ declarations + auto word-counts.
-- Output: <source_dir>/gen/<dir_basename>.macs.h
-- ============================================================
-- Convert `//` line comments to `/* */` block comments in a token.
-- C macros use `\` line-continuations; a `//` comment before `\` would
-- consume the continuation, breaking the macro. We convert `//` to
-- `/* */` so the multi-line macro structure is preserved.
-- Skips `//` sequences that are inside string or character literals
-- (a rough heuristic — sufficient for component bodies which don't
-- have those constructs).
--- @param s string
--- @return string
local function convert_line_comments_to_block(s)
local result = s
local i = 1
while i <= #result do
local c = result:byte(i)
if c == 47 and i + 1 <= #result and result:byte(i + 1) == 47 then
-- Found `//`. Find end of line.
local eol = i
while eol <= #result and result:byte(eol) ~= 10 do
eol = eol + 1
end
local before = result:sub(1, i - 1)
local comment = result:sub(i + 2, eol - 1) -- skip the `//`
local after
if eol <= #result and result:byte(eol) == 10 then
after = " */" .. result:sub(eol) -- keep the newline
else
after = " */"
end
result = before .. "/*" .. comment .. after
i = #before + 2 + #comment + 3 -- skip past converted comment
else
i = i + 1
end
end
return result
end
-- Compute the word count of a component body, accounting for
-- macro expansion. Each comma-separated entry in the body is a
-- "slot" that contributes its own word count. For most entries
-- (regular MIPS instructions) the count is 1. For `mac_Y(...)`
-- calls, the count is the word count of mac_Y (recursive lookup).
-- For encoding macros with a known multi-word count (e.g.
-- `mask_upper` = 2), the count is taken from `word_counts`.
--
-- ADAPTATION: the original used an inline recursive `rec()` that
-- walked the components list + word_counts table. The new code
-- delegates to word_count_eval.count_body_words(body, wc), which
-- is equivalent when `wc` is pre-populated with component macros.
-- The word-counts pass (which runs before components) scans all
-- existing gen/*.macs.h files, so on any build with checked-in
-- gen/ files, wc already contains every mac_X entry.
--- @param c Component
--- @param components Component[] -- (kept for API compatibility, unused)
--- @param wc table<string, integer>
--- @return integer
local function compute_component_word_count(c, components, wc)
return count_body_words(c.body, wc)
end
-- ============================================================
-- Per-component emit logic. Returns the body of lines for one
-- component (signature comment, #define mac_X(...) line with
-- backslash-continued tokens, then WORD_COUNT(mac_X, N) entry).
--
-- Extracted from emit_component_macros_h so M.run can call it
-- once per component AND extend ctx.shared.word_counts.
-- ============================================================
--- @param c Component
--- @param components Component[]
--- @param wc table<string, integer>
--- @return string[] -- list of lines for this component
local function build_component_lines(c, components, wc)
local lines = {}
-- Emit the signature comment (if any) above the macro.
if c.comment and c.comment ~= "" then
for line in (c.comment .. "\n"):gmatch("([^\n]*)\n") do
lines[#lines + 1] = line
end
end
-- Split body by top-level commas; filter empty tokens.
local tokens = {}
for _, t in ipairs(split_top_level_commas(c.body)) do
local trimmed = trim(t)
if trimmed ~= "" then tokens[#tokens + 1] = trimmed end
end
-- Determine the macro signature: with function args (function
-- form) or variadic-ignored (bare form).
local arg_names = extract_arg_names(c.args)
local sig
if arg_names and #arg_names > 0 then
sig = table.concat(arg_names, ", ")
else
sig = "..."
end
-- Compute the word count of the component body, accounting for
-- macro expansion. Each comma-separated entry in the body is one
-- "instruction slot", but a `mac_Y(...)` call expands to Y's
-- word count. The offset_gen and the metadata's WORD_COUNT table
-- both use this resolved count.
--
-- We compute it once per component (not per token) and emit
-- the value in the WORD_COUNT entry. The lookup table `counts_by_name`
-- is built from the components list (which find_component_atoms
-- already populated) and falls back to 1 for non-mac tokens.
local counts_by_name = {}
for _, cc in ipairs(components) do
local cc_count = compute_component_word_count(cc, components, wc)
counts_by_name["mac_" .. cc.name] = cc_count
end
local n = compute_component_word_count(c, components, wc)
if n > 0 then
-- Convert `//` line comments to `/* */` block comments in each
-- token. C macros use `\` line-continuations; if a `//` comment
-- appears before a `\`, the rest of the line (including the
-- continuation) is consumed by the `//`, breaking the macro.
-- Converting to `/* */` preserves the macro structure.
for j = 1, #tokens do
tokens[j] = convert_line_comments_to_block(tokens[j])
end
-- Emit the mac_<X>(<sig>) macro
lines[#lines + 1] = "#define mac_" .. c.name .. "(" .. sig .. ") \\"
lines[#lines + 1] = "\t" .. tokens[1] .. " \\"
for j = 2, #tokens do
lines[#lines + 1] = ",\t" .. tokens[j] .. " \\"
end
-- Strip the trailing line-continuation on the last body line.
-- The last 2 chars are always " \" (space + backslash).
-- No regex — just trim with string.sub.
local last = lines[#lines]
if last:sub(-2) == " \\" then
lines[#lines] = last:sub(1, -3)
end
end
-- Emit the WORD_COUNT(mac_<X>, N) entry.
lines[#lines + 1] = "WORD_COUNT(mac_" .. c.name .. ", " .. n .. ")"
lines[#lines + 1] = ""
return lines
end
-- ============================================================
-- Emit a per-source .macs.h header with the mac_X macros +
-- WORD_COUNT entries. Writes in BINARY mode so LF line endings
-- are preserved (the git blob is LF; Windows text-mode would
-- emit CRLF and break the byte-identical diff).
--
-- Honors ctx.dry_run: prints the intended path but does not
-- write the file.
-- ============================================================
--- @param ctx PassCtx
--- @param src SourceFile
--- @param components Component[]
--- @return string|nil -- path to the written file (nil on dry-run)
local function emit_component_macros_h(ctx, src, components)
if #components == 0 then return nil end
-- Output path: <src.dir>/gen/<src.dir's basename>.macs.h
-- The pre-rework convention uses the *directory* basename (not
-- the source file basename) — e.g. `code/duffle/lottes_tape.h`
-- produces `code/duffle/gen/duffle.macs.h`. This matches what
-- the C codebase #includes.
local out_dir = src.dir .. "/gen"
local out_path = out_dir .. "/" .. basename_no_ext(src.dir) .. ".macs.h"
local lines = {
-- #pragma once wrapped in #ifdef INTELLISENSE_DIRECTIVES, matching
-- the convention in lottes_tape.h. The build does manual unity
-- includes (the user controls include order), so the pragma
-- is only active for IDE/tooling.
"#ifdef INTELLISENSE_DIRECTIVES",
"#pragma once",
"#endif",
"// Auto-generated by tape_atom_annotation_pass.lua — DO NOT EDIT",
"// Source: " .. to_absolute_path(src.path),
"// Component atoms (MipsAtomComp_(ac_*)) -> macro variants (mac_*)",
"// + auto word-counts (so tape_atom.metadata.h stays manual-only",
"// for encoding macros).",
"",
-- Self-contained: define WORD_COUNT if not already defined.
-- The metadata file (tape_atom.metadata.h) defines it as
-- enum { words_##name = (count) };
-- We use the same definition here so the auto-generated
-- entries below expand to compile-time constants whether
-- the metadata file is included first or not.
"#ifndef WORD_COUNT",
"#define WORD_COUNT(name, count) enum { words_##name = (count) };",
"#endif",
"",
}
local wc = ctx.shared.word_counts
for _, c in ipairs(components) do
local comp_lines = build_component_lines(c, components, wc)
for _, l in ipairs(comp_lines) do lines[#lines + 1] = l end
end
local content = table.concat(lines, "\n") .. "\n"
if ctx.dry_run then
print(string.format(" -> %s (dry-run)", out_path))
return out_path
end
ensure_dir(out_dir)
write_file_lf(out_path, content)
print(string.format(" -> %s", out_path))
return out_path
end
-- ════════════════════════════════════════════════════════════════════════════
-- Pass entry
-- ════════════════════════════════════════════════════════════════════════════
--- @param ctx PassCtx
--- @return PassResult
function M.run(ctx)
-- PORT NOTE: copy find_component_atoms, find_function_args_for,
-- preceding_comment_block, extract_arg_names, find_component_atoms
-- (body parser), convert_line_comments_to_block, emit_component_macros_h
-- from tape_atom_annotation_pass.lua lines 604-1079.
-- Adaptations:
-- - Replace word_counts usage with word_count_eval.count_body_words
-- - Pass ctx.out_root / ctx.dry_run / ctx.verbose through
-- - Return { outputs, errors, warnings } shape
--
-- For each src in ctx.sources:
-- 1. find_component_atoms(src.text) -> [{name, body, args, line, comment}]
-- 2. for each component, compute its word count (recursively via
-- word_count_eval.count_body_words)
-- 3. emit "#define mac_X(args) body" + "WORD_COUNT(mac_X, N)" to
-- <src.dir>/gen/<src.basename>.macs.h
-- 4. Extend ctx.shared.word_counts with the computed counts
-- 5. Add {macs_h = path} to result.outputs
error("passes.components.run: implement by porting from " ..
"tape_atom_annotation_pass.lua:773-1079")
local outputs = {}
local errors = {}
local warnings = {}
for _, src in ipairs(ctx.sources) do
-- find_component_atoms operates on src.text
local components = find_component_atoms(src.text)
if #components > 0 then
local macs_path = emit_component_macros_h(ctx, src, components)
if macs_path then
table.insert(outputs, { macs_h = macs_path })
-- Extend the shared word_counts table so offsets sees the
-- component macros without re-reading the file.
local wc = ctx.shared.word_counts
for _, c in ipairs(components) do
wc["mac_" .. c.name] = compute_component_word_count(c, components, wc)
end
end
end
end
return { outputs = outputs, errors = errors, warnings = warnings }
end
return M
scripts/passes/offsets.lua
+373 -21
View File
@@ -7,10 +7,354 @@
-- The branch offset regression we just fixed in commit 98e27c2 must
-- NOT return. The fix was in duffle.lua's split_top_level_commas +
-- tape_atom_annotation_pass.lua's compute_component_word_count.
-- word_count_eval.count_token_words (Task 1) preserves the fix.
-- word_count_eval.count_token_words preserves the fix.
--
-- THIS MODULE ALSO REQUIRES the recent fix to duffle.lua's
-- split_top_level_commas (the second-half of the 98e27c2 fix):
-- top-level comments must be appended to the previous token, not
-- stripped, so the emit path preserves `// trailing comment` text
-- for convert_line_comments_to_block to convert to `/* */`.
--
-- Coding standard: tabs (1/level), EmmyLua annotations, no regex.
-- ════════════════════════════════════════════════════════════════════════════
-- Module-scope requires + package.path setup
-- ════════════════════════════════════════════════════════════════════════════
local script_path = arg and arg[0] or "?"
local last_sep = 0
for i = 1, #script_path do
local c = script_path:sub(i, i)
if c == "/" or c == "\\" then last_sep = i end
end
local script_dir = last_sep == 0 and "./" or script_path:sub(1, last_sep)
package.path = script_dir .. "../?.lua;" .. script_dir .. "../?/init.lua;" .. script_dir .. "?.lua;" .. package.path
local duffle = require("duffle")
local trim = duffle.trim
local read_ident = duffle.read_ident
local is_space = duffle.is_space
local is_alpha = duffle.is_alpha
local is_alnum = duffle.is_alnum
local skip_ws_and_cmt = duffle.skip_ws_and_cmt
local skip_str_or_cmt = duffle.skip_str_or_cmt
local split_top_level_commas = duffle.split_top_level_commas
local read_parens = duffle.read_parens
local read_braces = duffle.read_braces
local write_file = duffle.write_file
local dirname = duffle.dirname
local basename_no_ext = duffle.basename_no_ext
local ensure_dir = duffle.ensure_dir
local word_count_eval = require("word_count_eval")
local count_token_words = word_count_eval.count_token_words
-- ════════════════════════════════════════════════════════════════════════════
-- Local helpers (ported from offset_gen.meta.lua lines 67-93)
-- ════════════════════════════════════════════════════════════════════════════
local function starts_with(s, prefix)
if #s < #prefix then return false end
for i = 1, #prefix do
if s:sub(i, i) ~= prefix:sub(i, i) then return false end
end
return true
end
local function to_upper(s) return s:upper() end
local function to_alnum_underscore(s)
local out = ""
for i = 1, #s do
local c = s:sub(i, i)
if is_alnum(c) then out = out .. c
else out = out .. "_" end
end
return out
end
local function pad_right(s, w) return s .. string.rep(" ", w - #s) end
-- ════════════════════════════════════════════════════════════════════════════
-- Marker-call helpers (ported from offset_gen.meta.lua lines 148-205)
-- ════════════════════════════════════════════════════════════════════════════
--- Extract comma-separated identifier args from a parenthesized group
--- after a function-like macro call.
local function extract_ident_args(token, after_ident)
local arg_start = skip_ws_and_cmt(token, after_ident)
if token:sub(arg_start, arg_start) ~= "(" then return {}, nil end
local inner, after_paren = read_parens(token, arg_start)
local args = {}
local n = 1
local len = #inner
while n <= len do
n = skip_ws_and_cmt(inner, n)
if n > len then break end
local ident, after = read_ident(inner, n)
if ident and ident ~= "" then
table.insert(args, ident)
n = after
else
n = n + 1
end
n = skip_ws_and_cmt(inner, n)
if n <= len and inner:sub(n, n) == "," then n = n + 1 end
end
return args, after_paren
end
--- Scan a single token for atom_label/atom_offset markers, walking through
--- balanced groups transparently (so nested calls are found).
local function scan_for_atom_markers(token, at_pos, labels, branches)
local i = 1
local len = #token
while i <= len do
i = skip_ws_and_cmt(token, i)
if i > len then break end
local c = token:sub(i, i)
if is_alpha(c) then
local ident, after = read_ident(token, i)
if ident == "atom_label" then
local args, after_paren = extract_ident_args(token, after)
if #args >= 1 then labels[args[1]] = at_pos end
if after_paren then i = after_paren else i = after end
elseif ident == "atom_offset" then
local args, after_paren = extract_ident_args(token, after)
if #args >= 2 then table.insert(branches, {pos = at_pos, target = args[2], tag = args[1]}) end
if after_paren then i = after_paren else i = after end
else
i = after
end
else
local nx = skip_str_or_cmt(token, i)
if nx > i then i = nx else i = i + 1 end
end
end
end
--- Find the end position (just past the closing ')') of the first
--- atom_label/atom_offset call in `tok`. Returns 0 if no such call.
local function find_marker_call_end(tok)
local i = 1
local len = #tok
while i <= len do
i = skip_ws_and_cmt(tok, i)
if i > len then break end
local c = tok:sub(i, i)
if is_space(c) then
i = i + 1
elseif c == "/" then
-- comment — skip past it (delegated to duffle.skip_str_or_cmt)
local nx = skip_str_or_cmt(tok, i)
if nx > i then i = nx else i = i + 1 end
else
local ident, after = read_ident(tok, i)
if ident == "atom_label" or ident == "atom_offset" then
local j = skip_ws_and_cmt(tok, after)
if tok:sub(j, j) == "(" then
local _, end_paren = read_parens(tok, j)
return end_paren - 1
end
return 0
end
i = after or (i + 1)
end
end
return 0
end
-- ════════════════════════════════════════════════════════════════════════════
-- Atom scanner (ported from offset_gen.meta.lua lines 245-321)
-- ════════════════════════════════════════════════════════════════════════════
--- Skip C qualifier keywords (static, const, etc.) and return the position
--- past the last qualifier.
local function skip_qualifiers(source, i)
local keywords = {
["static"] = true, ["const"] = true, ["volatile"] = true,
["extern"] = true, ["register"] = true, ["auto"] = true,
["inline"] = true, ["typedef"] = true,
["internal"]= true, ["LP_"] = true, ["global"] = true, ["gkknown"] = true,
}
while true do
i = skip_ws_and_cmt(source, i)
local ident, after = read_ident(source, i)
if not ident then return i end
if keywords[ident] then i = after else return i end
end
end
--- Find every MipsAtom_(name) { ... } in a source.
local function find_atoms(source_text)
local atoms = {}
local len = #source_text
local i = 1
local function try_wrapped(after_pos)
local paren_pos = skip_ws_and_cmt(source_text, after_pos)
if source_text:sub(paren_pos, paren_pos) ~= "(" then return nil end
local inner, after_paren = read_parens(source_text, paren_pos)
local n = 1
while n <= #inner and is_space(inner:sub(n, n)) do n = n + 1 end
local ns = n
while n <= #inner and is_alnum(inner:sub(n, n)) do n = n + 1 end
local name = inner:sub(ns, n - 1)
if name == "" then return nil end
-- Find the brace after the parens.
local brace_pos = duffle.scan_to_char(source_text, "{", after_paren)
if not brace_pos then return nil end
local body, after_brace = read_braces(source_text, brace_pos)
return {name = name, body = body, after_brace = after_brace}
end
local function try_raw(after_pos)
local next_pos = skip_ws_and_cmt(source_text, after_pos)
local next_ident, next_after = read_ident(source_text, next_pos)
if not next_ident then return nil end
if not starts_with(next_ident, "code_") then return nil end
if #next_ident <= 5 then return nil end
local atom_name = next_ident:sub(6)
local brace_pos = duffle.scan_to_char(source_text, "{", next_after)
if not brace_pos then return nil end
local body, after_brace = read_braces(source_text, brace_pos)
return {name = atom_name, body = body, after_brace = after_brace}
end
while i <= len do
i = skip_ws_and_cmt(source_text, i); if i > len then break end
i = skip_qualifiers(source_text, i); if i > len then break end
local ident, after = read_ident(source_text, i)
if not ident then
i = i + 1
elseif ident == "MipsAtom_" then
local atom = try_wrapped(after)
if atom then
table.insert(atoms, {name = atom.name, body = atom.body})
i = atom.after_brace
else
i = i + 1
end
elseif ident == "MipsCode" then
local atom = try_raw(after)
if atom then
table.insert(atoms, {name = atom.name, body = atom.body})
i = atom.after_brace
else
i = after
end
else
i = after
end
end
return atoms
end
-- ════════════════════════════════════════════════════════════════════════════
-- Per-atom body scan (ported from offset_gen.meta.lua lines 207-239)
-- ════════════════════════════════════════════════════════════════════════════
--- Scan an atom body for labels + branches, count total words.
--- Returns (labels, branches, total_words).
local function scan_atom_body(body, word_counts)
local pos = 0
local labels = {}
local branches = {}
for _, tok in ipairs(split_top_level_commas(body)) do
local k = 1
local tlen = #tok
while k <= tlen and is_space(tok:sub(k, k)) do k = k + 1 end
local leading_ident = read_ident(tok, k)
if leading_ident == "atom_label" or leading_ident == "atom_offset" then
-- Marker call: record at the current pos, do NOT advance pos.
-- But the source pattern may bundle the marker with the next
-- instruction on a new line (no top-level comma between them).
-- In that case, the rest of `tok` after the marker call is
-- a real instruction that must still be counted.
scan_for_atom_markers(tok, pos, labels, branches)
local marker_end = find_marker_call_end(tok)
if marker_end > 0 and marker_end < #tok then
local rest = trim(tok:sub(marker_end + 1))
if rest ~= "" then
local rest_words = count_token_words(rest, word_counts)
pos = pos + rest_words
end
end
else
local words = count_token_words(tok, word_counts)
scan_for_atom_markers(tok, pos, labels, branches)
pos = pos + words
end
end
return labels, branches, pos
end
-- ════════════════════════════════════════════════════════════════════════════
-- Offset computation + header generation (ported lines 327-383)
-- ════════════════════════════════════════════════════════════════════════════
--- Compute branch offsets as (target_word - branch_word - 1).
local function compute_offsets(labels, branches)
local results = {}
for _, br in ipairs(branches) do
local target = labels[br.target]
if not target then
error("Branch target '" .. br.target .. "' has no atom_label (at word " .. br.pos .. ")")
end
table.insert(results, {target = br.target, tag = br.tag, offset = target - br.pos - 1})
end
return results
end
--- Generate the per-source .offsets.h header.
local function generate_header(source_path, atoms_data)
local basename = basename_no_ext(source_path)
local lines = {}
local function add(s) table.insert(lines, s) end
add("// Auto-generated by ps1_meta.lua (passes/offsets.lua) — DO NOT EDIT")
add("// Source: " .. source_path)
add("#pragma once")
add("")
add("#pragma region " .. basename)
add("")
add("")
for _, atom in ipairs(atoms_data) do
if #atom.offsets > 0 then
add("// --- atom: " .. atom.name .. " (" .. atom.total_words .. " words) ---")
add("")
local consts = {}
for _, r in ipairs(atom.offsets) do
table.insert(consts, {
macro_name = "_atom_offset_" .. r.tag .. "_" .. r.target,
enum_name = "atom_offset_" .. r.tag .. "_" .. r.target,
value = r.offset,
})
end
for _, c in ipairs(consts) do
add("#define " .. pad_right(c.macro_name, 44) .. " " .. c.value)
end
add("")
add("enum {")
for _, c in ipairs(consts) do
add(" " .. c.enum_name .. " = " .. c.macro_name .. ",")
end
add("};")
add("")
end
end
add("#pragma endregion " .. basename)
add("")
return table.concat(lines, "\n") .. "\n"
end
-- ════════════════════════════════════════════════════════════════════════════
-- M.run — orchestrator entry
-- ════════════════════════════════════════════════════════════════════════════
--- @class M
local M = {}
@@ -18,26 +362,34 @@ local M = {}
--- @param ctx PassCtx
--- @return PassResult
function M.run(ctx)
-- PORT NOTE: copy find_atoms, scan_atom_body, compute_offsets,
-- generate_header, process_source from tape_atom.offset_gen.meta.lua
-- lines 245-389.
-- Adaptations:
-- - Replace local word_count_of_token with word_count_eval.count_token_words
-- - Take (ctx, src) instead of (source_path, word_counts) — read
-- word counts from ctx.shared.word_counts
-- - Return { outputs, errors, warnings } shape
-- - Convert indentation from 8-space to tabs
--
-- For each src in ctx.sources:
-- 1. Find MipsAtom_ declarations in src.text (find_atoms)
-- 2. For each atom body, scan_atom_body to count words + find
-- atom_label/atom_offset markers
-- 3. compute_offsets (target_word - branch_word - 1 per pair)
-- 4. generate_header emits "#define atom_offset__F__T (N)" +
-- "enum { atom_offset__F__T = N, ... }" to <src.dir>/gen/<src.basename>.offsets.h
-- 5. Add {offsets_h = path} to result.outputs
error("passes.offsets.run: implement by porting from " ..
"tape_atom.offset_gen.meta.lua:245-389")
local outputs = {}
local errors = {}
local warnings = {}
for _, src in ipairs(ctx.sources) do
local atoms = find_atoms(src.text)
if #atoms > 0 then
local atoms_data = {}
for _, atom in ipairs(atoms) do
local labels, branches, total = scan_atom_body(atom.body, ctx.shared.word_counts)
local offsets = compute_offsets(labels, branches)
table.insert(atoms_data, {
name = atom.name,
total_words = total,
offsets = offsets,
})
end
local out_path = src.dir .. "/gen/" .. basename_no_ext(src.dir) .. ".offsets.h"
if not ctx.dry_run then
ensure_dir(dirname(out_path))
write_file(out_path, generate_header(src.path, atoms_data))
end
table.insert(outputs, { offsets_h = out_path })
end
end
return { outputs = outputs, errors = errors, warnings = warnings }
end
return M
scripts/passes/report.lua
+192 -16
View File
@@ -1,10 +1,163 @@
-- passes/report.lua
--
-- Render the per-project summary (build/gen/annotation_validation.txt).
-- Aggregates errors + warnings from upstream passes.
-- Render the per-project summary (build/gen/annotation_validation.txt)
-- + the per-source annotation reports (build/gen/<basename>.annotations.txt).
-- Aggregates errors + warnings from upstream annotation pass results.
--
-- The annotation pass stashes its per-source results in ctx.flags._annot_results
-- (set by passes/annotation.lua). This report pass renders them.
--
-- Coding standard: tabs (1/level), EmmyLua annotations, no regex.
-- ════════════════════════════════════════════════════════════════════════════
-- Module-scope requires + package.path setup
-- ════════════════════════════════════════════════════════════════════════════
local script_path = arg and arg[0] or "?"
local last_sep = 0
for i = 1, #script_path do
local c = script_path:sub(i, i)
if c == "/" or c == "\\" then last_sep = i end
end
local script_dir = last_sep == 0 and "./" or script_path:sub(1, last_sep)
package.path = script_dir .. "../?.lua;" .. script_dir .. "../?/init.lua;" .. script_dir .. "?.lua;" .. package.path
local duffle = require("duffle")
local ensure_dir = duffle.ensure_dir
local write_file = duffle.write_file
-- ════════════════════════════════════════════════════════════════════════════
-- Per-source annotation report (ported from tape_atom_annotation_pass.lua:1411-1486)
-- ════════════════════════════════════════════════════════════════════════════
local function render_source_report(source_path, result)
local lines = {}
local function add(s) lines[#lines + 1] = s end
add("========================================================")
add("ANNOTATION PASS — " .. source_path)
add("========================================================")
add("")
add(string.format("Atoms: %d Annotations: %d Pragmas: %d Binds structs: %d Macro decls: %d",
#result.atoms, #result.annots,
(result.pragmas and #result.pragmas or 0),
#result.binds, #result.macros))
add("")
add("── Atoms ────────────────────────────────────────────────")
for _, a in ipairs(result.atoms) do
add(string.format(" MipsAtom_(%s) line %d", a.name, a.line))
end
add("")
add("── Annotations ──────────────────────────────────────────")
for _, a in ipairs(result.annots) do
if a.error then
add(string.format(" ✗ line %d %s [ERROR: %s]", a.line, a.macro or "?", a.error))
else
local line = string.format(" %s line %d %s phase=%s",
a.kind == "work" and "●" or (a.kind == "bind" and "◆" or "○"),
a.line, a.name, a.phase or a.kind)
if a.binds then line = line .. " binds=" .. a.binds end
if #a.reads > 0 then line = line .. " reads={" .. table.concat(a.reads, ",") .. "}" end
if #a.writes > 0 then line = line .. " writes={" .. table.concat(a.writes, ",") .. "}" end
add(line)
end
end
add("")
add("── Binds_* structs ──────────────────────────────────────")
for _, b in ipairs(result.binds) do
add(string.format(" %s line %d %d bytes", b.name, b.line, b.bytes))
for _, f in ipairs(b.fields) do
add(string.format(" +%2d: %s", f.offset, f.name))
end
end
add("")
add("── Macro word-count declarations ─────────────────────────")
for _, m in ipairs(result.macros) do
add(string.format(" %s line %d words=%d", m.name, m.line, m.words))
end
add("")
add("── Atom pragmas (resource / region / group / cadence / async) ─")
if not result.pragmas or #result.pragmas == 0 then add(" (none)") end
for _, p in ipairs(result.pragmas or {}) do
local kvs = {}
for k, v in pairs(p.attrs) do kvs[#kvs + 1] = k .. "=" .. v end
table.sort(kvs)
add(string.format(" ◇ line %d %s {%s}", p.line, p.name, table.concat(kvs, ", ")))
end
add("")
add("── Errors ──────────────────────────────────────────────")
if #result.errors == 0 then add(" (none)") end
for _, e in ipairs(result.errors) do
add(string.format(" ✗ line %d %s", e.line, e.msg))
end
add("")
add("── Warnings ────────────────────────────────────────────")
if #result.warnings == 0 then add(" (none)") end
for _, w in ipairs(result.warnings) do
add(string.format(" ⚠ line %d %s", w.line, w.msg))
end
add("")
return table.concat(lines, "\n") .. "\n"
end
-- ════════════════════════════════════════════════════════════════════════════
-- Per-project summary (ported from tape_atom_annotation_pass.lua:1488-1528)
-- ════════════════════════════════════════════════════════════════════════════
local function render_project_report(all_results)
local lines = {}
local function add(s) lines[#lines + 1] = s end
local total_atoms, total_annots, total_macros, total_binds = 0, 0, 0, 0
local total_errors, total_warnings = 0, 0
for _, r in ipairs(all_results) do
total_atoms = total_atoms + #r.atoms
total_annots = total_annots + #r.annots
total_macros = total_macros + #r.macros
total_binds = total_binds + #r.binds
total_errors = total_errors + #r.errors
total_warnings = total_warnings + #r.warnings
end
add("========================================================")
add("ANNOTATION VALIDATION — project summary")
add("========================================================")
add("")
add(string.format("Atoms: %d", total_atoms))
add(string.format("Annotations: %d", total_annots))
add(string.format("Macros: %d", total_macros))
add(string.format("Binds: %d", total_binds))
add("")
add(string.format("Errors: %d", total_errors))
add(string.format("Warnings: %d", total_warnings))
add("")
if total_errors > 0 then
add("Per-source error counts:")
for _, r in ipairs(all_results) do
if #r.errors > 0 then
add(string.format(" %s : %d error(s)", r.source, #r.errors))
end
end
add("")
end
return table.concat(lines, "\n") .. "\n"
end
-- ════════════════════════════════════════════════════════════════════════════
-- M.run — orchestrator entry
-- ════════════════════════════════════════════════════════════════════════════
--- @class M
local M = {}
@@ -12,20 +165,43 @@ local M = {}
--- @param ctx PassCtx
--- @return PassResult
function M.run(ctx)
-- PORT NOTE: copy render_source_report + render_project_report
-- from tape_atom_annotation_pass.lua lines 1411-1528.
-- Adaptations:
-- - Read errors/warnings from ctx.upstream[pass_name].outputs/errors
-- (orchestrator-collected)
-- - Write to <ctx.out_root>/annotation_validation.txt (was
-- build/gen/annotation_validation.txt — same path)
-- - Return { outputs = {{summary_txt = path}}, errors = {}, warnings = {} }
--
-- The summary renders a per-atom table + error count + warning count,
-- matching the pre-rework format. See the spec's "Report" pass
-- description for the expected output.
error("passes.report.run: implement by porting render_source_report + " ..
"render_project_report from tape_atom_annotation_pass.lua:1411-1528")
local outputs = {}
local errors = {}
local warnings = {}
-- The annotation pass stashes per-source results in ctx.flags._annot_results.
-- Render each as build/gen/<basename>.annotations.txt and aggregate into
-- build/gen/annotation_validation.txt.
local annot_results = (ctx.flags and ctx.flags._annot_results) or {}
-- Render per-source reports.
for _, entry in ipairs(annot_results) do
local src = entry.source
local result = entry.result
local out_path = ctx.out_root .. "/" .. src.basename .. ".annotations.txt"
if not ctx.dry_run then
ensure_dir(ctx.out_root)
write_file(out_path, render_source_report(src.path, result))
end
table.insert(outputs, { annotations_txt = out_path })
end
-- Render project summary.
if not ctx.dry_run then
-- The project report references each source by its absolute path.
-- Augment the entries with a .source field for the per-source error counts.
local all_results = {}
for _, entry in ipairs(annot_results) do
entry.result.source = entry.source.path
table.insert(all_results, entry.result)
end
ensure_dir(ctx.out_root)
local summary_path = ctx.out_root .. "/annotation_validation.txt"
write_file(summary_path, render_project_report(all_results))
table.insert(outputs, { summary_txt = summary_path })
end
return { outputs = outputs, errors = errors, warnings = warnings }
end
return M
scripts/tape_atom.offset_gen.meta.lua
-488
View File
@@ -1,488 +0,0 @@
#!/usr/bin/env lua
-- tape_atom_offset_gen.lua
--
-- Finds every `MipsAtom_(name) { ... }` declaration in the given sources,
-- counts the words in each body using the WORD_COUNT manifest, computes
-- branch offsets for atom_label(name) / atom_offset(tag, name) markers,
-- and writes one header per source into <source_dir>/gen/<basename>.offsets.h
--
-- Generated header layout (per source):
-- #pragma region <basename>
-- #undef atom_offset
-- #define atom_offset(tag, name) atom_offset_##tag##_##name
-- // --- atom: <name> (<n> words) ---
-- #define atom_offset_<tag>_<target> (N) // preprocessor form
-- #undef atom_offset_<tag>_<target> // (so enum can reuse)
-- enum {
-- atom_offset_<tag>_<target> = N, // C enum form
-- };
-- #define atom_offset_<tag>_<target> (N) // re-define for preprocessor
-- #pragma endregion <basename>
--
-- Usage:
-- luajit gen_atom_offsets.lua <metadata.h> <source1> [source2 ...]
-- Make require("duffle") resolve to the sibling duffle.lua in this dir,
-- AND make require("lpeg") find the vendored LPeg DLL in the toolchain.
-- Both prepends are explicit (no :match / no Lua pattern — plain byte scan).
local script_path = arg[0]
local last_sep = 0
for i = 1, #script_path do
local c = script_path:sub(i, i)
if c == "/" or c == "\\" then last_sep = i end
end
local script_dir = last_sep == 0 and "./" or script_path:sub(1, last_sep)
package.path = script_dir .. "?.lua;" .. script_dir .. "?/init.lua;" .. package.path
package.cpath = "C:\\projects\\Pikuma\\ps1\\toolchain\\luajit-2.1\\lib\\lua\\5.1\\?.dll;" .. package.cpath
-- Shared primitives + domain tables live in scripts/duffle.lua.
local duffle = require("duffle")
-- Local aliases so the rest of this file reads cleanly. These resolve
-- to the same functions in duffle.lua (5.3-compatible, no regex).
local is_space = duffle.is_space
local is_alpha = duffle.is_alpha
local is_alnum = duffle.is_alnum
local trim = duffle.trim
local find_byte = duffle.find_byte
local read_file = duffle.read_file
local write_file = duffle.write_file
local ensure_dir = duffle.ensure_dir
local dirname = duffle.dirname
local basename_no_ext = duffle.basename_no_ext
local skip_str_or_cmt = duffle.skip_str_or_cmt
local skip_ws_and_cmt = duffle.skip_ws_and_cmt
local read_ident = duffle.read_ident
local read_parens = duffle.read_parens
local read_braces = duffle.read_braces
local read_brackets = duffle.read_brackets
local scan_to_char = duffle.scan_to_char
local split_top_level_commas = duffle.split_top_level_commas
local load_word_counts = duffle.load_word_counts
-- ============================================================
-- Offset-gen-specific helpers (not in duffle.lua)
-- ============================================================
local function starts_with(s, prefix)
if #s < #prefix then return false end
for i = 1, #prefix do
if s:sub(i, i) ~= prefix:sub(i, i) then return false end
end
return true
end
local function ends_with(s, suffix)
if #s < #suffix then return false end
local off = #s - #suffix
for i = 1, #suffix do
if s:sub(off + i, off + i) ~= suffix:sub(i, i) then return false end
end
return true
end
local function to_upper(s) return s:upper() end
local function to_alnum_underscore(s)
local out = ""
for i = 1, #s do
local c = s:sub(i, i)
if is_alnum(c) then out = out .. c
else out = out .. "_" end
end
return out
end
local function pad_right(s, w) return s .. string.rep(" ", w - #s) end
-- ============================================================
-- Extract comma-separated identifier args from a parenthesized group
-- after a function-like macro call.
-- ============================================================
local function extract_ident_args(token, after_ident)
local arg_start = skip_ws_and_cmt(token, after_ident)
if token:sub(arg_start, arg_start) ~= "(" then return {}, nil end
local inner, after_paren = read_parens(token, arg_start)
local args = {}
local n = 1
local len = #inner
while n <= len do
n = skip_ws_and_cmt(inner, n)
if n > len then break end
local ident, after = read_ident(inner, n)
if ident and ident ~= "" then
table.insert(args, ident)
n = after
else
n = n + 1
end
n = skip_ws_and_cmt(inner, n)
if n <= len and inner:sub(n, n) == "," then n = n + 1 end
end
return args, after_paren
end
-- ============================================================
-- Count words for a single comma-separated token
-- ============================================================
local function word_count_of_token(token, wc)
local s = trim(token)
if s == "" then return 0 end
local name, after = read_ident(s, 1)
if not name then return 1 end
if wc[name] then return wc[name] end
local j = skip_ws_and_cmt(s, after)
if s:sub(j, j) == "(" then
io.stderr:write(" warning: unknown macro '" .. name .. "', assuming 1 word\n")
end
return 1
end
-- ============================================================
-- Scan token for atom_label/atom_offset markers, walking through
-- balanced groups transparently (so nested calls are found)
-- ============================================================
local function scan_for_atom_markers(token, at_pos, labels, branches)
local i = 1
local len = #token
while i <= len do
i = skip_ws_and_cmt(token, i)
if i > len then break end
local c = token:sub(i, i)
if is_alpha(c) then
local ident, after = read_ident(token, i)
if ident == "atom_label" then
local args, after_paren = extract_ident_args(token, after)
if #args >= 1 then labels[args[1]] = at_pos end
if after_paren then i = after_paren else i = after end
elseif ident == "atom_offset" then
local args, after_paren = extract_ident_args(token, after)
if #args >= 2 then table.insert(branches, {pos = at_pos, target = args[2], tag = args[1]}) end
if after_paren then i = after_paren else i = after end
else
i = after
end
else
local nx = skip_str_or_cmt(token, i)
if nx > i then i = nx else i = i + 1 end
end
end
end
-- ============================================================
-- Scan atom body, count words, find markers
-- ============================================================
-- Find the end position (just past the closing ')') of the first
-- atom_label/atom_offset call in `tok`. Returns 0 if no such call.
local function find_marker_call_end(tok)
local i = 1
local len = #tok
while i <= len do
i = skip_ws_and_cmt(tok, i)
if i > len then break end
local c = tok:sub(i, i)
if is_alpha(c) then
local ident, after = read_ident(tok, i)
if ident == "atom_label" or ident == "atom_offset" then
local j = skip_ws_and_cmt(tok, after)
if tok:sub(j, j) == "(" then
local _, end_paren = read_parens(tok, j)
return end_paren - 1
end
return 0
end
i = after
else
local nx = skip_str_or_cmt(tok, i)
if nx > i then i = nx else i = i + 1 end
end
end
return 0
end
local function scan_atom_body(body, word_counts)
local pos = 0
local labels = {}
local branches = {}
local _dbg_count = 0
for _, tok in ipairs(split_top_level_commas(body)) do
local k = 1
local tlen = #tok
while k <= tlen and is_space(tok:sub(k, k)) do k = k + 1 end
local leading_ident = read_ident(tok, k)
if leading_ident == "atom_label" or leading_ident == "atom_offset" then
-- Marker call: record at the current pos, do NOT advance pos.
-- But the source pattern may bundle the marker with the next
-- instruction on a new line (no top-level comma between them).
-- In that case, the rest of `tok` after the marker call is a
-- real instruction that must still be counted.
scan_for_atom_markers(tok, pos, labels, branches)
local marker_end = find_marker_call_end(tok)
if marker_end > 0 and marker_end < #tok then
local rest = trim(tok:sub(marker_end + 1))
if rest ~= "" then
local rest_words = word_count_of_token(rest, word_counts)
pos = pos + rest_words
end
end
else
local words = word_count_of_token(tok, word_counts)
scan_for_atom_markers(tok, pos, labels, branches)
pos = pos + words
end
end
return labels, branches, pos
end
-- ============================================================
-- Find every MipsAtom_(name) { ... } in a source
-- ============================================================
local function skip_qualifiers(source, i)
local keywords = {
["static"] = true, ["const"] = true, ["volatile"] = true,
["extern"] = true, ["register"] = true, ["auto"] = true,
["inline"] = true, ["typedef"] = true,
["internal"]= true, ["LP_"] = true, ["global"] = true, ["gkknown"] = true
}
while true do
i = skip_ws_and_cmt(source, i)
local ident, after = read_ident(source, i)
if not ident then return i end
if keywords[ident] then i = after else return i end
end
end
local function find_atoms(source_text)
local atoms = {}
local len = #source_text
local i = 1
local function try_wrapped(after_pos)
local paren_pos = skip_ws_and_cmt(source_text, after_pos)
if source_text:sub(paren_pos, paren_pos) ~= "(" then return nil end
local inner, after_paren = read_parens(source_text, paren_pos)
local n = 1
while n <= #inner and is_space(inner:sub(n, n)) do n = n + 1 end
local ns = n
while n <= #inner and is_alnum(inner:sub(n, n)) do n = n + 1 end
local name = inner:sub(ns, n - 1)
if name == "" then return nil end
local brace_pos = scan_to_char(source_text, "{", after_paren)
if not brace_pos then return nil end
local body, after_brace = read_braces(source_text, brace_pos)
return {name = name, body = body, after_brace = after_brace}
end
local function try_raw(after_pos)
local next_pos = skip_ws_and_cmt(source_text, after_pos)
local next_ident, next_after = read_ident(source_text, next_pos)
if not next_ident then return nil end
if not starts_with(next_ident, "code_") then return nil end
if #next_ident <= 5 then return nil end
local atom_name = next_ident:sub(6)
local brace_pos = scan_to_char(source_text, "{", next_after)
if not brace_pos then return nil end
local body, after_brace = read_braces(source_text, brace_pos)
return {name = atom_name, body = body, after_brace = after_brace}
end
while i <= len do
i = skip_ws_and_cmt(source_text, i); if i > len then break end
i = skip_qualifiers(source_text, i); if i > len then break end
local ident, after = read_ident(source_text, i)
if not ident then
i = i + 1
elseif ident == "MipsAtom_" then
local atom = try_wrapped(after)
if atom then
table.insert(atoms, {name = atom.name, body = atom.body})
i = atom.after_brace
else
i = i + 1
end
elseif ident == "MipsCode" then
local atom = try_raw(after)
if atom then
table.insert(atoms, {name = atom.name, body = atom.body})
i = atom.after_brace
else
i = after
end
else
i = after
end
end
return atoms
end
-- ============================================================
-- Compute branch offsets (target - branch - 1)
-- ============================================================
local function compute_offsets(labels, branches)
local results = {}
for _, br in ipairs(branches) do
local target = labels[br.target]
if not target then
error("Branch target '" .. br.target .. "' has no atom_label (at word " .. br.pos .. ")")
end
table.insert(results, {target = br.target, tag = br.tag, offset = target - br.pos - 1 })
end
return results
end
-- ============================================================
-- Generate header for one source
-- ============================================================
local function generate_header(source_path, atoms_data)
local basename = basename_no_ext(source_path)
local guard = to_alnum_underscore(to_upper(basename)) .. "_OFFSETS_H"
local lines = {}
local function add(s) table.insert(lines, s) end
add("// Auto-generated by tape_atom_offset_gen.meta.lua — DO NOT EDIT")
add("// Source: " .. source_path)
add("#pragma once")
add("")
add("#pragma region " .. basename)
add("")
-- add("// Dispatch macro: token-pastes <tag>_<target> to the enum name")
-- add("#undef atom_offset")
-- add("#define atom_offset(tag, name) atom_offset_##tag##_##name")
add("")
for _, atom in ipairs(atoms_data) do
if #atom.offsets > 0 then
add("// --- atom: " .. atom.name .. " (" .. atom.total_words .. " words) ---")
add("")
local consts = {}
for _, r in ipairs(atom.offsets) do
table.insert(consts, {
macro_name = "_atom_offset_" .. r.tag .. "_" .. r.target,
enum_name = "atom_offset_" .. r.tag .. "_" .. r.target,
value = r.offset
})
end
for _, c in ipairs(consts) do add("#define " .. pad_right(c.macro_name, 44) .. " " .. c.value .. "") end
add("")
add("enum {")
for _, c in ipairs(consts) do add(" " .. c.enum_name .. " = " .. c.macro_name .. ",") end
add("};")
add("")
end
end
add("#pragma endregion " .. basename)
add("")
return table.concat(lines, "\n") .. "\n"
end
-- ============================================================
-- Process one source
-- ============================================================
local function process_source(source_path, word_counts)
local source = read_file(source_path)
local atoms_raw = find_atoms(source)
if #atoms_raw == 0 then
-- io.stderr:write(" note: no MipsAtom_ declarations in " .. source_path .. "\n")
return
end
local atoms_data = {}
for _, atom in ipairs(atoms_raw) do
local labels, branches, total = scan_atom_body(atom.body, word_counts)
local offsets = compute_offsets(labels, branches)
table.insert(atoms_data, {
name = atom.name,
total_words = total,
offsets = offsets
})
end
local basename = basename_no_ext(source_path)
local dir_basename = basename_no_ext(dirname(source_path))
local out_dir = dirname(source_path) .. "/gen"
ensure_dir(out_dir)
local out_path = out_dir .. "/" .. dir_basename .. ".offsets.h"
write_file(out_path, generate_header(source_path, atoms_data))
local total_branches = 0
for _, a in ipairs(atoms_data) do total_branches = total_branches + #a.offsets end
print(" " .. basename .. ": " .. #atoms_data .. " atom(s), " .. total_branches .. " branch(es)")
for _, a in ipairs(atoms_data) do
for _, r in ipairs(a.offsets) do
print(" " .. a.name .. " -> " .. r.tag .. ":" .. r.target .. " : " .. r.offset)
end
end
end
-- ============================================================
-- Main
-- ============================================================
-- Scan a directory recursively for files matching a pattern.
-- No regex (per the no_regex constraint) — uses plain byte matching.
local function scan_dir(dir, suffix)
local results = {}
local p = io.popen('dir /b /s "' .. dir .. '\\' .. suffix .. '" 2>nul')
if not p then return results end
for raw_line in p:lines() do
-- dir /b outputs paths with backslashes; normalize to forward.
local path = raw_line:gsub("\\", "/")
results[#results + 1] = path
end
p:close()
return results
end
local function main(args)
if #args < 2 then
print("Usage: luajit gen_atom_offsets.lua <metadata.h> <source1> [source2 ...]")
os.exit(1)
end
-- Find the project root (parent of the metadata file's dir).
local meta_dir = dirname(args[1])
if #meta_dir > 0 and (meta_dir:sub(-1) == "/" or meta_dir:sub(-1) == "\\") then
meta_dir = meta_dir:sub(1, -2)
end
local project_root = dirname(meta_dir)
-- Scan the project for all *.macs.h files and load their WORD_COUNT
-- entries as the source of truth. The metaprogram's recursive
-- counting (which accounts for macro-in-macro expansion) gives
-- the correct values; the metadata file's manually-maintained
-- entries are NOT loaded here (they may be wrong for recursive cases).
local word_counts = {}
local macs_files = scan_dir(project_root, "*.macs.h")
for _, macs_path in ipairs(macs_files) do
-- Convert forward slashes to backslashes for Windows file open.
local win_path = macs_path:gsub("/", "\\")
local ok, mc = pcall(load_word_counts, win_path)
if ok and type(mc) == "table" then
for name, count in pairs(mc) do
word_counts[name] = count
end
end
end
-- Also load the metadata's encoding-macro entries (which are NOT
-- in the macs.h, since the macs.h is component-only). These are
-- the regular 1-word-per-macro entries like load_word, add_ui, etc.
local meta_counts = load_word_counts(args[1])
for name, count in pairs(meta_counts) do
if not word_counts[name] then
word_counts[name] = count
end
end
for i = 2, #args do
local source_path = args[i]
process_source(source_path, word_counts)
end
end
main({...})
scripts/tape_atom_annotation_pass.lua
-1628
View File
File diff suppressed because it is too large Load Diff