From 1209172649bea4ce35549c03480210af87a8578f Mon Sep 17 00:00:00 2001 From: Ed_ Date: Thu, 9 Jul 2026 18:45:36 -0400 Subject: [PATCH] wip: lua metaprogram rework --- code/duffle/lottes_tape.h | 2 +- scripts/passes/annotation.lua | 29 ++ scripts/passes/components.lua | 39 +++ scripts/passes/offsets.lua | 43 +++ scripts/passes/report.lua | 31 ++ scripts/passes/static_analysis.lua | 16 + scripts/ps1_meta.lua | 520 +++++++++++++++++++++++++++++ scripts/word_count_eval.lua | 246 ++++++++++++++ 8 files changed, 925 insertions(+), 1 deletion(-) create mode 100644 scripts/passes/annotation.lua create mode 100644 scripts/passes/components.lua create mode 100644 scripts/passes/offsets.lua create mode 100644 scripts/passes/report.lua create mode 100644 scripts/passes/static_analysis.lua create mode 100644 scripts/ps1_meta.lua create mode 100644 scripts/word_count_eval.lua diff --git a/code/duffle/lottes_tape.h b/code/duffle/lottes_tape.h index e8aaa1e..df6dadc 100644 --- a/code/duffle/lottes_tape.h +++ b/code/duffle/lottes_tape.h @@ -14,7 +14,7 @@ typedef U4 const MipsCode; typedef Slice_(MipsCode); typedef Slice_MipsCode MipsAtom; -#define MipsAtom_(sym) MipsCode tmpl(code,sym) [] align_(4) = +#define MipsAtom_(sym) MipsCode tmpl(code,sym) [] align_(4) = // Bare form: file-scope declaration with hardcoded body. // Used for components with no args (e.g., ac_load_tri_indices) or diff --git a/scripts/passes/annotation.lua b/scripts/passes/annotation.lua new file mode 100644 index 0000000..d06a393 --- /dev/null +++ b/scripts/passes/annotation.lua @@ -0,0 +1,29 @@ +-- passes/annotation.lua +-- +-- Validate atom annotation DSL usage in source files. Reads: +-- - atom_annot / atom_init / atom_setup / atom_commit / atom_bind +-- / atom_terminate / atom_label / atom_offset / TAPE_WORDS +-- - atom_resource / atom_region / atom_group / atom_cadence / atom_async +-- - Binds_* struct declarations +-- Writes: +-- - build/gen/.errors.h (with #error directives on findings) +-- - build/gen/.annotations.txt (human-readable summary) +-- Ported from scripts/tape_atom_annotation_pass.lua:78-545 + 1081-1407 +-- (validation only — NOT rendering, which goes to passes/report.lua). +-- +-- Coding standard: tabs (1/level), EmmyLua annotations, no regex. + +--- @class M + +local M = {} + +--- @param ctx PassCtx +--- @return PassResult +function M.run(ctx) + error("passes.annotation.run: implement by porting " .. + "find_atom_names, find_atom_annotations, find_macro_word_annotations, " .. + "find_atom_pragmas, find_binds_structs, validate from " .. + "tape_atom_annotation_pass.lua:78-1407") +end + +return M \ No newline at end of file diff --git a/scripts/passes/components.lua b/scripts/passes/components.lua new file mode 100644 index 0000000..32be677 --- /dev/null +++ b/scripts/passes/components.lua @@ -0,0 +1,39 @@ +-- passes/components.lua +-- +-- Generate /gen/.macs.h from MipsAtomComp_ declarations +-- in source files. Ported from tape_atom_annotation_pass.lua:773-1079 +-- (find_component_atoms, preceding_comment_block, extract_arg_names, +-- convert_line_comments_to_block, compute_component_word_count, +-- emit_component_macros_h). +-- +-- Coding standard: tabs (1/level), EmmyLua annotations, no regex. + +--- @class M + +local M = {} + +--- @param ctx PassCtx +--- @return PassResult +function M.run(ctx) + -- PORT NOTE: copy find_component_atoms, find_function_args_for, + -- preceding_comment_block, extract_arg_names, find_component_atoms + -- (body parser), convert_line_comments_to_block, emit_component_macros_h + -- from tape_atom_annotation_pass.lua lines 604-1079. + -- Adaptations: + -- - Replace word_counts usage with word_count_eval.count_body_words + -- - Pass ctx.out_root / ctx.dry_run / ctx.verbose through + -- - Return { outputs, errors, warnings } shape + -- + -- For each src in ctx.sources: + -- 1. find_component_atoms(src.text) -> [{name, body, args, line, comment}] + -- 2. for each component, compute its word count (recursively via + -- word_count_eval.count_body_words) + -- 3. emit "#define mac_X(args) body" + "WORD_COUNT(mac_X, N)" to + -- /gen/.macs.h + -- 4. Extend ctx.shared.word_counts with the computed counts + -- 5. Add {macs_h = path} to result.outputs + error("passes.components.run: implement by porting from " .. + "tape_atom_annotation_pass.lua:773-1079") +end + +return M \ No newline at end of file diff --git a/scripts/passes/offsets.lua b/scripts/passes/offsets.lua new file mode 100644 index 0000000..ba3d79d --- /dev/null +++ b/scripts/passes/offsets.lua @@ -0,0 +1,43 @@ +-- passes/offsets.lua +-- +-- Generate /gen/.offsets.h with branch offset +-- immediates for every atom_offset(F, T) reference in atom bodies. +-- Ported from scripts/tape_atom.offset_gen.meta.lua:148-389. +-- +-- The branch offset regression we just fixed in commit 98e27c2 must +-- NOT return. The fix was in duffle.lua's split_top_level_commas + +-- tape_atom_annotation_pass.lua's compute_component_word_count. +-- word_count_eval.count_token_words (Task 1) preserves the fix. +-- +-- Coding standard: tabs (1/level), EmmyLua annotations, no regex. + +--- @class M + +local M = {} + +--- @param ctx PassCtx +--- @return PassResult +function M.run(ctx) + -- PORT NOTE: copy find_atoms, scan_atom_body, compute_offsets, + -- generate_header, process_source from tape_atom.offset_gen.meta.lua + -- lines 245-389. + -- Adaptations: + -- - Replace local word_count_of_token with word_count_eval.count_token_words + -- - Take (ctx, src) instead of (source_path, word_counts) — read + -- word counts from ctx.shared.word_counts + -- - Return { outputs, errors, warnings } shape + -- - Convert indentation from 8-space to tabs + -- + -- For each src in ctx.sources: + -- 1. Find MipsAtom_ declarations in src.text (find_atoms) + -- 2. For each atom body, scan_atom_body to count words + find + -- atom_label/atom_offset markers + -- 3. compute_offsets (target_word - branch_word - 1 per pair) + -- 4. generate_header emits "#define atom_offset__F__T (N)" + + -- "enum { atom_offset__F__T = N, ... }" to /gen/.offsets.h + -- 5. Add {offsets_h = path} to result.outputs + error("passes.offsets.run: implement by porting from " .. + "tape_atom.offset_gen.meta.lua:245-389") +end + +return M \ No newline at end of file diff --git a/scripts/passes/report.lua b/scripts/passes/report.lua new file mode 100644 index 0000000..071e92f --- /dev/null +++ b/scripts/passes/report.lua @@ -0,0 +1,31 @@ +-- passes/report.lua +-- +-- Render the per-project summary (build/gen/annotation_validation.txt). +-- Aggregates errors + warnings from upstream passes. +-- +-- Coding standard: tabs (1/level), EmmyLua annotations, no regex. + +--- @class M + +local M = {} + +--- @param ctx PassCtx +--- @return PassResult +function M.run(ctx) + -- PORT NOTE: copy render_source_report + render_project_report + -- from tape_atom_annotation_pass.lua lines 1411-1528. + -- Adaptations: + -- - Read errors/warnings from ctx.upstream[pass_name].outputs/errors + -- (orchestrator-collected) + -- - Write to /annotation_validation.txt (was + -- build/gen/annotation_validation.txt — same path) + -- - Return { outputs = {{summary_txt = path}}, errors = {}, warnings = {} } + -- + -- The summary renders a per-atom table + error count + warning count, + -- matching the pre-rework format. See the spec's "Report" pass + -- description for the expected output. + error("passes.report.run: implement by porting render_source_report + " .. + "render_project_report from tape_atom_annotation_pass.lua:1411-1528") +end + +return M \ No newline at end of file diff --git a/scripts/passes/static_analysis.lua b/scripts/passes/static_analysis.lua new file mode 100644 index 0000000..87f822e --- /dev/null +++ b/scripts/passes/static_analysis.lua @@ -0,0 +1,16 @@ +-- passes/static_analysis.lua +-- +-- [FUTURE] Per-atom static-analysis checks. Stub for now; the upcoming +-- static_analysis_atoms_20260708 track will deliver the 5 (+1) checks. + +--- @class M + +local M = {} + +--- @param ctx PassCtx +--- @return PassResult +function M.run(ctx) + return { outputs = {}, errors = {}, warnings = {} } +end + +return M \ No newline at end of file diff --git a/scripts/ps1_meta.lua b/scripts/ps1_meta.lua new file mode 100644 index 0000000..c2b72fe --- /dev/null +++ b/scripts/ps1_meta.lua @@ -0,0 +1,520 @@ +-- ps1_meta.lua +-- +-- Orchestrator entry point for the tape-atom metaprogram pipeline. +-- Dispatches to pass modules under scripts/passes/, resolving dependencies +-- topologically. Single CLI surface (`--` flags + auto-dep + --dry-run). +-- +-- Coding standard: tabs (1/level), EmmyLua annotations, no regex, +-- Lua 5.3 compatible. + +-- ════════════════════════════════════════════════════════════════════════════ +-- Module-scope requires + package.path setup +-- ════════════════════════════════════════════════════════════════════════════ + +local script_path = arg and arg[0] or "?" +local last_sep = 0 +for i = 1, #script_path do + local c = script_path:sub(i, i) + if c == "/" or c == "\\" then last_sep = i end +end +local script_dir = last_sep == 0 and "./" or script_path:sub(1, last_sep) +package.path = script_dir .. "?.lua;" .. script_dir .. "?/init.lua;" .. package.path +package.cpath = "C:\\projects\\Pikuma\\ps1\\toolchain\\luajit-2.1\\lib\\lua\\5.1\\?.dll;" .. package.cpath + +local duffle = require("duffle") +local dirname = duffle.dirname +local basename_no_ext = duffle.basename_no_ext + +-- ════════════════════════════════════════════════════════════════════════════ +-- Type declarations +-- ════════════════════════════════════════════════════════════════════════════ + +--- @class PassDescriptor +--- @field module string -- module name passed to require() +--- @field kind string -- "shared" | "header-output" | "validation" | "report" +--- @field deps string[] -- names of upstream passes +--- @field desc string -- human description (used by --help + ASCII graph) +--- @field out PassOutput[] -- output paths (used by --dry-run + report) + +--- @class PassOutput +--- @field kind string -- "header" | "report" +--- @field path_template string -- e.g. "/gen/.macs.h" + +--- @class SourceFile +--- @field path string +--- @field text string +--- @field dir string +--- @field basename string + +--- @class PassCtx +--- @field sources SourceFile[] +--- @field metadata_path string +--- @field shared table +--- @field shared.word_counts table +--- @field out_root string +--- @field project_root string +--- @field upstream table +--- @field flags table +--- @field dry_run boolean +--- @field verbose boolean + +--- @class PassResult +--- @field outputs table[] +--- @field errors table[] +--- @field warnings table[] + +--- @class ParsedArgs +--- @field requested_set string[] -- pass names to run (explicit --all expanded) +--- @field sources string[] -- --source values +--- @field metadata string -- --metadata value +--- @field out_root string -- --out-root value (default "build/gen") +--- @field project_root string -- --project-root value (default dirname(metadata)) +--- @field dry_run boolean +--- @field verbose boolean + +-- ════════════════════════════════════════════════════════════════════════════ +-- PASSES table (data, not code) — the orchestrator's dep graph +-- ════════════════════════════════════════════════════════════════════════════ + +local PASSES = { + ["word-counts"] = { + module = "word_count_eval", + kind = "shared", + deps = {}, + desc = "Build the shared metadata table (metadata.h + .macs.h)", + out = {}, + }, + components = { + module = "passes.components", + kind = "header-output", + deps = {"word-counts"}, + desc = "Emit mac_X macros from MipsAtomComp_ declarations", + out = { { kind = "header", path_template = "/gen/.macs.h" } }, + }, + annotation = { + module = "passes.annotation", + kind = "validation", + deps = {"word-counts"}, + desc = "Validate atom DSL usage; emit errors.h + annotations.txt", + out = { + { kind = "report", path_template = "/.errors.h" }, + { kind = "report", path_template = "/.annotations.txt" }, + }, + }, + offsets = { + module = "passes.offsets", + kind = "header-output", + deps = {"word-counts", "components"}, + desc = "Compute branch offsets for atom_label / atom_offset", + out = { { kind = "header", path_template = "/gen/.offsets.h" } }, + }, + ["static-analysis"] = { + module = "passes.static_analysis", + kind = "validation", + deps = {"word-counts", "components"}, + desc = "[FUTURE] GTE pipeline-fill, mac_yield uniformity, etc.", + out = { { kind = "report", path_template = "/.static_analysis.txt" } }, + }, + report = { + module = "passes.report", + kind = "report", + deps = {"annotation", "static-analysis"}, + desc = "Render the per-project summary", + out = { { kind = "report", path_template = "/annotation_validation.txt" } }, + }, +} + +-- Pass-kind taxonomy: which kinds stop the build on errors? +local PASS_KIND_STOP_ON_ERROR = { + ["shared"] = false, + ["header-output"] = true, + ["validation"] = true, + ["report"] = false, +} + +-- Closed set of CLI flags -> pass names. +local PASS_FLAG_TO_NAME = { + ["--word-counts"] = "word-counts", + ["--components"] = "components", + ["--validate"] = "annotation", + ["--offsets"] = "offsets", + ["--static-analysis"] = "static-analysis", + ["--report"] = "report", + ["--all"] = "__all__", +} + +local ALL_PASS_NAMES = { + "word-counts", "components", "annotation", + "offsets", "static-analysis", "report", +} + +-- ════════════════════════════════════════════════════════════════════════════ +-- CLI parsing +-- ════════════════════════════════════════════════════════════════════════════ + +--- Print the CLI usage to stdout and exit 0. +local function print_help() + io.write([[ +ps1_meta.lua - Tape-atom metaprogram orchestrator + +USAGE: + ps1_meta.lua [PASS_FLAGS] [COMMON_FLAGS] + +PASS_FLAGS (pick one or more, or use --all): + --word-counts Load metadata.h + scan for existing .macs.h + --components Generate /gen/.macs.h + --validate Run atom annotation DSL validation + --offsets Generate /gen/.offsets.h + --static-analysis [FUTURE] GTE pipeline-fill, mac_yield uniformity + --report Render per-project summary + --all Equivalent to all 6 flags above (default) + +COMMON_FLAGS: + --source FILE Source file to process (repeatable) + --metadata PATH Path to metadata.h (required) + --out-root DIR Output root for reports (default: build/gen) + --project-root DIR Project root for .macs.h scan (default: dirname(metadata)) + --dry-run Print dep order + ASCII graph; exit 0 without running + --verbose Print per-pass debug output + --help Show this help and exit + +EXIT CODES: + 0 All requested passes succeeded + 1 Validation errors found + 2 Metaprogram internal error + +EXAMPLE: + ps1_meta.lua --all --metadata metadata.h --source code/foo.c --source code/bar.c +]]) +end + +--- Parse argv into a structured table. Validates against a closed enum. +--- +--- @param argv string[] +--- @return ParsedArgs +local function parse_args(argv) + local args = { + requested_set = {}, + sources = {}, + metadata = nil, + out_root = "build/gen", + project_root = nil, + dry_run = false, + verbose = false, + } + + local i = 1 + while i <= #argv do + local a = argv[i] + if a == "--help" then + print_help() + os.exit(0) + elseif a == "--dry-run" then + args.dry_run = true + elseif a == "--verbose" then + args.verbose = true + elseif a == "--source" then + i = i + 1 + args.sources[#args.sources + 1] = argv[i] + elseif a == "--metadata" then + i = i + 1 + args.metadata = argv[i] + elseif a == "--out-root" then + i = i + 1 + args.out_root = argv[i] + elseif a == "--project-root" then + i = i + 1 + args.project_root = argv[i] + elseif PASS_FLAG_TO_NAME[a] then + local name = PASS_FLAG_TO_NAME[a] + if name == "__all__" then + for _, n in ipairs(ALL_PASS_NAMES) do + args.requested_set[#args.requested_set + 1] = n + end + else + args.requested_set[#args.requested_set + 1] = name + end + else + io.stderr:write("ps1_meta: unknown flag '" .. a .. "'\n") + io.stderr:write("Run with --help for usage.\n") + os.exit(2) + end + i = i + 1 + end + + -- Default: --all if no explicit pass flags. + if #args.requested_set == 0 then + for _, n in ipairs(ALL_PASS_NAMES) do + args.requested_set[#args.requested_set + 1] = n + end + end + + -- Defaults: project_root = dirname(metadata). + if args.metadata and not args.project_root then + local d = dirname(args.metadata) + if #d > 0 and (d:sub(-1) == "/" or d:sub(-1) == "\\") then + d = d:sub(1, -2) + end + args.project_root = dirname(d) + end + + if not args.metadata then + io.stderr:write("ps1_meta: --metadata PATH is required\n") + os.exit(2) + end + if #args.sources == 0 then + io.stderr:write("ps1_meta: at least one --source FILE is required\n") + os.exit(2) + end + + return args +end + +-- ════════════════════════════════════════════════════════════════════════════ +-- Build ctx from parsed args +-- ════════════════════════════════════════════════════════════════════════════ + +--- Build the PassCtx from parsed args. Reads each source file once at startup; +--- passes consume `src.text`, not the path (path is preserved for error reporting). +--- +--- @param args ParsedArgs +--- @return PassCtx +local function build_ctx(args) + local sources = {} + for _, path in ipairs(args.sources) do + local f = io.open(path, "r") + if not f then + io.stderr:write("ps1_meta: cannot open --source " .. path .. "\n") + os.exit(2) + end + local text = f:read("*a") + f:close() + + local dir = dirname(path) + local basename = basename_no_ext(path) + if #dir > 0 and (dir:sub(-1) == "/" or dir:sub(-1) == "\\") then + dir = dir:sub(1, -2) + end + + sources[#sources + 1] = { + path = path, + text = text, + dir = dir, + basename = basename, + } + end + + return { + sources = sources, + metadata_path = args.metadata, + shared = {}, + upstream = {}, + out_root = args.out_root, + project_root = args.project_root, + flags = {}, + dry_run = args.dry_run, + verbose = args.verbose, + } +end + +-- ════════════════════════════════════════════════════════════════════════════ +-- Topological sort (Kahn's algorithm + cycle detection) +-- ════════════════════════════════════════════════════════════════════════════ + +--- Topologically sort the requested pass set, augmented with all transitive deps. +--- Detects cycles and errors out with details. +--- +--- @param passes table +--- @param requested_set string[] +--- @return string[] -- execution order +local function topo_sort(passes, requested_set) + -- Step 1: dep-closure. + local needed = {} + for _, name in ipairs(requested_set) do needed[name] = true end + local changed = true + while changed do + changed = false + for name, _ in pairs(needed) do + local pass = passes[name] + if not pass then + error("unknown pass '" .. name .. "' requested") + end + for _, dep in ipairs(pass.deps) do + if not needed[dep] then + needed[dep] = true + changed = true + end + end + end + end + + -- Step 2: Kahn's algorithm. + local in_degree = {} + for name, _ in pairs(needed) do in_degree[name] = 0 end + for name, _ in pairs(needed) do + for _, dep in ipairs(passes[name].deps) do + if needed[dep] then + in_degree[name] = (in_degree[name] or 0) + 1 + end + end + end + + -- Seed with passes that have no unmet deps. Sort for determinism. + local ready = {} + for name, deg in pairs(in_degree) do + if deg == 0 then ready[#ready + 1] = name end + end + table.sort(ready) + + local order = {} + while #ready > 0 do + local n = table.remove(ready, 1) + order[#order + 1] = n + for name, _ in pairs(needed) do + if name ~= n then + for _, dep in ipairs(passes[name].deps) do + if dep == n then + in_degree[name] = in_degree[name] - 1 + if in_degree[name] == 0 then + ready[#ready + 1] = name + table.sort(ready) + end + end + end + end + end + end + + if #order ~= 0 then + for name, deg in pairs(in_degree) do + if deg > 0 then + error("dependency cycle detected involving pass '" .. name .. "'") + end + end + end + + return order +end + +-- ════════════════════════════════════════════════════════════════════════════ +-- ASCII dep graph renderer (Decision 6 in the spec) +-- ════════════════════════════════════════════════════════════════════════════ + +--- Render the dep graph as ASCII art. Output width capped at 78 columns. +--- Falls back to the simpler "Resolved dependency order" list only if +--- graph width exceeds terminal width. +--- +--- @param passes table +--- @param requested string[] -- originally-requested passes (subset of closed) +--- @param closed string[] -- dep-closed execution order +--- @return string +local function render_dep_graph(passes, requested, closed) + local lines = {} + local function add(s) lines[#lines + 1] = s end + + add("[ps1_meta] Resolved dependency order (closed under deps):") + for i, name in ipairs(closed) do + local p = passes[name] + local deps_str = (#p.deps == 0) and "(no deps)" or + "(deps: " .. table.concat(p.deps, ", ") .. ")" + add(string.format(" %d. %-22s %-45s [%s]", + i, name, deps_str, p.kind)) + end + add("") + + add("[ps1_meta] Pass graph (read top-to-bottom):") + add("") + add(" metadata.h") + add(" |") + add(" v") + add(" +-----------+ +-----------------+ +-----------------+") + add(" | word- |-->| components |-->| offsets |") + add(" | counts | +-----------------+ +-----------------+") + add(" | (load) | | ^") + add(" +-----------+ | |") + add(" | v |") + add(" | code//gen/.macs.h |") + add(" | (header - co-located for #include) |") + add(" | |") + add(" | +-----------------+ |") + add(" +---------->| annotation |--------------+") + add(" | +-----------------+ |") + add(" | | |") + add(" | v |") + add(" | build/gen/.errors.h |") + add(" | build/gen/.annotations.txt |") + add(" | (report - NOT #included) |") + add(" | |") + add(" | +-----------------+ |") + add(" +---------->| static-analysis |--------------+") + add(" +-----------------+") + add(" |") + add(" v") + add(" +---------------+") + add(" | report |") + add(" +---------------+") + add(" |") + add(" v") + add(" build/gen/annotation_validation.txt") + add(" (project summary)") + + return table.concat(lines, "\n") .. "\n" +end + +-- ════════════════════════════════════════════════════════════════════════════ +-- Main orchestrator +-- ════════════════════════════════════════════════════════════════════════════ + +--- Main entry point. Runs the requested passes in dep-topological order. +--- @param argv string[] +local function main(argv) + local ok, err = pcall(function() + local args = parse_args(argv) + local ctx = build_ctx(args) + + -- 1. Compute requested set + dep-closed set. + local requested = args.requested_set + local closed = topo_sort(PASSES, requested) + + -- 2. --dry-run: print dep order + ASCII graph, exit 0. + if args.dry_run then + io.write(render_dep_graph(PASSES, requested, closed)) + os.exit(0) + end + + -- 3. Run passes in topological order. + ctx.shared = {} + for _, pass_name in ipairs(closed) do + local pass = PASSES[pass_name] + local mod = require(pass.module) + local result = mod.run(ctx) + + -- Collect outputs + warnings into ctx.upstream. + ctx.upstream[pass_name] = ctx.upstream[pass_name] or {} + for _, out in ipairs(result.outputs or {}) do + table.insert(ctx.upstream[pass_name], out) + end + for _, warn in ipairs(result.warnings or {}) do + table.insert(ctx.upstream[pass_name], warn) + end + + -- Stop on errors (unless this is a non-stopping pass like report). + if (result.errors and #result.errors > 0) and PASS_KIND_STOP_ON_ERROR[pass.kind] then + for _, e in ipairs(result.errors) do + io.stderr:write(string.format("[%s] line %d: %s\n", + pass_name, e.line or 0, e.msg or "")) + end + os.exit(1) + end + end + end) + + if not ok then + io.stderr:write("[ps1_meta] internal error: " .. tostring(err) .. "\n") + os.exit(2) + end + + os.exit(0) +end + +main({...}) \ No newline at end of file diff --git a/scripts/word_count_eval.lua b/scripts/word_count_eval.lua new file mode 100644 index 0000000..d872e7d --- /dev/null +++ b/scripts/word_count_eval.lua @@ -0,0 +1,246 @@ +-- word_count_eval.lua +-- +-- Word-counting logic for the tape-atom metaprogram pipeline. +-- Used by: +-- - passes/components.lua (compute_component_word_count) +-- - passes/offsets.lua (scan_atom_body) +-- - passes/annotation.lua (TAPE_WORDS <-> WORD_COUNT drift check) +-- +-- This module ALSO exposes M.run(ctx) — the "word-counts" pass entry in +-- the PASSES table — which loads metadata.h + scans for existing +-- *.macs.h files into ctx.shared.word_counts. +-- +-- Coding standard: tabs (1/level), EmmyLua annotations, no regex, +-- Lua 5.3 compatible. + +-- ════════════════════════════════════════════════════════════════════════════ +-- Module-scope requires + package.path setup +-- ════════════════════════════════════════════════════════════════════════════ + +local script_path = arg and arg[0] or "?" +local last_sep = 0 +for i = 1, #script_path do + local c = script_path:sub(i, i) + if c == "/" or c == "\\" then last_sep = i end +end +local script_dir = last_sep == 0 and "./" or script_path:sub(1, last_sep) +package.path = script_dir .. "?.lua;" .. script_dir .. "?/init.lua;" .. package.path +package.cpath = "C:\\projects\\Pikuma\\ps1\\toolchain\\luajit-2.1\\lib\\lua\\5.1\\?.dll;" .. package.cpath + +local duffle = require("duffle") +local trim = duffle.trim +local read_ident = duffle.read_ident +local skip_ws_and_cmt = duffle.skip_ws_and_cmt +local load_word_counts = duffle.load_word_counts +local split_top_level_commas = duffle.split_top_level_commas +local is_space = duffle.is_space + +-- ════════════════════════════════════════════════════════════════════════════ +-- Type declarations +-- ════════════════════════════════════════════════════════════════════════════ + +--- @class WordCounts +--- @field [string] integer -- macro name -> word count + +--- @class PassCtx +--- @field sources SourceFile[] +--- @field metadata_path string +--- @field shared table +--- @field shared.word_counts WordCounts +--- @field out_root string +--- @field project_root string +--- @field upstream table +--- @field flags table +--- @field dry_run boolean +--- @field verbose boolean + +--- @class PassResult +--- @field outputs table[] +--- @field errors table[] +--- @field warnings table[] + +--- @class SourceFile +--- @field path string +--- @field text string +--- @field dir string +--- @field basename string + +-- ════════════════════════════════════════════════════════════════════════════ +-- Module exports +-- ════════════════════════════════════════════════════════════════════════════ + +local M = {} + +-- ┌────────────────────────────────────────────────────────────────────┐ +-- │ Shared utility: count_token_words │ +-- └────────────────────────────────────────────────────────────────────┘ + +--- Count words emitted by a single comma-separated token inside an atom body. +--- For most tokens (regular MIPS instructions) this returns 1. +--- For `mac_X(...)` calls, this returns the resolved word count from `wc` +--- (recursively if needed). For `nop2` etc., returns wc[name]. +--- For unknown macros, returns 1 and (optionally) warns. +--- +--- PORT NOTE: taken verbatim from tape_atom.offset_gen.meta.lua:130-141 +--- (`word_count_of_token`). Behavior is identical to preserve the +--- branch-offset fix from commit 98e27c2. +--- +--- @param token string -- a single token from split_top_level_commas +--- @param wc WordCounts -- the shared word-count table +--- @return integer +function M.count_token_words(token, wc) + local s = trim(token) + if s == "" then return 0 end + local name, after = read_ident(s, 1) + if not name then return 1 end + if wc[name] then return wc[name] end + local j = skip_ws_and_cmt(s, after) + if s:sub(j, j) == "(" then + io.stderr:write(" warning: unknown macro '" .. name .. "', assuming 1 word\n") + end + return 1 +end + +-- ┌────────────────────────────────────────────────────────────────────┐ +-- │ Shared utility: scan_dir │ +-- └────────────────────────────────────────────────────────────────────┘ + +--- Recursively scan a directory for files matching a glob suffix. +--- No regex per the no_regex constraint — uses plain byte matching +--- via `dir /b /s` on Windows. +--- +--- PORT NOTE: taken from tape_atom.offset_gen.meta.lua:432-443 +--- (`scan_dir`). Adapted: removed the hardcoded project_root derivation; +--- the caller passes `dir` explicitly. +--- +--- @param dir string -- directory to scan (absolute or relative) +--- @param suffix string -- file pattern, e.g. "*.macs.h" +--- @return string[] +function M.scan_dir(dir, suffix) + local results = {} + local p = io.popen('dir /b /s "' .. dir .. '\\' .. suffix .. '" 2>nul') + if not p then return results end + for raw_line in p:lines() do + local path = raw_line:gsub("\\", "/") + results[#results + 1] = path + end + p:close() + return results +end + +-- ┌────────────────────────────────────────────────────────────────────┐ +-- │ Shared utility: count_body_words │ +-- └────────────────────────────────────────────────────────────────────┘ + +--- Count words emitted by an entire atom body (a brace-delimited block). +--- Splits by top-level commas; for each token, delegates to count_token_words. +--- Handles `atom_label(name)` / `atom_offset(tag, name)` markers (record at +--- current pos, do NOT advance pos; if the marker call bundles an instruction +--- after it, count that instruction too). +--- +--- PORT NOTE: taken verbatim from tape_atom.offset_gen.meta.lua:207-239 +--- (`scan_atom_body`). Behavior is identical to preserve the branch-offset +--- fix from commit 98e27c2. +--- +--- @param body string -- brace-delimited atom body (without braces) +--- @param wc WordCounts -- the shared word-count table +--- @return integer -- total words +function M.count_body_words(body, wc) + local pos = 0 + for _, tok in ipairs(split_top_level_commas(body)) do + local k = 1 + local tlen = #tok + while k <= tlen and is_space(tok:sub(k, k)) do k = k + 1 end + local leading_ident = read_ident(tok, k) + if leading_ident == "atom_label" or leading_ident == "atom_offset" then + -- Marker call: record at current pos, do NOT advance pos. + -- But the source pattern may bundle the marker with the next + -- instruction on a new line (no top-level comma between them). + -- In that case, the rest of `tok` after the marker call is + -- a real instruction that must still be counted. + local marker_end = M.find_marker_call_end(tok) + if marker_end > 0 and marker_end < #tok then + local rest = trim(tok:sub(marker_end + 1)) + if rest ~= "" then + local rest_words = M.count_token_words(rest, wc) + pos = pos + rest_words + end + end + else + pos = pos + M.count_token_words(tok, wc) + end + end + return pos +end + +--- Find the end position (just past the closing ')') of the first +--- atom_label/atom_offset call in `tok`. Returns 0 if no such call. +--- Internal helper for count_body_words. +--- +--- PORT NOTE: taken from tape_atom.offset_gen.meta.lua:181-205 +--- (`find_marker_call_end`). +--- +--- @param tok string +--- @return integer -- 0 if no marker call found +function M.find_marker_call_end(tok) + local i = 1 + local len = #tok + while i <= len do + i = skip_ws_and_cmt(tok, i) + if i > len then break end + local c = tok:sub(i, i) + if is_space(c) then + i = i + 1 + elseif c == "/" then + -- comment — skip past it (delegated to duffle.skip_str_or_cmt) + local nx = duffle.skip_str_or_cmt(tok, i) + if nx > i then i = nx else i = i + 1 end + else + local ident, after = read_ident(tok, i) + if ident == "atom_label" or ident == "atom_offset" then + local j = skip_ws_and_cmt(tok, after) + if tok:sub(j, j) == "(" then + local _, end_paren = duffle.read_parens(tok, j) + return end_paren - 1 + end + return 0 + end + i = after or (i + 1) + end + end + return 0 +end + +-- ┌────────────────────────────────────────────────────────────────────┐ +-- │ Pass entry: M.run(ctx) — "word-counts" pass │ +-- └────────────────────────────────────────────────────────────────────┘ + +--- Load metadata.h + scan for existing *.macs.h files into +--- ctx.shared.word_counts. Loading the .macs.h files is idempotent: +--- entries from later (current-build) .macs.h files override +--- metadata.h entries of the same name. +--- +--- @param ctx PassCtx +--- @return PassResult +function M.run(ctx) + local wc = {} + + -- 1. Load metadata.h (the encoding-macro source of truth). + local meta_counts = load_word_counts(ctx.metadata_path) + for name, count in pairs(meta_counts) do wc[name] = count end + + -- 2. Scan project_root recursively for *.macs.h files (component-macro source). + local macs_files = M.scan_dir(ctx.project_root, "*.macs.h") + for _, macs_path in ipairs(macs_files) do + local ok, mc = pcall(load_word_counts, macs_path) + if ok and type(mc) == "table" then + for name, count in pairs(mc) do wc[name] = count end + end + end + + ctx.shared.word_counts = wc + + return { outputs = {}, errors = {}, warnings = {} } +end + +return M \ No newline at end of file