ed/pikuma_ps1
1
0
Fork 0
mirror of https://github.com/Ed94/pikuma_ps1.git synced 2026-08-25 02:20:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Review Pass: Type annotations.

Browse Source
This commit is contained in:
ed
2026-08-19 23:18:32 -04:00
parent c226e8a7d3
commit 449216967b
19 changed files with 5602 additions and 429 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/passes/annotation.lua
+112 -56
View File
@@ -10,7 +10,9 @@
-- Bootstrap follows the entry scripts; `scripts/duffle_paths.lua` sets package.path and package.cpath. See `ps1_meta.lua` for the rationale.
-- `debug.getinfo(1, "S").source` locates this file for standalone and orchestrated runs, then `duffle_paths.lua` returns the loaded `duffle` module.
--- @type string
local _bootstrap_dir = debug.getinfo(1, "S").source:match("^@?(.*[/\\])") or "./"
--- @type DuffleExport
local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua")
-- The annotation pass reads the source-derived registries from scan_source:
@@ -21,28 +23,8 @@ local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua")
-- Type declarations
-- ════════════════════════════════════════════════════════════════════════════
--- @class SourceFile
--- @field path string -- Absolute path to the source file
--- @field text string -- Full source text
--- @field dir string -- Directory containing the source
--- @field basename string -- Filename without extension
--- @field scan table -- Pre-scanned SourceScan payload (from duffle.scan_source)
--- @class PassCtx
--- @field sources SourceFile[]
--- @field metadata_path string
--- @field shared table
--- @field shared.word_counts table<string, integer>
--- @field out_root string
--- @field project_root string
--- @field upstream table<string, table>
--- @field flags table
--- @field verbose boolean
--- @class PassResult
--- @field outputs table[]
--- @field errors table[]
--- @field warnings table[]
-- SourceFile, PassCtx, PassResult, PassShared, Corpus: see ps1_meta.lua
-- SourceScan, AtomEntry, BindsEntry, RegTypeDefault, AtomViewEntry: see scan_source.lua
--- @class AtomAnnotation
--- @field atom_name string -- Atom name (scan.atom_infos row)
@@ -52,43 +34,51 @@ local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua")
--- @field writes string[] -- R_* names (write targets)
--- @field errors string[]|nil -- Parse-time errors from scan_source (atom_info body malformed)
--- @class DebugSkipMarker -- Sub-shape of scan_source.lua's @class DebugSkipMarker
--- @field marker_kind string -- Exact marker ident read from source. Only "atom_dbg_skip" (bare) is positive.
--- @field marker_line integer
--- @field args string|nil -- Trimmed text inside the parens (nil when has_parens is false)
--- @field has_parens boolean
--- @field is_bare boolean -- true iff marker_kind == "atom_dbg_skip" AND has_parens == false (the only positive form)
--- @field pending boolean -- true while awaiting the following declaration
--- @field superseded_by_marker_line integer|nil -- Set on a marker that was bumped out of the pending slot
--- @field target_kind string|nil -- "atom" | "comp_bare" | "comp_proc" | "unrelated" once observed
--- @class Finding
--- @field line integer -- Source line (or 0 for pass-level)
--- @field msg string -- Finding message
--- @class RegTypeOccurrence
--- @field reg string
--- @field type_name string
--- @field source_line integer
--- @class Findings
--- @field errors Finding[]
--- @field warnings Finding[]
--- @field info Finding[]
--- @field errors PassFinding[]
--- @field warnings PassFinding[]
--- @field info PassFinding[]
--- @class PipeCtx
--- @field atom_index table<string, AtomEntry> -- raw_name or name -> scan.atoms row (kind atom/atom_proc)
--- @field binds_index table<string, BindsStruct> -- Name -> BindsStruct
--- @field annot_counts table<string, integer> -- Name -> annotation count (for unique_annotation check)
--- @field types table<string, RegTypeDefault> -- From scan_source
--- @field atom_views table<string, AtomViewEntry> -- From scan_source
--- @field seen_defaults table<string, integer> -- Duplicate atom_dbg_reg_default detection
--- @field seen_field table<string, integer> -- Binds_* -> count of fields (set/checked by check_binds_no_duplicate_fields)
--- @field _scan SourceScan -- Full scan payload (typed-view sub-calls live here)
--- @field atom_index table<string, AtomEntry> -- raw_name or name -> scan.atoms row (kind atom/atom_proc)
--- @field binds_index table<string, BindsEntry>
--- @field annot_counts table<string, integer> -- bag: atom name -> annotation count
--- @field types table<string, RegTypeDefault>
--- @field atom_views table<string, AtomViewEntry>
--- @field seen_defaults table<string, integer> -- bag: register ident -> occurrence count
--- @field seen_field table<string, integer> -- bag: leftover field-count slot
--- @field _scan SourceScan
--- @field word_counts WordCounts|nil
--- @field register_alias_registry table<string, AliasEntry>|nil
--- @field type_name_registry table<string, TypeNameEntry>|nil
--- @field type_occurrences RegTypeOccurrence[]|nil
--- @field atom_infos_list AtomInfoEntry[]|nil
--- @field binds_list BindsEntry[]|nil
--- @class AnnotatedResult
--- @field atoms AtomEntry[]
--- @field annots AtomAnnotation[]
--- @field macros MacroEntry[]
--- @field binds BindsEntry[]
--- @field errors Finding[]
--- @field warnings Finding[]
--- @field info Finding[]
--- @field errors PassFinding[]
--- @field warnings PassFinding[]
--- @field info PassFinding[]
--- @field source string|nil
--- @class CheckRule
--- @field per_annot (fun(item: AtomAnnotation, pipe_ctx: PipeCtx, findings: Findings): nil)|nil
--- @class SourceScan
--- @field type_occurrences RegTypeOccurrence[]|nil
--- @class AnnotationPass
--- @field validate fun(ctx: PassCtx, src: SourceFile, corpus_pipe_ctx: PipeCtx|nil): AnnotatedResult
--- @field run fun(ctx: PassCtx): PassResult
-- ════════════════════════════════════════════════════════════════════════════
-- Per-check functions (the CHECK_RULES table's payload)
@@ -100,6 +90,7 @@ local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua")
--- @param info AtomAnnotation
--- @param pipe_ctx PipeCtx
--- @param findings Findings
--- @return nil
local function check_atom_decl_exists(info, pipe_ctx, findings)
if not pipe_ctx.atom_index[info.atom_name] then
findings.errors[#findings.errors + 1] = {
@@ -111,9 +102,12 @@ end
--- Check: Every atom may have AT MOST ONE annotation.
--- Post-loop: Needs full-corpus `annot_counts` from pipe_ctx.
--- @param _item AtomAnnotation|nil
--- @param pipe_ctx PipeCtx
--- @param findings Findings
--- @return nil
local function check_unique_annotation(_item, pipe_ctx, findings)
--- @type string, integer
for name, n in pairs(pipe_ctx.annot_counts) do
if n > 1 then
findings.errors[#findings.errors + 1] = {
@@ -129,6 +123,7 @@ end
--- @param info AtomAnnotation
--- @param pipe_ctx PipeCtx
--- @param findings Findings
--- @return nil
local function check_binds_struct_exists(info, pipe_ctx, findings)
if not info.binds then return end
if pipe_ctx.binds_index[info.binds] then return end
@@ -142,11 +137,14 @@ end
--- Check: TAPE_WORDS(mac_X, N) ↔ WORD_COUNT(mac_X, N) drift.
--- Three outcomes: missing (error), mismatch (error), match (info).
--- @param m MacroEntry
--- @param wc table<string, integer> -- Shared word-count table (from ctx.shared.word_counts)
--- @param m MacroEntry
--- @param pipe_ctx PipeCtx
--- @param findings Findings
--- @return nil
local function check_macro_word_drift(m, pipe_ctx, findings)
--- @type WordCounts
local wc = (pipe_ctx and pipe_ctx.word_counts) or {}
--- @type integer|nil
local declared = wc[m.name]
if not declared then
findings.errors[#findings.errors + 1] = {
@@ -173,9 +171,12 @@ end
--- @param _src SourceFile -- unused (kept for the per_source shape)
--- @param pipe_ctx PipeCtx
--- @param findings Findings
--- @return nil
local function check_semantic_reg_defaults(_src, pipe_ctx, findings)
-- Detect duplicate defaults using the ordered occurrence list (the out.types hash only retains the last declaration).
--- @type table<string, integer> -- bag: register ident -> first source line
local seen_first_line = {}
--- @type integer, RegTypeOccurrence
for _, occ in ipairs(pipe_ctx.type_occurrences or {}) do
if seen_first_line[occ.reg] == nil then
seen_first_line[occ.reg] = occ.source_line
@@ -188,8 +189,11 @@ local function check_semantic_reg_defaults(_src, pipe_ctx, findings)
}
end
end
--- @type table<string, AliasEntry>
local reg_registry = pipe_ctx.register_alias_registry or {}
--- @type table<string, TypeNameEntry>
local type_registry = pipe_ctx.type_name_registry or {}
--- @type string, RegTypeDefault
for reg, def in pairs(pipe_ctx.types or {}) do
if not reg_registry[reg] then
findings.errors[#findings.errors + 1] = {
@@ -223,11 +227,16 @@ end
--- @param _src SourceFile
--- @param pipe_ctx PipeCtx
--- @param findings Findings
--- @return nil
local function check_atom_reg_types(_src, pipe_ctx, findings)
--- @type table<string, AliasEntry>
local reg_registry = pipe_ctx.register_alias_registry or {}
--- @type table<string, TypeNameEntry>
local type_registry = pipe_ctx.type_name_registry or {}
--- @type integer, AtomInfoEntry
for _, ai in ipairs(pipe_ctx.atom_infos_list or {}) do
if ai.reg_type_overrides then
--- @type string, RegTypeOverride
for reg, ov in pairs(ai.reg_type_overrides) do
if not reg_registry[reg] then
findings.errors[#findings.errors + 1] = {
@@ -254,11 +263,14 @@ end
--- @param _src SourceFile
--- @param pipe_ctx PipeCtx
--- @param findings Findings
--- @return nil
local function check_atom_view_layout(_src, pipe_ctx, findings)
--- @type string, AtomViewEntry
for atom_name, view in pairs(pipe_ctx.atom_views or {}) do
if not view.binds_name then
-- The atom had atom_reg_types but no atom_view; no layout check needed.
else
--- @type BindsEntry|nil
local bs = pipe_ctx.binds_index[view.binds_name]
if not bs then
findings.errors[#findings.errors + 1] = {
@@ -280,15 +292,20 @@ local function check_atom_view_layout(_src, pipe_ctx, findings)
end
--- Check: Binds_* structs require unique field names because atom_view uses those names for typed-field lookup in gdb.
--- @param _src SourceFile
--- @param _src SourceFile
--- @param pipe_ctx PipeCtx
--- @param findings Findings
--- @return nil
local function check_binds_no_duplicate_fields(_src, pipe_ctx, findings)
--- @type integer, BindsEntry
for _, bs in ipairs(pipe_ctx.binds_list or {}) do
--- @type table<string, integer> -- bag: field name -> occurrence count
local seen = {}
--- @type integer, TypeField
for _, f in ipairs(bs.fields or {}) do
seen[f.name] = (seen[f.name] or 0) + 1
end
--- @type string, integer
for name, count in pairs(seen) do
if count > 1 then
findings.errors[#findings.errors + 1] = {
@@ -312,11 +329,14 @@ end
--- 5. pending + no target_kind -> dangling (no following declaration)
--- 6. unsupported target_kind -> marker precedes an unrelated declaration
--- Valid markers stamp `debug_skip` on whole-atom, bare-component, and proc-component declaration records in scan_source.lua.
--- @param marker DebugSkipMarker
--- @param marker DebugSkipMarker
--- @param _pipe_ctx PipeCtx -- Unused; kept for consistency with per_annot
--- @param findings Findings
--- @param findings Findings
--- @return nil
local function check_skip_marker(marker, _pipe_ctx, findings)
--- @type string
local kind = marker.marker_kind
--- @type integer
local line = marker.marker_line
-- Left `scan.debug_skip_markers` with production records for `atom_dbg_skip` only; other identifiers take the walker's unrelated branch.
@@ -372,12 +392,16 @@ end
--- @param _src SourceFile
--- @param pipe_ctx PipeCtx
--- @param findings Findings
--- @return nil
local function check_wave_context_migration(_src, pipe_ctx, findings)
if not (pipe_ctx.types and next(pipe_ctx.types)) then return end
if not (pipe_ctx.atom_infos_list) then return end
--- @type table<string, AliasEntry>
local reg_registry = pipe_ctx.register_alias_registry or {}
--- @type integer, AtomInfoEntry
for _, ai in ipairs(pipe_ctx.atom_infos_list) do
if ai.reg_type_overrides then
--- @type string, RegTypeOverride
for reg, _ in pairs(ai.reg_type_overrides) do
if not reg_registry[reg] then
findings.warnings[#findings.warnings + 1] = {
@@ -405,6 +429,7 @@ end
--
-- Adding a new check = 1 row here + 1 function above. The `validate()` dispatch loop never needs editing.
--- @type CheckRule[]
local CHECK_RULES = {
{ name = "atom_decl_exists", per_annot = check_atom_decl_exists },
{ name = "binds_struct_exists", per_annot = check_binds_struct_exists },
@@ -428,8 +453,11 @@ local CHECK_RULES = {
--- @param ctx PassCtx
--- @return PipeCtx
local function build_corpus_pipe_ctx(ctx)
--- @type PipeCtx
local view = duffle.corpus_view(ctx)
--- @type table<string, integer> -- bag: atom name -> annotation count
local annot_counts = {}
--- @type integer, AtomInfoEntry
for _, info in ipairs(view.atom_infos) do
if info and info.atom_name then
annot_counts[info.atom_name] = (annot_counts[info.atom_name] or 0) + 1
@@ -448,12 +476,16 @@ end
--- @return AnnotatedResult
local function validate(ctx, src, corpus_pipe_ctx)
corpus_pipe_ctx = corpus_pipe_ctx or build_corpus_pipe_ctx(ctx)
--- @type SourceScan
local scan = src.scan
-- Build a per-source pipe_ctx: shared lookups come from `corpus_pipe_ctx`, while declarations, bodies, types, views, defaults, and occurrences come from `src.scan`.
--- @type table<string, integer> -- bag: register ident -> occurrence count
local seen_defaults = {}; for reg, _ in pairs (scan.types or {}) do seen_defaults[reg] = (seen_defaults[reg] or 0) + 1 end
--- @type AtomInfoEntry[]
local atom_infos_list = {}; for _, ai in ipairs(scan.atom_infos or {}) do atom_infos_list[#atom_infos_list + 1] = ai end
--- @type PipeCtx
local pipe_ctx = {
atom_index = {},
binds_index = {},
@@ -468,22 +500,28 @@ local function validate(ctx, src, corpus_pipe_ctx)
register_alias_registry = corpus_pipe_ctx.register_alias_registry,
type_name_registry = corpus_pipe_ctx.type_name_registry,
}
--- @type AtomEntry[]
local atoms = {}
--- @type integer, AtomEntry
for _, a in ipairs(scan.atoms) do
if a.kind == "atom" or a.kind == "atom_proc" then
atoms[#atoms + 1] = a
pipe_ctx.atom_index[a.raw_name or a.name] = a
end
end
--- @type integer, BindsEntry
for _, b in ipairs(scan.binds) do pipe_ctx.binds_index[b.name] = b end
-- Findings live in a single struct with three lists (errors / warnings / info).
-- Each check writes to the list appropriate for its severity.
--- @type Findings
local findings = { errors = {}, warnings = {}, info = {} }
-- Lift parse-time errors already recorded in scan_source's atom_info payload into this pass's findings list.
--- @type integer, AtomInfoEntry
for _, info in ipairs(scan.atom_infos) do
if info.errors then
--- @type integer, string
for _, msg in ipairs(info.errors) do
findings.errors[#findings.errors + 1] = {
line = info.info_line,
@@ -494,6 +532,7 @@ local function validate(ctx, src, corpus_pipe_ctx)
end
-- THE per-annotation pipeline. ONE loop. CHECK_RULES dispatches per_annot rules.
--- @type integer, AtomInfoEntry
for _, info in ipairs(scan.atom_infos) do
duffle.run_check_rules(CHECK_RULES, "per_annot", info, pipe_ctx, findings)
end
@@ -503,13 +542,16 @@ local function validate(ctx, src, corpus_pipe_ctx)
-- scan_source records each marker in scan.debug_skip_markers; this loop validates each record independently and emits at most one error per marker.
-- Valid markers stamp `debug_skip = true` on the following atom or component declaration, which downstream consumers read directly.
--- @type DebugSkipMarker[]
local skip_markers = scan.debug_skip_markers or {}
--- @type integer, DebugSkipMarker
for _, marker in ipairs(skip_markers) do
duffle.run_check_rules(CHECK_RULES, "per_skip_marker", marker, pipe_ctx, findings)
end
-- Per-macro rules (TAPE_WORDS vs WORD_COUNT drift).
pipe_ctx.word_counts = corpus_pipe_ctx.word_counts
--- @type integer, MacroEntry
for _, m in ipairs(scan.macros) do
duffle.run_check_rules(CHECK_RULES, "per_macro", m, pipe_ctx, findings)
end
@@ -540,8 +582,7 @@ end
-- M.run — orchestrator entry
-- ════════════════════════════════════════════════════════════════════════════
--- @class M
--- @type AnnotationPass
local M = {}
-- Expose `validate` for downstream passes (e.g. report.lua) that need to re-render the per-source results into a per-MODULE report.
@@ -550,31 +591,46 @@ M.validate = validate
--- @param ctx PassCtx
--- @return PassResult
function M.run(ctx)
--- @type PassOutputEntry[]
local outputs = {}
--- @type PassFinding[]
local errors = {}
--- @type PassFinding[]
local warnings = {}
-- Build the shared pipe_ctx once for this run; every validate() call sees the same cross-source registries.
-- The corpus owns the canonical cross-source registries; per-source scans retain body / declaration ownership.
--- @type PipeCtx
local corpus_pipe_ctx = build_corpus_pipe_ctx(ctx)
--- @type Corpus
local corpus = ctx.shared.corpus
-- Group `corpus.sources_by_dir` by module, validate every source in each bucket, and emit one errors.h per directory.
--- @type table<string, SourceFile[]>
local by_dir = (corpus and corpus.sources_by_dir) or {}
--- @type string, SourceFile[]
for dir, dir_sources in pairs(by_dir) do
--- @type string
local dir_basename = dir:match("([^/\\]+)$") or dir
--- @type integer
local dir_atoms = 0
--- @type PassFinding[]
local dir_errors = {}
--- @type PassFinding[]
local dir_warnings = {}
--- @type integer, SourceFile
for _, src in ipairs(dir_sources) do
--- @type AnnotatedResult
local result = validate(ctx, src, corpus_pipe_ctx)
result.source = src.path -- tag for downstream rendering
dir_atoms = dir_atoms + #result.atoms
--- @type integer, PassFinding
for _, e in ipairs(result.errors) do
dir_errors[#dir_errors + 1] = { line = e.line, msg = e.msg, source = src.path }
errors [#errors + 1] = { line = e.line, msg = e.msg }
end
--- @type integer, PassFinding
for _, w in ipairs(result.warnings) do
dir_warnings[#dir_warnings + 1] = { line = w.line, msg = w.msg }
warnings [#warnings + 1] = { line = w.line, msg = w.msg }
scripts/passes/atoms_source_map.lua
+149 -26
View File
@@ -37,8 +37,11 @@
-- Bootstrap: load `duffle_paths.lua` via `debug.getinfo(1, "S").source`
-- (works both standalone + when require'd). `duffle_paths.lua` sets package.path then returns `require("duffle")`
-- at the bottom, so the dofile value IS the duffle module.
--- @type string
local _bootstrap_dir = debug.getinfo(1, "S").source:match("^@?(.*[/\\])") or "./"
--- @type DuffleExport
local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua")
--- @type ElfDwarfMod
local elf_dwarf = require("elf_dwarf")
-- ════════════════════════════════════════════════════════════════════════════
@@ -47,6 +50,7 @@ local elf_dwarf = require("elf_dwarf")
-- Format version emitted as the first line. Bump + add a migration test if the format changes;
-- the gdb runtime loader rejects mismatches (E2).
--- @type integer
local FORMAT_VERSION = 1
-- ════════════════════════════════════════════════════════════════════════════
@@ -54,29 +58,71 @@ local FORMAT_VERSION = 1
-- ════════════════════════════════════════════════════════════════════════════
--- @class AtomSourceMapCtx
--- @field shared table -- `ctx.shared`
--- @field shared.corpus table -- source-order registry; single writer is build_ctx
--- @field shared.word_counts table
--- @field out_root string -- output root (e.g. "build/gen")
--- @field flags table -- `ctx.flags`; reads `flags.gdb_runtime` + `flags.elf_path`
--- @field shared PassShared
--- @field out_root string
--- @field flags PassFlags
--- @field project_root string|nil
--- @class WordMapEntry
--- @field pos integer
--- @field line integer
--- @field text string
--- @field body_line integer
--- @field gpr_keys string[]|nil
--- @field invocation InvocationRecord|nil
--- @class NmAddr
--- @field [1] integer -- st_value
--- @field [2] integer -- st_size
--- @class GdbAtomRecord
--- @field idx integer|nil
--- @field name string
--- @field src_path string
--- @field file_base string
--- @field addr integer
--- @field size_bytes integer
--- @field words integer
--- @field entries WordMapEntry[]
--- @class ElfDwarfMod
--- @field read_nm fun(elf_path: Path): table<string, NmAddr>
--- @class AtomSourceMapPass
--- @field render_source_map fun(src: SourceFile): string
--- @field render_provenance fun(src: SourceFile, wc: WordCounts): string
--- @field render_atom_source_map fun(atom: AtomEntry): string
--- @field render_atom_provenance fun(atom: AtomEntry, wc: WordCounts, rel_path: string): string
--- @field run fun(ctx: PassCtx): PassResult
--- @class AtomEntry
--- @field paths AtomPaths|nil
-- ════════════════════════════════════════════════════════════════════════════
-- Atom-path renderers
-- ════════════════════════════════════════════════════════════════════════════
--- Join word boundaries (from `items`) to per-word call text + source lines (from `word_events`).
--- @param atom table
--- @return table[], integer
--- @param atom AtomEntry
--- @return WordMapEntry[]
--- @return integer
local function canonical_word_entries(atom)
--- @type AtomPaths
local paths = atom.paths or {}
--- @type WordEvent[]
local events = paths.word_events or {}
--- @type EmissionItem[]
local word_items = {}
--- @type integer, EmissionItem
for _, item in ipairs(paths.items or {}) do
if item.kind == "word" then word_items[#word_items + 1] = item end
end
--- @type WordMapEntry[]
local entries = {}
--- @type integer, WordEvent
for index, event in ipairs(events) do
--- @type EmissionItem
local item = word_items[index] or {}
entries[#entries + 1] = {
pos = event.i or (index - 1),
@@ -97,18 +143,25 @@ end
--- `WORD N CALL <src-path>:<src-line> RAW` (raw `.word` outside any mac_* component)
--- Component identity comes from the outermost invocation record; the count-table lookup confirms the component was declared in `corpus.word_counts`
--- (populated by word_count_eval + components passes).
--- @param src table
--- @param atom table
--- @param wc table -- identity alias of corpus.word_counts
--- @return string[], integer
--- @param src SourceFile
--- @param atom AtomEntry
--- @param wc WordCounts
--- @return string[]
--- @return integer
local function emit_provenance_stanza(src, atom, wc)
--- @type string[]
local lines = {}
--- @type string
local rel_path = src.path:gsub("\\\\", "/")
--- @type WordMapEntry[], integer
local entries, total = canonical_word_entries(atom)
lines[#lines + 1] = string.format('ATOM %s "%s" 0', atom.raw_name or atom.name, rel_path)
--- @type integer, WordMapEntry
for _, entry in ipairs(entries) do
--- @type InvocationRecord|nil
local inv = entry.invocation
--- @type integer|nil
local macro_count = inv and wc["mac_" .. inv.component_name]
if inv and macro_count ~= nil then
lines[#lines + 1] = string.format('WORD %d CALL %s:%d MACRO %s "%s:%d" BODY %d'
@@ -125,10 +178,11 @@ local function emit_provenance_stanza(src, atom, wc)
end
--- Render the full provenance file content for one source.
--- @param src table
--- @param wc table
--- @param src SourceFile
--- @param wc WordCounts
--- @return string
local function render_provenance(src, wc)
--- @type string[]
local lines = {}
lines[#lines + 1] = "# FORMAT_VERSION 1"
lines[#lines + 1] = "# auto-generated by ps1_meta.lua (passes/atoms_source_map.lua) — DO NOT EDIT"
@@ -138,13 +192,19 @@ local function render_provenance(src, wc)
lines[#lines + 1] = "# dwarf_injection to synthesize DW_TAG_inlined_subroutine instances + per-word"
lines[#lines + 1] = "# line program rows for native source-level step into component bodies."
--- @param atom AtomEntry
--- @return nil
local function append(atom)
--- @type string[]
local stanza = emit_provenance_stanza(src, atom, wc)
--- @type integer, string
for _, line in ipairs(stanza) do lines[#lines + 1] = line end
end
--- @type integer, AtomEntry
for _, atom in ipairs(src.scan.atoms or {}) do
if atom.paths then append(atom) end
end
--- @type integer, AtomEntry
for _, atom in ipairs(src.scan.raw_atoms or {}) do
if atom.paths then append(atom) end
end
@@ -154,16 +214,20 @@ end
--- Render one atom's stanza for the sourcemap.txt form (ATOM header line, N WORD lines, ENDATOM marker).
--- Returns (lines, total_words).
--- @param src table
--- @param atom table
--- @param wc table
--- @return string[], integer
--- @param src SourceFile
--- @param atom AtomEntry
--- @return string[]
--- @return integer
local function emit_atom_stanza(src, atom)
--- @type string[]
local lines = {}
--- @type string
local rel_path = src.path:gsub("\\\\", "/")
--- @type WordMapEntry[], integer
local entries, total = canonical_word_entries(atom)
lines[#lines + 1] = string.format('ATOM %s "%s" 0', atom.raw_name or atom.name, rel_path)
--- @type integer, WordMapEntry
for _, entry in ipairs(entries) do
lines[#lines + 1] = string.format("WORD %d LINE %d TEXT %s",
entry.pos, entry.line, entry.text)
@@ -175,21 +239,27 @@ end
--- Render the full source map file content for one source (one .atoms.sourcemap.txt per source). Mirrors offsets.lua's
--- `project_atoms` shape: scan.atoms + scan.raw_atoms, no kind filter.
--- @param src table
--- @param wc table
--- @param src SourceFile
--- @return string
local function render_source_map(src)
--- @type string[]
local lines = {}
lines[#lines + 1] = "# FORMAT_VERSION " .. FORMAT_VERSION
lines[#lines + 1] = "# auto-generated by ps1_meta.lua (passes/atoms_source_map.lua) — DO NOT EDIT"
--- @param atom AtomEntry
--- @return nil
local function append(atom)
--- @type string[]
local stanza = emit_atom_stanza(src, atom)
--- @type integer, string
for _, line in ipairs(stanza) do lines[#lines + 1] = line end
end
--- @type integer, AtomEntry
for _, atom in ipairs(src.scan.atoms or {}) do
if atom.paths then append(atom) end
end
--- @type integer, AtomEntry
for _, atom in ipairs(src.scan.raw_atoms or {}) do
if atom.paths then append(atom) end
end
@@ -211,19 +281,29 @@ end
--- Build the list of atoms with addresses + word entries. Shared helper for the gdb-runtime file emission.
--- @param ctx PassCtx
--- @return table[] -- list of {idx, name, src_path, file_base, addr, size_bytes, words, entries}
--- @return GdbAtomRecord[]
local function build_atom_table(ctx)
--- @type table<string, NmAddr>
local addrs = elf_dwarf.read_nm(ctx.flags.elf_path)
--- @type Corpus|nil
local corpus = ctx.shared and ctx.shared.corpus
--- @type GdbAtomRecord[]
local matched = {}
--- @type integer, SourceFile
for _, src in ipairs(corpus.source_order or {}) do
--- @type string
local file_base = src.path:match("([^/\\\\]+)$") or src.path
--- @param atom AtomEntry
--- @return nil
local function append(atom)
if not atom.paths then return end
--- @type string
local name = atom.raw_name or atom.name
--- @type NmAddr|nil
local info = addrs[name]
if not info then return end
--- @type WordMapEntry[], integer
local entries, total = canonical_word_entries(atom)
matched[#matched + 1] = {
name = name,
@@ -235,12 +315,18 @@ local function build_atom_table(ctx)
entries = entries,
}
end
--- @type integer, AtomEntry
for _, atom in ipairs((src.scan or {}).atoms or {}) do append(atom) end
--- @type integer, AtomEntry
for _, atom in ipairs((src.scan or {}).raw_atoms or {}) do append(atom) end
end
-- Deterministic order: sort by address (matches `nm` output ordering).
--- @param a GdbAtomRecord
--- @param b GdbAtomRecord
--- @return boolean
table.sort(matched, function(a, b) return a.addr < b.addr end)
--- @type integer, GdbAtomRecord
for i, a in ipairs(matched) do a.idx = i - 1 end
return matched
end
@@ -252,12 +338,14 @@ end
---
--- Why hardcoded per-atom: gdb's `$` substitution doesn't concat inside var names — `$__atom_name_$__i` in a `while`
--- loop resolves to one literal identifier, not `name_i`. Compile-time emission is the only path.
--- @param lines table -- output line buffer (mutated in place)
--- @param matched table -- list of atom records from `build_atom_table`
--- @param lines string[]
--- @param matched GdbAtomRecord[]
--- @return nil
local function append_gdb_commands(lines, matched)
-- ── tape_atoms ──
-- Hardcoded one printf per atom. No loop.
lines[#lines + 1] = "define tape_atoms"
--- @type integer, GdbAtomRecord
for _, a in ipairs(matched) do
-- gdb 12.1 quirk: literals in printf args require an attached target.
-- Use the per-atom convenience vars set above as printf args.
@@ -273,6 +361,7 @@ local function append_gdb_commands(lines, matched)
-- ── break_atom (generic) + per-atom break_atom_X ──
lines[#lines + 1] = "define break_atom"
lines[#lines + 1] = ' echo "Usage: break_atom_<exact_name> (pick from the list below)"'
--- @type integer, GdbAtomRecord
for _, a in ipairs(matched) do
lines[#lines + 1] = string.format(' printf " break_atom_%%-32s\\n", $__atom_name_%d', a.idx)
end
@@ -282,6 +371,7 @@ local function append_gdb_commands(lines, matched)
lines[#lines + 1] = "end"
lines[#lines + 1] = ""
--- @type integer, GdbAtomRecord
for _, a in ipairs(matched) do
lines[#lines + 1] = string.format("define break_atom_%s", a.name)
lines[#lines + 1] = string.format(" break *$__atom_addr_%d", a.idx)
@@ -296,6 +386,7 @@ local function append_gdb_commands(lines, matched)
-- ── step_atom / next_atom ──
-- Hardcoded one tbreak per atom. No loop.
lines[#lines + 1] = "define step_atom"
--- @type integer, GdbAtomRecord
for _, a in ipairs(matched) do
lines[#lines + 1] = string.format(" tbreak *$__atom_addr_%d", a.idx)
end
@@ -319,6 +410,7 @@ local function append_gdb_commands(lines, matched)
lines[#lines + 1] = "define where_in_atom"
lines[#lines + 1] = " set $__pc = (unsigned int)$pc"
lines[#lines + 1] = " set $__matched = 0"
--- @type integer, GdbAtomRecord
for _, a in ipairs(matched) do
-- Precompute end_addr (gdb 12.1's expression evaluator chokes on `addr + words*4`).
lines[#lines + 1] = string.format(" set $__end_%d = $__atom_addr_%d + $__atom_words_%d * 4", a.idx, a.idx, a.idx)
@@ -328,14 +420,17 @@ local function append_gdb_commands(lines, matched)
lines[#lines + 1] = string.format(" set $__word = ($__pc - $__atom_addr_%d) / 4", a.idx)
lines[#lines + 1] = string.format(' printf "word: %%d/%%d\\n", $__word, $__atom_words_%d', a.idx)
-- One inner-if per WORD entry. Each word's line + text hardcoded.
--- @type integer, WordMapEntry
for _, we in ipairs(a.entries) do
lines[#lines + 1] = string.format(" if $__word == %d", we.pos)
-- Escape TEXT for printf format string.
--- @type string
local escaped_text = we.text:gsub("%%", "%%%%"):gsub('"', '\\"')
lines[#lines + 1] = string.format(' printf "source: %%s:%%d %%s\\n", $__atom_file_%d, %d, "%s"', a.idx, we.line, escaped_text)
lines[#lines + 1] = " end"
end
-- Fallback for words beyond the source map (shouldn't happen if nm matches).
--- @type integer
local max_word = 0
if #a.entries > 0 then max_word = a.entries[#a.entries].pos end
lines[#lines + 1] = string.format(' if $__word > %d', max_word)
@@ -361,6 +456,7 @@ local function append_gdb_commands(lines, matched)
lines[#lines + 1] = " set $__in_atom = 0"
lines[#lines + 1] = " set $__did_step = 0"
lines[#lines + 1] = " set $__pc = (unsigned int)$pc"
--- @type integer, GdbAtomRecord
for _, a in ipairs(matched) do
-- Precompute end_addr in the convenience var (single expression gdb handles).
lines[#lines + 1] = string.format(" set $__end_%d = $__atom_addr_%d + $__atom_words_%d * 4", a.idx, a.idx, a.idx)
@@ -395,8 +491,10 @@ end
--- Emit the gdb-runtime file (post-link). Pure gdb scripting — addresses come from `mipsel-none-elf-nm -S`, get embedded
--- in `<ctx.out_root>/gdb_tape_atoms_runtime.gdb`, and load via `set $var = ...` + `define ... end` blocks at gdb source-time.
--- @param ctx PassCtx
--- @return nil
local function emit_gdb_runtime(ctx)
if not (ctx.flags and ctx.flags.gdb_runtime) then return end
--- @type string|nil
local elf_path = ctx.flags.elf_path
if not elf_path or elf_path == "" then
io.stderr:write("[atoms_source_map] --gdb-runtime requires --elf <elf>\n")
@@ -408,12 +506,14 @@ local function emit_gdb_runtime(ctx)
return
end
--- @type GdbAtomRecord[]
local matched = build_atom_table(ctx)
if #matched == 0 then
io.stderr:write("[atoms_source_map] --gdb-runtime: no atoms matched against nm symbols (stale scan?).\n")
return
end
--- @type string[]
local lines = {}
lines[#lines + 1] = "# Auto-generated by ps1_meta.lua (passes/atoms_source_map.lua)"
lines[#lines + 1] = "# DO NOT EDIT — re-run ps1_meta.lua --atoms-source-map --gdb-runtime to regenerate"
@@ -435,6 +535,7 @@ local function emit_gdb_runtime(ctx)
-- Per-atom convenience vars (used as printf args; literals aren't accepted
-- without an attached target on gdb 12.1).
--- @type integer, GdbAtomRecord
for _, a in ipairs(matched) do
lines[#lines + 1] = string.format('set $__atom_name_%d = "%s"', a.idx, gdb_escape(a.name))
lines[#lines + 1] = string.format("set $__atom_addr_%d = 0x%x", a.idx, a.addr)
@@ -451,10 +552,13 @@ local function emit_gdb_runtime(ctx)
-- Confirmation line for the source operator.
lines[#lines + 1] = 'printf "[gdb_tape_atoms] runtime loaded %d atoms from %s\\n", $__atom_count, $__elf_path'
--- @type string
local out_path
-- Move out of `<out_root>/gdb_tape_atoms_runtime.gdb` to `<out_root>/../gdb_tape_atoms_runtime.gdb` when the conventional `<out_root>` is `<build>/gen`
-- (any equivalent spelling — relative, absolute backslash, absolute forward-slash, trailing-separator variants).
-- This puts the gdb runtime alongside the ELF at `build/` rather than under the report subdir.
--- @param p string
--- @return boolean
local function ends_with_gen_dir(p)
if type(p) ~= "string" then return false end
return p:match("[/\\]gen[/\\]?$") ~= nil or p == "build/gen" or p == "build\\gen"
@@ -462,6 +566,7 @@ local function emit_gdb_runtime(ctx)
if ends_with_gen_dir(ctx.out_root) then
-- Strip the trailing `/gen` segment, then write the runtime script under `build/`.
-- e.g. "C:/projects/Pikuma/ps1/build/gen" -> "C:/projects/Pikuma/ps1/build".
--- @type string
local parent = ctx.out_root:gsub("[/\\]gen[/\\]?$", "")
out_path = parent .. "/gdb_tape_atoms_runtime.gdb"
else
@@ -476,6 +581,7 @@ end
-- M — module exports
-- ════════════════════════════════════════════════════════════════════════════
--- @type AtomSourceMapPass
local M = {}
-- Expose the pure render functions so `report.lua` and the focused tests can call them directly without triggering the file-emit path.
@@ -483,19 +589,26 @@ M.render_source_map = render_source_map
M.render_provenance = render_provenance
--- Render ONE atom's sourcemap stanza.
--- @param atom table -- atom record (must have `atom.paths` populated)
--- @param atom AtomEntry
--- @return string
function M.render_atom_source_map(atom)
assert(type(atom) == "table", "render_atom_source_map: atom must be a table")
assert(type(atom.paths) == "table", "render_atom_source_map: atom.paths must be a table")
--- @type WordMapEntry[], integer
local entries, total = canonical_word_entries(atom)
--- @type string[]
local lines = {}
lines[#lines + 1] = string.format("ATOM %s %d", (atom.raw_name or atom.name), total)
--- @type integer, WordMapEntry
for _, entry in ipairs(entries) do
--- @type string
local word_line = string.format("WORD %d LINE %d TEXT %s",
entry.pos, entry.line, entry.text)
--- @type string[]
local keys = {}
--- @type integer
for pos = 1, 16 do
--- @type string|nil
local k = entry.gpr_keys and entry.gpr_keys[pos]
if type(k) == "string" and k:sub(1, 7) == "reguse:" then
keys[#keys + 1] = k
@@ -514,19 +627,24 @@ end
---
--- `rel_path` is the source path (forward-slashes) embedded in every `CALL` line.
--- The .md caller (report.lua) is expected to derive this once per `## <source>` heading and pass it down for each atom in that source.
--- @param atom table -- atom record (must have `atom.paths` populated)
--- @param wc table -- identity alias of `corpus.word_counts`
--- @param rel_path string -- source path (forward-slashes) for `CALL` fields
--- @param atom AtomEntry
--- @param wc WordCounts
--- @param rel_path string
--- @return string
function M.render_atom_provenance(atom, wc, rel_path)
assert(type(atom) == "table", "render_atom_provenance: atom must be a table")
assert(type(atom.paths) == "table", "render_atom_provenance: atom.paths must be a table")
assert(type(rel_path) == "string", "render_atom_provenance: rel_path must be a string")
--- @type WordMapEntry[], integer
local entries, total = canonical_word_entries(atom)
--- @type string[]
local lines = {}
lines[#lines + 1] = string.format("ATOM %s %d", (atom.raw_name or atom.name), total)
--- @type integer, WordMapEntry
for _, entry in ipairs(entries) do
--- @type InvocationRecord|nil
local inv = entry.invocation
--- @type integer|nil
local macro_count = inv and wc and wc["mac_" .. inv.component_name]
if inv and macro_count ~= nil then
lines[#lines + 1] = string.format('WORD %d CALL %s:%d MACRO %s "%s:%d" BODY %d'
@@ -546,16 +664,21 @@ end
--- @param ctx PassCtx
--- @return PassResult
function M.run(ctx)
--- @type PassOutputEntry[]
local outputs = {}
--- @type PassFinding[]
local errors = {}
--- @type PassFinding[]
local warnings = {}
--- @type Corpus|nil
local corpus = ctx.shared and ctx.shared.corpus
if type(corpus) ~= "table" or type(corpus.source_order) ~= "table" then
error("atoms_source_map.run requires ctx.shared.corpus.source_order (canonical corpus).", 0)
end
-- Word counts come from `corpus.word_counts` (populated by word_count_eval + components passes).
--- @type WordCounts
local wc = corpus.word_counts or {}
if not next(wc) then
warnings[#warnings + 1] = {
scripts/passes/auto_reg.lua
+104 -4
View File
@@ -18,12 +18,26 @@
--- Pool exhaustion: If a phase declares more `R_<Sym>` mappings than the 10-register pool can hold,
--- emit `phase_register_pool_exhausted` as a build-stopping error.
--- @class AutoRegResult
--- @field outputs table[] -- {kind=, path=} entries
--- @field errors table[] -- {line=, msg=} entries (build-stops)
--- @field warnings table[] -- {line=, msg=} entries (build-continues)
--- @alias GprIdent string
--- @class GprAllocMap
--- @field [string] GprIdent -- bag: auto-reg symbol -> physical GPR
--- @class AutoRegOutput
--- @field auto_reg_h string
--- @class AutoRegResult
--- @field outputs AutoRegOutput[]
--- @field errors PassFinding[]
--- @field warnings PassFinding[]
--- @class AutoRegPass
--- @field run fun(ctx: PassCtx): AutoRegResult
--- @field POOL GprIdent[]
--- @type string
local _bootstrap_dir = debug.getinfo(1, "S").source:match("^@?(.*[/\\])") or "./"
--- @type DuffleExport
local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua")
--- ════════════════════════════════════════════════════════════════════════════
@@ -42,6 +56,7 @@ local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua")
--- R_K0/K1 (codes 26-27) — Kernel / interrupt handler reserves. Never touched by user code.
--- R_GP/SP/FP/RA (codes 28-31) — R_SP/R_FP/R_RA are tape-runtime carriers between tape_enter and tape_exit; R_GP stays the host global pointer.
---
--- @type GprIdent[]
local POOL = {
"R_V0", "R_V1",
"R_T0", "R_T1", "R_T2", "R_T3",
@@ -57,6 +72,7 @@ local POOL = {
-- Only the POOL entries matter for auto_reg — non-pool aliases
-- (R_AT=1, R_A0..A3=4..7, R_T8=24, R_T9=25, R_K0/K1=26..27, R_GP/SP/FP/RA=28..31)
-- are deliberately omitted — see the comment block above for the WHY of each exclusion.
--- @type table<integer, GprIdent> -- bag: MIPS GPR code -> POOL ident
local INT_CODE_TO_POOL_GPR = {
[2] = "R_V0", [3] = "R_V1",
[4] = "R_A0", [5] = "R_A1", [6] = "R_A2", [7] = "R_A3",
@@ -68,8 +84,12 @@ local INT_CODE_TO_POOL_GPR = {
}
-- Stable sort for deterministic allocation order.
--- @param tbl table<string, string> -- bag: key set only; values unused
--- @return string[]
local function stable_sort_keys(tbl)
--- @type string[]
local keys = {}
--- @type string
for k in pairs(tbl) do keys[#keys + 1] = k end
table.sort(keys)
return keys
@@ -77,15 +97,25 @@ end
-- Allocate one phase's auto-reg mappings.
-- Returns (allocated_map, errors). On pool exhaustion, errors is populated and the function halts.
--- @param phase_label string
--- @param decls table<string, string> -- bag: auto-reg symbol -> decl payload
--- @return GprAllocMap
--- @return PassFinding[]
local function allocate_phase(phase_label, decls)
-- Deep-copy POOL into a fresh sequence table. The original `table.unpack and table.unpack(POOL) or { unpack(POOL) }`
-- idiom wraps the unpacked values in a single inner table under LuaJIT 5.1 (`table.unpack` is nil; the `or` returns one value),
-- which corrupts the pool into `{ {R_T0, R_T1, ...} }` — making `table.remove(pool, 1)` return the inner table on iteration.
--- @type GprIdent[]
local pool = {}
--- @type integer
for i = 1, #POOL do pool[i] = POOL[i] end
--- @type GprAllocMap
local result = {}
--- @type PassFinding[]
local errors = {}
--- @type integer, string
for _, sym in ipairs(stable_sort_keys(decls)) do
--- @type GprIdent|nil
local next_gpr = table.remove(pool, 1)
if not next_gpr then
errors[#errors + 1] = {
@@ -110,12 +140,19 @@ end
-- Each entry's `code` is the integer MIPS GPR number (0..31); INT_CODE_TO_POOL_GPR translates it back to the physical GPR ident.
-- Aliases whose `code` points to a non-POOL GPR (e.g. R_S0, R_T8, R_K1) are ignored —
-- they don't affect the auto_reg pool, and they're already excluded from POOL above.
--- @param corpus Corpus
--- @return table<GprIdent, boolean>
--- @return table<string, GprIdent>
local function build_user_pins(corpus)
--- @type table<GprIdent, boolean> -- bag: pinned physical GPR -> true
local user_pinned = {}
--- @type table<string, GprIdent> -- bag: alias ident -> physical GPR
local alias_to_gpr = {}
if not corpus.register_alias_registry then return user_pinned, alias_to_gpr end
--- @type string, AliasEntry
for alias_name, alias_entry in pairs(corpus.register_alias_registry) do
if alias_entry.has_atom_reg and alias_entry.code then
--- @type GprIdent|nil
local gpr = INT_CODE_TO_POOL_GPR[alias_entry.code]
if gpr then
user_pinned[gpr] = true
@@ -133,22 +170,32 @@ end
--- Clash-detection and source-pool-exclusion logic only needs the presence of each GPR (boolean test),
--- but keeping count preserves the original find_hardcoded_rn shape so callers can switch without churn.
--- The alias pattern is sorted lexicographically to keep the regex deterministic.
--- @param body_text string
--- @param alias_to_gpr table<string, GprIdent> -- bag: alias ident -> physical GPR
--- @return table<GprIdent, integer>
local function find_used_gprs(body_text, alias_to_gpr)
--- @type table<GprIdent, integer> -- bag: physical GPR -> hit count
local found = {}
-- (a) Hardcoded physical GPRs (R_T0..R_T7, R_V0..R_V1, R_A0..R_A3, R_S0..R_S7).
--- @type GprIdent
for gpr in body_text:gmatch("(R_T%d+|R_V%d+|R_A%d+|R_S%d+)") do
found[gpr] = (found[gpr] or 0) + 1
end
-- (b) Alias references (R_<Alias>) resolved to physical GPRs via the registry.
-- Sorted by name so the regex is byte-stable across runs.
if alias_to_gpr and next(alias_to_gpr) then
--- @type string[]
local aliases = {}
--- @type string
for alias_name in pairs(alias_to_gpr) do
aliases[#aliases + 1] = alias_name
end
table.sort(aliases)
--- @type string
local pattern = "(" .. table.concat(aliases, "|") .. ")"
--- @type string
for alias_name in body_text:gmatch(pattern) do
--- @type GprIdent|nil
local gpr = alias_to_gpr[alias_name]
if gpr and not found[gpr] then
found[gpr] = 1
@@ -159,10 +206,17 @@ local function find_used_gprs(body_text, alias_to_gpr)
end
-- Emit one gen/auto_reg.h header per directory.
--- @param out_dir string
--- @param dir string
--- @param sources SourceFile[]
--- @param mappings GprAllocMap
--- @return string|nil
local function emit_auto_reg_h(out_dir, dir, sources, mappings)
if not mappings or next(mappings) == nil then return end
--- @type string
local out_path = out_dir .. "/" .. "auto_reg.h"
duffle.ensure_dir(out_dir)
--- @type string[]
local lines = {
"#ifdef INTELLISENSE_DIRECTIVES",
"#pragma once",
@@ -170,14 +224,18 @@ local function emit_auto_reg_h(out_dir, dir, sources, mappings)
"// Auto-generated by ps1_meta.lua (passes/auto_reg.lua) — DO NOT EDIT",
"// Directory: " .. dir:gsub("/", "\\"),
}
--- @type integer, SourceFile
for _, src in ipairs(sources) do
lines[#lines + 1] = "// source: " .. src.path
end
lines[#lines + 1] = "// Per-phase register allocations resolved by the lua pass."
lines[#lines + 1] = "// R_<Sym>_Code = <chosen GPR's _Code constant> for every marker in this directory."
lines[#lines + 1] = ""
--- @type integer, string
for _, sym in ipairs(stable_sort_keys(mappings)) do
--- @type GprIdent
local gpr = mappings[sym]
--- @type string
local gpr_code = gpr .. "_Code"
lines[#lines + 1] = "#define " .. sym .. "_Code " .. gpr_code
end
@@ -191,15 +249,20 @@ end
-- Pass entry
-- ════════════════════════════════════════════════════════════════════════════
--- @type AutoRegPass
local M = {}
--- @param ctx PassCtx
--- @return AutoRegResult
function M.run(ctx)
--- @type AutoRegOutput[]
local outputs = {}
--- @type PassFinding[]
local errors = {}
--- @type PassFinding[]
local warnings = {}
--- @type Corpus|nil
local corpus = ctx.shared and ctx.shared.corpus
if type(corpus) ~= "table" then
error("auto_reg.run requires ctx.shared.corpus", 0)
@@ -210,16 +273,22 @@ function M.run(ctx)
-- MUST NOT be allocated to any auto-reg marker — they're preserved across atoms by the wave-context discipline.
-- The corpus's register_alias_registry is the source of truth for these opt-in pins.
-- Body references to those aliases (via alias_to_gpr) are also excluded on a per-atom basis in step 2 below.
--- @type table<GprIdent, boolean>, table<string, GprIdent>
local user_pinned, alias_to_gpr = build_user_pins(corpus)
-- 1. Allocate phase pools first (phase declarations take precedence over per-atom declarations).
--- @type table<string, GprAllocMap> -- bag: phase_label -> alloc map
local phase_allocations = {}
--- @type string, table<string, string>
for phase_label, decls in pairs(corpus.phase_auto_regs or {}) do
--- @type GprAllocMap, PassFinding[]
local mapping, errs = allocate_phase(phase_label, decls)
--- @type string, GprIdent
for sym, gpr in pairs(mapping) do
phase_allocations[phase_label] = phase_allocations[phase_label] or {}
phase_allocations[phase_label][sym] = gpr
end
--- @type integer, PassFinding
for _, e in ipairs(errs) do
errors[#errors + 1] = e
end
@@ -229,15 +298,21 @@ function M.run(ctx)
-- Otherwise, allocate a private pool for the atom.
-- The phase membership is in `corpus.atom_phases[phase_label].atoms` (an array of atom names declared via `atom_phase(<phase>)`
-- in the atom's `atom_info` line). Build a reverse map `atom_name -> phase_label` so the lookup is O(1) per atom scope.
--- @type table<AtomName, string> -- bag: atom name -> phase label
local atom_name_to_phase = {}
--- @type string, AtomPhaseGroup
for phase_label, entry in pairs(corpus.atom_phases or {}) do
--- @type integer, AtomName
for _, atom_name in ipairs(entry.atoms or {}) do
atom_name_to_phase[atom_name] = phase_label
end
end
--- @type table<AtomName, GprAllocMap> -- bag: atom scope -> alloc map
local atom_allocations = {}
--- @type AtomName, table<string, string>
for atom_scope, decls in pairs(corpus.atom_auto_regs or {}) do
--- @type string|nil
local phase_label = atom_name_to_phase[atom_scope]
-- Build the atom's source pool: start with the full POOL, subtract:
-- (a) every GPR already committed (phase allocations + prior atom allocations)
@@ -248,25 +323,36 @@ function M.run(ctx)
-- the original `source_pool = phase_allocations[phase_label]` form used the phase
-- allocation MAP as a pool, but that map has no array part, so `table.remove(source_pool, 1)`
-- returned nil and every atom-with-phase marker errored with `phase_register_pool_exhausted`.
--- @type table<GprIdent, boolean> -- bag: committed or body-referenced GPR -> true
local used = {}
--- @type integer, GprAllocMap
for _, m in pairs(phase_allocations) do for _, gpr in pairs(m) do used[gpr] = true end end
--- @type integer, GprAllocMap
for _, m in pairs(atom_allocations) do for _, gpr in pairs(m) do used[gpr] = true end end
-- (c) Body references — scan the atom body for hardcoded + alias-resolved GPRs.
-- Folded into `used` so the source_pool exclusion is a single check.
--- @type AtomEntry|nil
local atom = corpus.atoms_by_name and corpus.atoms_by_name[atom_scope]
if atom and atom.body then
--- @type table<GprIdent, integer>
local body_used = find_used_gprs(atom.body, alias_to_gpr)
--- @type GprIdent
for gpr in pairs(body_used) do used[gpr] = true end
end
--- @type GprIdent[]
local source_pool = {}
--- @type integer, GprIdent
for _, gpr in ipairs(POOL) do
-- Exclude (a) prior commitments, (b) USER-PINNED GPRs (wave-context carriers declared via atom_reg + _Code defs, preserved across atoms globally).
if not used[gpr] and not user_pinned[gpr] then
source_pool[#source_pool + 1] = gpr
end
end
--- @type GprAllocMap
local result = {}
--- @type integer, string
for _, sym in ipairs(stable_sort_keys(decls)) do
--- @type GprIdent|nil
local next_gpr = table.remove(source_pool, 1)
if not next_gpr then
errors[#errors + 1] = {
@@ -289,10 +375,14 @@ function M.run(ctx)
-- This warning is kept as a defensive safety net for cases the body scanner might miss
-- (e.g. macros that expand to register references the scanner cannot resolve).
-- For each resolved (scope, sym) -> R_Tn mapping, scan the atom body source for used GPRs.
--- @type AtomName, GprAllocMap
for atom_scope, decls in pairs(atom_allocations) do
--- @type AtomEntry|nil
local atom = corpus.atoms_by_name and corpus.atoms_by_name[atom_scope]
if atom and atom.body then
--- @type table<GprIdent, integer>
local used_in_body = find_used_gprs(atom.body, alias_to_gpr)
--- @type string, GprIdent
for sym, allocated_gpr in pairs(decls) do
if used_in_body[allocated_gpr] and used_in_body[allocated_gpr] > 0 then
warnings[#warnings + 1] = {
@@ -308,26 +398,36 @@ function M.run(ctx)
-- 4. Emit per-directory gen/auto_reg.h.
-- For each source directory that has atom_auto_regs or phase_auto_regs entries, emit one header.
--- @type table<string, SourceFile[]>
local sources_by_dir = corpus.sources_by_dir or {}
--- @type string, SourceFile[]
for dir, sources in pairs(sources_by_dir) do
--- @type GprAllocMap
local per_dir_mappings = {}
--- @type integer, SourceFile
for _, src in ipairs(sources) do
-- Collect every (sym -> gpr) entry that originated from a source in this directory.
-- `src.scan.atom_auto_regs` is keyed by ATOM SCOPE NAME; `pairs(t)` iterates KEYS so `scope_name` here is the scope ident (e.g. "cube_g4_face").
-- The previous `for _, scan_atom_auto` form silently assigned the VALUE (a `{sym = sym}` table) to the variable,
-- which made `atom_allocations[scan_atom_auto]` a table-indexed lookup that never resolved.
--- @type string
for scope_name in pairs(src.scan and src.scan.atom_auto_regs or {}) do
--- @type string, GprIdent
for sym, gpr in pairs(atom_allocations[scope_name] or {}) do
per_dir_mappings[sym] = gpr
end
end
--- @type string
for scope_name in pairs(src.scan and src.scan.phase_auto_regs or {}) do
--- @type string, GprIdent
for sym, gpr in pairs(phase_allocations[scope_name] or {}) do
per_dir_mappings[sym] = gpr
end
end
end
--- @type string
local out_dir = dir .. "/gen"
--- @type string|nil
local out_path = emit_auto_reg_h(out_dir, dir, sources, per_dir_mappings)
if out_path then outputs[#outputs + 1] = { auto_reg_h = out_path } end
end
scripts/passes/components.lua
+203 -53
View File
@@ -22,7 +22,9 @@
-- Uses `debug.getinfo` to find this file's own directory, so it works both standalone and when require'd from the orchestrator.
-- Bootstrap: load `duffle_paths.lua` via `debug.getinfo(1, "S").source` (works both standalone + when require'd).
-- duffle_paths.lua sets package.path then returns `require("duffle")` at the bottom, so the dofile value IS the duffle module.
--- @type string
local _bootstrap_dir = debug.getinfo(1, "S").source:match("^@?(.*[/\\])") or "./"
--- @type DuffleExport
local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua")
-- ════════════════════════════════════════════════════════════════════════════
@@ -30,62 +32,75 @@ local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua")
-- ════════════════════════════════════════════════════════════════════════════
-- Atom component declaration identifiers.
--- @type string
local ATOM_COMP_PROC = "MipsAtomComp_Proc_"
--- @type string
local MIPS_ATOM = "Slice_MipsCode" -- prefix on the function declaration that wraps an AtomComp_Proc_
-- Component-name prefixes.
--- @type string
local AC_PREFIX = "ac_" -- arg to MipsAtomComp_(ac_X); the X is the atom name
--- @type integer
local AC_PREFIX_LEN = 3
--- @type string
local MAC_PREFIX = "mac_" -- prefix on generated macros; the rest is the atom name
--- @type integer
local MAC_PREFIX_LEN = 4
-- ASCII byte values used in tokenization.
--- @type integer
local BYTE_NEWLINE = 10
--- @type integer
local BYTE_SLASH = 47
-- Output gen subdirectory + filename (per-directory aggregation; the directory name is the namespace).
--- @type string
local GEN_SUBDIR = "gen"
--- @type string
local MACS_FILENAME = "macs.h"
-- ════════════════════════════════════════════════════════════════════════════
-- Type declarations
-- ════════════════════════════════════════════════════════════════════════════
--- @class SourceFile
--- @field path string -- Absolute path to the source file
--- @field text string -- Full source text
--- @field dir string -- Directory containing the source
--- @field basename string -- Filename without extension
--- @field scan table -- Pre-scanned SourceScan payload (from duffle.scan_source)
--- @class PassCtx
--- @field sources SourceFile[] -- All source files in the build
--- @field metadata_path string -- Path to word_count.metadata.h
--- @field shared table -- Cross-pass shared state
--- @field out_root string -- Output root (e.g. "build/gen")
--- @field project_root string -- Project root (e.g. "code/")
--- @field upstream table<string, table> -- Per-pass upstream outputs
--- @field flags table -- CLI flags
--- @field verbose boolean -- Log diagnostic info
--- @class PassResult
--- @field outputs table[] -- {kind=, path=} entries describing emit files
--- @field errors table[] -- {line=, msg=} entries; build-stops
--- @field warnings table[] -- {line=, msg=} entries; build-succeeds
-- SourceFile, PassCtx, PassResult: see ps1_meta.lua
-- DuffleExport: see duffle.lua
-- SourceScan, AtomEntry, CorpusCollision, CollisionSite: see scan_source.lua
-- BodyToken: see emission_model.lua
-- WordCounts: see word_count_eval.lua
-- ComponentBodyEntry: see duffle_emit.lua
-- InstructionRow, GteCommandRow: see duffle_isa.lua
--- @class Component
--- @field name string -- Atom name (without `ac_` prefix)
--- @field body string -- Brace-delimited body (without the braces)
--- @field args string|nil -- Function-args string (function form only)
--- @field line integer -- Source line of the declaration
--- @field comment string|nil -- Scanner-owned `declaration_comment`; the components pass reads it from the scanner record
--- @field kind string -- "comp_bare" | "comp_proc" (atom_proc is NOT a component — see `project_components`)
--- @field debug_skip boolean -- Mirror of `a.debug_skip` (scanner-owned); true iff a bare `atom_dbg_skip` marker immediately preceded the declaration
--- @field name string -- Atom name (without `ac_` prefix)
--- @field body string -- Brace-delimited body (without the braces)
--- @field body_off integer|nil -- Byte offset of body[1] in source
--- @field body_tokens BodyToken[]|nil
--- @field args string|nil -- Function-args string (function form only)
--- @field arg_names string[]|nil -- Formal names with leading `ab` dropped
--- @field line integer -- Source line of the declaration
--- @field comment string|nil -- Scanner-owned `declaration_comment`; the components pass reads it from the scanner record
--- @field kind string -- "comp_bare" | "comp_proc" (atom_proc is NOT a component — see `project_components`)
--- @field debug_skip boolean -- Mirror of `a.debug_skip` (scanner-owned); true iff a bare `atom_dbg_skip` marker immediately preceded the declaration
--- @class ComponentMeta
--- @field cycle_cost integer
--- @field gp0_contrib integer
--- @class ComponentMetaMap
--- @field [string] ComponentMeta -- bag: bare component name -> meta
--- @class MacsOutput
--- @field macs_h string
--- @class ComponentsPass
--- @field run fun(ctx: PassCtx): PassResult
-- ════════════════════════════════════════════════════════════════════════════
-- Local helpers (file I/O + path normalization)
-- ════════════════════════════════════════════════════════════════════════════
--- @type ComponentsPass
local M = {}
-- ════════════════════════════════════════════════════════════════════════════
@@ -108,6 +123,7 @@ local M = {}
--- @param before_pos integer
--- @return string|nil
local function find_function_args_for(source, name, before_pos)
--- @type string|nil, string|nil
local _, args_inner = duffle.find_function_decl_for(source, before_pos, #MIPS_ATOM)
return args_inner
end
@@ -124,9 +140,13 @@ end
--- @return string[]|nil
local function extract_arg_names(args_str)
if not args_str or args_str == "" then return nil end
--- @type string[]
local names = {}
--- @type string[]
local tokens = duffle.split_top_level_commas(args_str)
--- @type integer, string
for _, tok in ipairs(tokens) do
--- @type string
local trimmed = duffle.trim(tok)
if trimmed ~= "" then
-- Strip trailing block comment (/* ... */) from the token, if present.
@@ -134,13 +154,16 @@ local function extract_arg_names(args_str)
-- not block comments embedded WITHIN a token between a parameter and a trailing comma.
-- Without this strip, the identifier-walk below stops at the `/` of `*/` and returns
-- the wrong name (or nothing). See `test_extract_arg_names_handles_trailing_block_comments`.
--- @type integer
local trimmed_end = #trimmed
if trimmed_end >= 2 and trimmed:sub(trimmed_end - 1, trimmed_end) == "*/" then
-- Find the matching `/*` that opens the trailing comment.
-- Walk back from the `*/` looking for `/*` (whitespace + `/*`).
--- @type integer
local close_pos = trimmed_end - 1 -- position of the second-to-last char
-- Walk back: skip trailing whitespace, then look for the `/*` opener.
while close_pos > 1 do
--- @type string
local ch = trimmed:sub(close_pos, close_pos)
if ch == " " or ch == "\t" or ch == "\n" or ch == "\r" then
close_pos = close_pos - 1
@@ -149,7 +172,9 @@ local function extract_arg_names(args_str)
end
end
-- Now scan back from close_pos for the `/*` opener (slashes are at close_pos-1 and close_pos-2).
--- @type integer|nil
local opener_pos = nil
--- @type integer
local scan = close_pos - 3
while scan >= 1 do
if trimmed:sub(scan, scan + 1) == "/*" then
@@ -169,8 +194,10 @@ local function extract_arg_names(args_str)
trimmed_end = #trimmed
if trimmed_end >= 4 and trimmed:sub(trimmed_end, trimmed_end) == "]" then
-- Walk back: skip digits, expect `[`.
--- @type integer
local bracket_pos = trimmed_end - 1
while bracket_pos > 1 do
--- @type string
local ch = trimmed:sub(bracket_pos, bracket_pos)
if ch >= "0" and ch <= "9" then
bracket_pos = bracket_pos - 1
@@ -185,8 +212,10 @@ local function extract_arg_names(args_str)
if trimmed == "" then goto continue end
-- Find the identifier at the end: walk back over trailers (whitespace + `*` + `[]`),
-- then walk back over the identifier chars (alnum + `_`).
--- @type integer
local ident_end = #trimmed
while ident_end > 0 do
--- @type string
local ch = trimmed:sub(ident_end, ident_end)
if ch == " " or ch == "\t" or ch == "*" or ch == "]" or ch == "[" then
ident_end = ident_end - 1
@@ -194,8 +223,10 @@ local function extract_arg_names(args_str)
break
end
end
--- @type integer
local ident_start = ident_end
while ident_start > 0 do
--- @type string
local ch = trimmed:sub(ident_start, ident_start)
if duffle.is_alnum_byte(string.byte(ch)) or ch == "_" then
ident_start = ident_start - 1
@@ -204,6 +235,7 @@ local function extract_arg_names(args_str)
end
end
ident_start = ident_start + 1
--- @type string
local name = trimmed:sub(ident_start, ident_end)
if name ~= "" then names[#names + 1] = name end
::continue::
@@ -213,7 +245,10 @@ local function extract_arg_names(args_str)
return names
end
--- @param args_str string|nil
--- @return string[]|nil
local function formal_arg_names(args_str)
--- @type string[]|nil
local names = extract_arg_names(args_str)
if not names then return nil end
if names[1] == "ab" then table.remove(names, 1) end
@@ -233,10 +268,12 @@ end
--- Carries the scanner-owned `debug_skip` flag forward so the generated projection can emit `/* atom_dbg_skip */`
--- before the authored comment and so `update_canonical_components` can mirror the same field onto `corpus.components[name]`.
--- @param source string -- the full source text (needed for backward lookups)
--- @param scan table -- SourceScan from duffle.scan_source
--- @param scan SourceScan
--- @return Component[]
local function project_components(source, scan)
--- @type Component[]
local out = {}
--- @type integer, AtomEntry
for _, a in ipairs(scan.atoms) do
-- Only `MipsAtomComp_(ac_X)` (kind="comp_bare") and `MipsAtomComp_Proc_(ac_X, ...)` (kind="comp_proc")
-- are COMPONENTS — they get inlined via `mac_<name>` aliases inside atom bodies.
@@ -248,9 +285,11 @@ local function project_components(source, scan)
-- Function-args lookup is meaningful for `MipsAtomComp_Proc_` components
-- (the macro sits inside `FI_ Slice_MipsCode ac_X(...)`); the alias expansion
-- discards the `ab` (atom-builder) arg the same way both forms do.
--- @type string|nil
local args = find_function_args_for(source, a.raw_name, a.ident_pos)
-- Comment ownership: scan_source.lua stamps `declaration_comment` on the record by walking backward past any associated bare marker.
-- The pass reads `declaration_comment` directly.
--- @type string
local comment = a.declaration_comment or ""
out[#out + 1] = {
line = a.line,
@@ -282,22 +321,30 @@ end
--- @param s string
--- @return string
local function convert_line_comments_to_block(s)
--- @type string
local result = s
--- @type integer
local pos = 1
--- @type integer
local len = #result
while pos <= len do
--- @type boolean
local is_double_slash = result:byte(pos) == BYTE_SLASH
and pos + 1 <= len and result:byte(pos + 1) == BYTE_SLASH
if not is_double_slash then
pos = pos + 1
else
-- Find end of line.
--- @type integer
local eol = pos
while eol <= len and result:byte(eol) ~= BYTE_NEWLINE do
eol = eol + 1
end
--- @type string
local before = result:sub(1, pos - 1)
--- @type string
local comment = result:sub(pos + 2, eol - 1) -- skip the `//`
--- @type string
local after
if eol <= len and result:byte(eol) == BYTE_NEWLINE then
after = " */" .. result:sub(eol) -- keep the newline
@@ -335,10 +382,13 @@ end
--- @param tok string
--- @return string
local function strip_leading_delay_marker(tok)
--- @type string|nil
local ident = duffle.read_ident(tok, 1)
if not ident or not duffle.DELAY_MARKERS[ident] then return tok end
--- @type string
local rest = tok:sub(#ident + 1):match("^%s*(.*)$") or ""
while rest:sub(1, 2) == "/*" do
--- @type integer|nil
local close = rest:find("*/", 3, true)
if not close then return "" end
rest = rest:sub(close + 2):match("^%s*(.*)$") or ""
@@ -350,22 +400,29 @@ end
--- in a single source's `count_all_components` pass; the in-progress -1 sentinel detects cycles (A -> B -> A).
--- @param name string -- the component name (without `mac_`)
--- @param comp_by_name table<string, Component>
--- @param wc table<string, integer>
--- @param cache table<string, integer>
--- @param wc WordCounts
--- @param cache table<string, integer> -- bag: name -> count; -1 in-progress sentinel
--- @return integer
local function word_count_rec(name, comp_by_name, wc, cache)
if cache[name] ~= nil then return cache[name] end
cache[name] = -1 -- mark in-progress (cycle detection)
--- @type Component|nil
local cc = comp_by_name[name]
--- @type integer
local n
if cc then
n = 0
--- @type BodyToken[]
local tokens = cc.body_tokens
--- @type integer, BodyToken
for _, t in ipairs(tokens) do
--- @type string
local trimmed = t.tok
if trimmed ~= "" then
--- @type string
local work = trimmed
while true do
--- @type string|nil
local marker = duffle.read_ident(work, 1)
if marker and duffle.DELAY_MARKERS[marker] then
work = strip_leading_delay_marker(work)
@@ -375,6 +432,7 @@ local function word_count_rec(name, comp_by_name, wc, cache)
end
end
if work ~= "" then
--- @type string|nil
local lookup = strip_mac_prefix(duffle.read_ident(work, 1))
if lookup == "atom_label" or lookup == "atom_offset" then
-- Pure metaprogram anchors; emit zero words.
@@ -405,13 +463,18 @@ end
--- references hit memoized values instead of re-walking the body.
--- Cycle detection (A -> B -> A) is preserved via the in-progress `-1` sentinel in `cache`.
--- @param components Component[]
--- @param wc table<string, integer>
--- @return table<string, integer> -- map of component name (without `mac_`) -> word count
--- @param wc WordCounts
--- @return table<string, integer> -- bag: bare component name -> word count
local function count_all_components(components, wc)
--- @type table<string, Component>
local comp_by_name = {}
--- @type integer, Component
for _, cc in ipairs(components) do comp_by_name[cc.name] = cc end
--- @type table<string, integer> -- bag: memo; -1 in-progress sentinel
local cache = {}
--- @type table<string, integer> -- bag: bare name -> word count
local counts = {}
--- @type integer, Component
for _, c in ipairs(components) do
counts[c.name] = word_count_rec(c.name, comp_by_name, wc, cache)
end
@@ -435,28 +498,39 @@ end
--- Missing component: cycle 1, gp0 0.
--- @param name string -- component bare name (e.g. "yield", "pack_color_word")
--- @param comp_by_name table<string, Component>
--- @param latency table<string, integer>
--- @param cache table<string, {cycle_cost=integer, gp0_contrib=integer}>
--- @return {cycle_cost=integer, gp0_contrib=integer}
--- @param latency table<string, integer> -- bag: ident -> cycle cost
--- @param cache ComponentMetaMap
--- @return ComponentMeta
local function component_meta_rec(name, comp_by_name, latency, cache)
if cache[name] ~= nil then return cache[name] end
cache[name] = { cycle_cost = -1, gp0_contrib = -1 }
--- @type Component|nil
local cc = comp_by_name[name]
--- @type integer
local cycle_cost
--- @type integer
local gp0_contrib
if cc then
--- @type boolean
local skip_cycle = (name == "yield")
--- @type boolean
local skip_gp0 = name:match("^insert_ot_tag") ~= nil
cycle_cost = 0
gp0_contrib = 0
if not skip_cycle or not skip_gp0 then
--- @type BodyToken[]
local tokens = cc.body_tokens
--- @type integer, BodyToken
for _, t in ipairs(tokens) do
--- @type string
local trimmed = t.tok
if trimmed ~= "" then
--- @type string|nil
local ident = duffle.read_ident(trimmed, 1)
if ident and ident:sub(1, MAC_PREFIX_LEN) == MAC_PREFIX then
--- @type string
local nested = ident:sub(MAC_PREFIX_LEN + 1)
--- @type ComponentMeta
local nested_meta = component_meta_rec(nested, comp_by_name, latency, cache)
if not skip_cycle then
cycle_cost = cycle_cost + nested_meta.cycle_cost
@@ -466,7 +540,9 @@ local function component_meta_rec(name, comp_by_name, latency, cache)
end
else
if not skip_cycle then
--- @type InstructionRow|nil
local isa = duffle.instr(ident)
--- @type GteCommandRow|nil
local gte = duffle.gte(ident)
cycle_cost = cycle_cost + ((isa and isa.cycles) or (gte and gte.cycles) or latency[ident] or 1)
end
@@ -499,13 +575,18 @@ end
--- Compute `cycle_cost` + `gp0_contrib` for every component in `components` in a single pass.
--- One memoization cache; a nested `mac_Y` inside a `mac_X` body computes both fields once.
--- @param components Component[]
--- @param latency table<string, integer>
--- @return table<string, {cycle_cost=integer, gp0_contrib=integer}>
--- @param latency table<string, integer> -- bag: ident -> cycle cost
--- @return ComponentMetaMap
local function compute_components_metadata(components, latency)
--- @type table<string, Component>
local comp_by_name = {}
--- @type integer, Component
for _, cc in ipairs(components) do comp_by_name[cc.name] = cc end
--- @type ComponentMetaMap
local cache = {}
--- @type ComponentMetaMap
local out = {}
--- @type integer, Component
for _, c in ipairs(components) do
out[c.name] = component_meta_rec(c.name, comp_by_name, latency, cache)
end
@@ -521,10 +602,14 @@ end
--- @param s string
--- @return string[]
local function split_comment_lines(s)
--- @type string[]
local out = {}
--- @type integer
local pos = 1
--- @type integer
local s_len = #s
while pos <= s_len do
--- @type integer|nil
local nl = s:find("\n", pos, true)
if not nl then
out[#out + 1] = s:sub(pos)
@@ -544,6 +629,7 @@ end
--- @param args_str string|nil
--- @return string
local function signature_from_args(args_str)
--- @type string[]|nil
local names = formal_arg_names(args_str)
if names then
return table.concat(names, ", ")
@@ -553,7 +639,10 @@ end
--- Strip the trailing `" \"` (space + backslash) line continuation from the last body line.
--- The last 2 chars are always that pair.
--- @param lines string[]
--- @return nil
local function strip_trailing_continuation(lines)
--- @type string
local last = lines[#lines]
if last:sub(-2) == " \\" then
lines[#lines] = last:sub(1, -3)
@@ -580,12 +669,15 @@ end
--- @param tok string -- a single token from split_top_level_commas (already trimmed at the start, may contain trailing whitespace + block comment)
--- @return boolean
local function is_pure_delay_marker_token(tok)
--- @type table<string, boolean> -- bag: delay-marker ident -> true
local markers = duffle.DELAY_MARKERS
if type(markers) ~= "table" then return false end
-- Identify a leading delay-marker identifier (e.g. `GteDelay_`).
--- @type integer
local ident_end = 1
while ident_end <= #tok do
--- @type string
local ch = tok:sub(ident_end, ident_end)
if ch:match("[%w_]") then
ident_end = ident_end + 1
@@ -593,16 +685,20 @@ local function is_pure_delay_marker_token(tok)
break
end
end
--- @type string
local ident = tok:sub(1, ident_end - 1)
if not markers[ident] then return false end
-- Walk the remainder: only whitespace and block comments are allowed.
--- @type integer
local scan = ident_end
while scan <= #tok do
--- @type string
local ch = tok:sub(scan, scan)
if ch:match("%s") then
scan = scan + 1
elseif ch == "/" and tok:sub(scan + 1, scan + 1) == "*" then
--- @type integer|nil
local close = tok:find("*/", scan + 2, true)
if not close then return false end
scan = close + 2
@@ -641,14 +737,22 @@ end
--- the annotation IS preserved in the generated header
--- (so the comment + marker remain visible to anyone reading `gen/macs.h`), but the C preprocessor expands the marker to empty, so leaving the `,`
--- separator out is what stops the `,,` syntax error. See `token_skips_leading_comma` for the contract.
--- @param lines string[]
--- @param c Component
--- @param sig string
--- @param tokens string[]
--- @return nil
local function emit_macro_body(lines, c, sig, tokens)
--- @type integer
for tok_idx = 1, #tokens do
tokens[tok_idx] = convert_line_comments_to_block(tokens[tok_idx])
end
if #tokens == 0 then return end
lines[#lines + 1] = "#define mac_" .. c.name .. "(" .. sig .. ") \\"
lines[#lines + 1] = "\t" .. tokens[1] .. " \\"
--- @type integer
for tok_idx = 2, #tokens do
--- @type string
local sep = token_skips_leading_comma(tokens[tok_idx]) and "\t" or ",\t"
lines[#lines + 1] = sep .. tokens[tok_idx] .. " \\"
end
@@ -660,11 +764,11 @@ end
--- For skipped components, a `/* atom_dbg_skip */` marker comment is emitted immediately before the authored comment block.
--- The marker is a single line, the comment comes next, and the `#define` line follows. The `debug_skip` stamp is scanner-owned
--- (`a.debug_skip == true` on the declaration record); the components pass projects it directly.
--- @param c Component
--- @param components Component[]
--- @param wc table<string, integer>
--- @param c Component
--- @param counts table<string, integer> -- bag: bare component name -> word count
--- @return string[] -- list of lines for this component
local function build_component_lines(c, counts)
--- @type string[]
local lines = {}
-- Marker comment: emitted once for every skipped component.
@@ -675,15 +779,20 @@ local function build_component_lines(c, counts)
end
if c.comment and c.comment ~= "" then
--- @type integer, string
for _, line in ipairs(split_comment_lines(c.comment)) do
lines[#lines + 1] = line
end
end
--- @type string[]
local tokens = duffle.split_top_level_commas(c.body)
--- @type integer
for i = 1, #tokens do tokens[i] = duffle.trim(tokens[i]) end
--- @type string
local sig = signature_from_args(c.args)
-- Direct lookup against the per-source precomputed `counts` table (built once by count_all_components).
--- @type integer
local n = counts[c.name]
if n > 0 then
@@ -707,10 +816,13 @@ end
--- @param sources SourceFile[] -- Sources contributing to this directory (for the header comment)
--- @return string[]
local function header_boilerplate(dir, sources)
--- @type string[]
local source_lines = { "// Directory: " .. duffle.to_absolute_path(dir) .. "/" }
--- @type integer, SourceFile
for _, src in ipairs(sources) do
source_lines[#source_lines + 1] = "// source: " .. duffle.to_absolute_path(src.path)
end
--- @type string
local source_blob = table.concat(source_lines, "\n")
return {
-- #pragma once wrapped in #ifdef INTELLISENSE_DIRECTIVES, matching the convention in lottes_tape.h.
@@ -739,7 +851,9 @@ end
--- @return string -- Output directory
--- @return string -- Full output path
local function compute_macs_h_path(dir)
--- @type string
local out_dir = dir .. "/" .. GEN_SUBDIR
--- @type string
local out_path = out_dir .. "/" .. MACS_FILENAME
return out_dir, out_path
end
@@ -750,19 +864,24 @@ end
--- @param dir string -- Absolute source directory
--- @param sources SourceFile[] -- Sources contributing to this directory (for the header comment)
--- @param components Component[] -- Aggregated components from all sources in this directory
--- @param counts table<string, integer> -- Precomputed word counts (from count_all_components)
--- @param counts table<string, integer> -- bag: bare component name -> word count
--- @return string|nil -- Path to the written file (nil if no components)
local function emit_component_macros_h(ctx, dir, sources, components, counts)
if #components == 0 then return nil end
--- @type string, string
local out_dir, out_path = compute_macs_h_path(dir)
--- @type string[]
local lines = header_boilerplate(dir, sources)
--- @type integer, Component
for _, c in ipairs(components) do
--- @type integer, string
for _, l in ipairs(build_component_lines(c, counts)) do
lines[#lines + 1] = l
end
end
--- @type string
local content = table.concat(lines, "\n") .. "\n"
duffle.ensure_dir(out_dir)
duffle.write_file_lf(out_path, content)
@@ -776,12 +895,16 @@ end
--- (internal) Extend `corpus.word_counts` with this source's component macros so offsets sees them without re-reading the file.
--- First declaration wins: a later caller's count is dropped (the existing entry from the first source is preserved).
--- @param corpus table -- the corpus
--- @param corpus Corpus
--- @param components Component[]
--- @param counts table<string, integer> -- precomputed word counts (from count_all_components)
--- @param counts table<string, integer> -- bag: bare component name -> word count
--- @return nil
local function update_canonical_word_counts(corpus, components, counts)
--- @type WordCounts
local wc = corpus.word_counts
--- @type integer, Component
for _, c in ipairs(components) do
--- @type string
local key = "mac_" .. c.name
if wc[key] == nil then
wc[key] = counts[c.name]
@@ -790,27 +913,33 @@ local function update_canonical_word_counts(corpus, components, counts)
end
--- @class ComponentDef
--- @field name string -- Bare name (without ac_/mac_ prefix)
--- @field line integer -- Definition source line (line of `MipsAtomComp_(ac_X)` / `MipsAtomComp_Proc_(ac_X, ...)`)
--- @field path string -- Absolute source path of the definition
--- @field kind string -- "comp_bare" | "comp_proc" (atom_proc is NOT a component)
--- @field debug_skip boolean -- Mirror of the scanner-owned `a.debug_skip`; consumers read this directly
--- @field name string -- Bare name (without ac_/mac_ prefix)
--- @field line integer -- Definition source line (line of `MipsAtomComp_(ac_X)` / `MipsAtomComp_Proc_(ac_X, ...)`)
--- @field path string -- Absolute source path of the definition
--- @field kind string -- "comp_bare" | "comp_proc" (atom_proc is NOT a component)
--- @field debug_skip boolean -- Mirror of the scanner-owned `a.debug_skip`; consumers read this directly
--- @field cycle_cost integer|nil -- From metadata[c.name]; nil when the body was not costed
--- @field gp0_contrib integer|nil -- From metadata[c.name]; nil when the body was not costed
--- (internal) Populate `corpus.components` with this source's components-by-name map.
--- First declaration wins; later declarations of the same bare name are dropped and recorded as a collision via `corpus.collisions` (kind = "component").
--- The pass does NOT write to `ctx.shared.components`.
--- No parallel skip map is built here; consumers that need the per-component skip state read `corpus.components[name].debug_skip` directly.
--- The `cycle_cost` + `gp0_contrib` fields are populated from `metadata[c.name]` (computed by `compute_components_metadata` against the original `MipsAtomComp_` body).
--- @param corpus table -- the corpus
--- @param corpus Corpus
--- @param src SourceFile
--- @param components Component[]
--- @param metadata table<string, {cycle_cost=integer, gp0_contrib=integer}>
--- @param metadata ComponentMetaMap
--- @return nil
local function update_canonical_components(corpus, src, components, metadata)
--- @type string
local rel_path = src.path:gsub("\\", "/")
--- @type integer, Component
for _, c in ipairs(components) do
-- Keyed by bare name (e.g. `yield`, `load_tri_indices`).
-- The atoms_source_map pass looks up components by bare name from the corpus;
-- `mac_` prefix lives at the call-site identifier and is stripped before lookup.
--- @type ComponentMeta|nil
local m = metadata and metadata[c.name] or nil
if corpus.components[c.name] == nil then
corpus.components[c.name] = {
@@ -825,9 +954,12 @@ local function update_canonical_components(corpus, src, components, metadata)
else
-- A second declaration of the same bare name: record a typed collision so static-analysis + the report can surface it.
-- Identical-shape declarations (same path + line) reuse the first-wins entry without a collision record.
--- @type ComponentDef
local existing = corpus.components[c.name]
if existing.path ~= rel_path or existing.line ~= c.line then
--- @type string
local kind = c.kind or "comp_bare"
--- @type string
local first_kind = existing.kind or "comp_bare"
corpus.collisions[#corpus.collisions + 1] = {
kind = "component",
@@ -845,12 +977,15 @@ end
--- (internal) Populate `corpus.component_body_index` with this source's body index entries.
--- First declaration wins; later declarations are dropped (no separate collision record: the components collision is already surfaced by `update_canonical_components`).
--- The pass writes to `corpus.component_body_index` only (the corpus owns this projection).
--- @param corpus table -- the corpus
--- @param corpus Corpus
--- @param src SourceFile
--- @param components Component[]
--- @param scan table -- the SourceScan payload (for line_of)
--- @param scan SourceScan
--- @return nil
local function update_canonical_component_body_index(corpus, src, components, scan)
--- @type (fun(pos: integer): integer)|nil
local line_of = scan and scan.line_of
--- @type integer, Component
for _, c in ipairs(components) do
if corpus.component_body_index[c.name] == nil then
corpus.component_body_index[c.name] = {
@@ -869,11 +1004,15 @@ end
--- @param ctx PassCtx
--- @return PassResult
function M.run(ctx)
--- @type MacsOutput[]
local outputs = {}
--- @type PassFinding[]
local errors = {}
--- @type PassFinding[]
local warnings = {}
-- Corpus ownership gate.
--- @type Corpus|nil
local corpus = ctx.shared and ctx.shared.corpus
if type(corpus) ~= "table" then
error("components.run requires ctx.shared.corpus.", 0)
@@ -895,14 +1034,21 @@ function M.run(ctx)
-- Per-directory aggregation: every source in the same directory contributes to one `gen/macs.h`.
-- The directory itself is the namespace. `corpus.sources_by_dir` preserves source-order within each bucket (matches `corpus.source_order`).
--- @type table<string, SourceFile[]>
local sources_by_dir = corpus.sources_by_dir or duffle.group_sources_by_dir(corpus.source_order)
--- @type string, SourceFile[]
for dir, sources in pairs(sources_by_dir) do
-- Aggregate components from every source in this directory.
-- `project_components` returns nil for sources with no `MipsAtomComp_` declarations; we skip those.
--- @type Component[]
local aggregated_components = {}
--- @type table<SourceFile, ComponentMetaMap>
local metadata_per_source = {}
--- @type integer, SourceFile
for _, src in ipairs(sources) do
--- @type Component[]
local per_source = project_components(src.text, src.scan) or {}
--- @type integer, Component
for _, c in ipairs(per_source) do
aggregated_components[#aggregated_components + 1] = c
end
@@ -913,13 +1059,17 @@ function M.run(ctx)
if #aggregated_components > 0 then
-- Compute word counts across the aggregated set. `corpus.word_counts` carries the
-- same-source + prior-directory entries so the recursive lookup sees both.
--- @type table<string, integer> -- bag: bare name -> word count
local counts = count_all_components(aggregated_components, corpus.word_counts)
--- @type string|nil
local macs_path = emit_component_macros_h(ctx, dir, sources, aggregated_components, counts)
if macs_path then
outputs[#outputs + 1] = { macs_h = macs_path }
-- Populate the projections AFTER disk emission (byte-identical `.macs.h` contract).
update_canonical_word_counts(corpus, aggregated_components, counts)
--- @type integer, SourceFile
for _, src in ipairs(sources) do
--- @type Component[]
local per_source = project_components(src.text, src.scan) or {}
if #per_source > 0 then
update_canonical_components(corpus, src, per_source, metadata_per_source[src])
scripts/passes/dwarf_injection.lua
+707 -46
View File
File diff suppressed because it is too large Load Diff
scripts/passes/emission_model.lua
+153
View File
@@ -26,12 +26,106 @@
---
--- `passes.scan_source` strips its private `_code_macros` / `_code_macro_bodies` tables before this pass runs.
--- @class BodyToken
--- @field tok string
--- @field rel integer
--- @class EmissionItem
--- @field kind string
--- @field encoder string|nil
--- @field args string[]|nil
--- @field i integer|nil
--- @field word_count integer|nil
--- @field line integer|nil
--- @field call_text string|nil
--- @field root_call_text string|nil
--- @field invocation_ids integer[]|nil
--- @field outermost_invocation_id integer|nil
--- @field gpr_keys string[]|nil
--- @field ident string|nil
--- @field isa_kind string|nil
--- @field nop_words integer|nil
--- @field is_yield boolean|nil
--- @field is_load boolean|nil
--- @field is_branch boolean|nil
--- @field is_unconditional_jump boolean|nil
--- @field is_terminal_jump boolean|nil
--- @field gp0_shape string|nil
--- @field name string|nil
--- @field target string|nil
--- @field word_index integer|nil
--- @field consuming_encoder string|nil
--- @field consuming_arg_pos integer|nil
--- @field invocation_id integer|nil
--- @class WordEvent
--- @field i integer
--- @field encoder string
--- @field args string[]
--- @field def_path string
--- @field def_line integer
--- @field call_text string|nil
--- @field root_call_text string|nil
--- @field invocation_ids integer[]
--- @field outermost_invocation_id integer
--- @field word_count integer
--- @field gpr_keys string[]|nil
--- @field ident string
--- @field kind string
--- @field nop_words integer
--- @field is_yield boolean
--- @field is_load boolean
--- @field is_branch boolean
--- @field is_unconditional_jump boolean
--- @field is_terminal_jump boolean
--- @field gp0_shape string|nil
--- @field body_line integer|nil
--- @field call_line integer|nil
--- @field call_path string|nil
--- @class EmissionMarker
--- @field kind string
--- @field name string
--- @field line integer
--- @field word_index integer
--- @field target string|nil
--- @field consuming_encoder string|nil
--- @field consuming_arg_pos integer|nil
--- @class EmitError
--- @field kind string
--- @field line integer|nil
--- @field msg string
--- @field source string|nil
--- @field schema_name string|nil
--- @class EmitWarning
--- @field kind string
--- @field line integer|nil
--- @field msg string
--- @class AtomPaths
--- @field tokens BodyToken[]
--- @field line_in_body table<integer, integer> -- bag: body byte offset -> 1-based line
--- @field items EmissionItem[]
--- @field word_events WordEvent[]
--- @field markers EmissionMarker[]
--- @field invocations InvocationRecord[]
--- @field errors EmitError[]
--- @field warnings EmitWarning[]
--- @class EmissionModelPass
--- @field run fun(ctx: PassCtx): PassResult
--- @type EmissionModelPass
local M = {}
-- ─────────────────────────────────────────────────────────────────────────
-- Bootstrap: load `duffle_paths.lua` via debug.getinfo so the module works standalone (run as `luajit passes/emission_model.lua`) and when require'd from the orchestrator.
-- ─────────────────────────────────────────────────────────────────────────
--- @type string
local _bootstrap_dir = debug.getinfo(1, "S").source:match("^@?(.*[/\\])") or "./"
--- @type DuffleExport
local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua")
-- ─────────────────────────────────────────────────────────────────────────
@@ -49,7 +143,13 @@ local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua")
--
-- After this function, every `inv.call_line` is physical. DWARF and provenance output read it directly.
-- The word-event loop forwards the already-physical `outer_inv.call_line` into `we.call_line` for words inside an invocation.
--- @param projection EmissionProjection
--- @param atom_record AtomEntry
--- @param src SourceFile
--- @param corpus Corpus
--- @return nil
local function stamp_root_provenance(projection, atom_record, src, corpus)
--- @type (fun(pos: integer): integer)|nil
local root_line_of = src.scan and src.scan.line_of
assert(type(root_line_of) == "function"
, "emission_model: src.scan.line_of is required (canonical LineIndex closure over the source text) to stamp physical provenance")
@@ -58,10 +158,14 @@ local function stamp_root_provenance(projection, atom_record, src, corpus)
-- `root_body_line` is the physical source line of the ATOM HEADER byte containing the opening `{`; that byte is one byte BEFORE `atom_record.body_off`.
-- The walker assigns line 2 to the body's first content line because line 1 is the trailing `\n` after `{`. Body-text line k therefore maps to `root_body_line + (k - 1)`.
-- `body_off - 1` points at the opening `{`, whose line index identifies the header line. `body_off` points after `{` and would shift every word row forward by one line.
--- @type integer
local root_body_line = root_line_of(atom_record.body_off - 1) or atom_record.line or 0
--- @type table<string, ComponentBodyEntry>
local component_index = corpus.component_body_index or {}
--- @type EmissionItem[]
local word_items = {}
--- @type integer, EmissionItem
for _, item in ipairs(projection.items) do
if item.kind == "word" then word_items[#word_items + 1] = item end
end
@@ -69,14 +173,21 @@ local function stamp_root_provenance(projection, atom_record, src, corpus)
-- Resolve one word's physical body line, where the byte containing that word appears in source.
-- * Component expansions carry `invocation_ids`; the component's full-file `line_of` leaves `item.line` physical.
-- * Raw tokens in the root atom body carry an empty `invocation_ids` list and a body-relative `item.line`; convert them here.
--- @param event WordEvent
--- @param item EmissionItem
--- @return integer
local function body_line_for(event, item)
--- @type integer[]
local ids = event.invocation_ids or {}
-- The innermost open invocation identifies which line index the walker used.
-- A component `line_of` makes `item.line` physical; the atom's `body_text` line index makes it body-relative.
if ids and #ids > 0 then
--- @type integer
local inner_id = ids[#ids]
--- @type InvocationRecord|nil
local inner_inv = inner_id and projection.invocations[inner_id]
if inner_inv then
--- @type ComponentBodyEntry|nil
local component = component_index[inner_inv.component_name]
if component and component.line_of then
-- Walker used `comp.line_of`, which is the source's physical LineIndex. item.line is already physical.
@@ -92,7 +203,9 @@ local function stamp_root_provenance(projection, atom_record, src, corpus)
-- Stamp the root source path onto invocation records whose `call_path` the walker left empty.
-- The walker passes `body_entry.source` to `emit_invoke_begin`; `M.project_emission` creates the root `body_entry` with source `""`, leaving its `call_path` empty.
-- This stamp gives every invocation a physical `call_path` matching `passes/atoms_source_map.lua`'s in-memory provenance projection.
--- @type string
local root_path = src.path or ""
--- @type integer, InvocationRecord
for _, inv in ipairs(projection.invocations) do
if inv.call_path == nil or inv.call_path == "" then
inv.call_path = root_path
@@ -102,6 +215,7 @@ local function stamp_root_provenance(projection, atom_record, src, corpus)
-- Normalize `inv.call_line` to a physical source line.
-- * ROOT invocations (`parent_id == 0`) carry body-relative `call_line` values from `M.LineIndex(body_text)`; convert them once with `root_body_line`.
-- * INNER invocations (`parent_id ~= 0`) carry physical `call_line` values from the component's `line_of`; retain them unchanged.
--- @type integer, InvocationRecord
for _, inv in ipairs(projection.invocations) do
if inv.parent_id == 0 then
inv.call_line = (root_body_line or 0) + (inv.call_line or 1) - 1
@@ -111,13 +225,20 @@ local function stamp_root_provenance(projection, atom_record, src, corpus)
-- Build `body_lines` for each invocation.
-- `atoms_source_map` and `dwarf_injection` read `inv.body_lines[k]` directly from the invocation record created here.
-- Component words already carry physical `item.line` values from the walker's COMPONENT line index, so `body_line_for` returns them unchanged.
--- @type integer, InvocationRecord
for _, inv in ipairs(projection.invocations) do
--- @type integer
local sw = inv.start_word
--- @type integer
local ew = inv.end_word
--- @type integer[]
local bls = {}
--- @type integer
for i = sw, ew do
--- @type EmissionItem|nil
local it = projection.items and projection.items[i]
if it and it.kind == "word" then
--- @type WordEvent
local fake_event = { invocation_ids = { inv.id } }
bls[#bls + 1] = body_line_for(fake_event, it) or 0
end
@@ -128,14 +249,20 @@ local function stamp_root_provenance(projection, atom_record, src, corpus)
-- Resolve each `word_event`'s physical `body_line` and `call_line`.
-- For words inside an invocation, `we.call_line` identifies the OUTER atom source line containing the `mac_X(...)` token that triggered expansion.
-- The root-invocation conversion above makes every `inv.call_line` physical; forward it directly and use each raw word's `body_line` as the fallback.
--- @type integer, WordEvent
for index, we in ipairs(projection.word_events) do
--- @type EmissionItem
local item = word_items[index] or {}
--- @type integer
local body_line = body_line_for(we, item)
item.line = body_line
we.body_line = body_line
--- @type integer
local call_line = body_line
--- @type integer
local outer_id = we.outermost_invocation_id or 0
--- @type InvocationRecord|nil
local outer_inv = projection.invocations[outer_id]
if outer_inv then
-- `outer_inv.call_line` is physical after the conversion loop above, so use it directly.
@@ -151,15 +278,24 @@ end
-- Project one atom record into `atom.paths`.
-- Mutates the atom record in-place and returns the projection (for pass-level error/warning accumulation).
--- @param atom_record AtomEntry
--- @param src SourceFile
--- @param corpus Corpus
--- @return EmissionProjection
local function project_atom(atom_record, src, corpus)
--- @type string
local body = atom_record.body or ""
--- @type WordCounts
local wc = corpus.word_counts or {}
--- @type table<string, ComponentBodyEntry>
local cbi = corpus.component_body_index or {}
--- @type RegUseSchema|nil
local schema = nil
if atom_record.reg_use_schema_name then
schema = corpus.reg_use_schemas and corpus.reg_use_schemas[atom_record.reg_use_schema_name]
end
-- That construction site stamps `invocation.debug_skip` while appending each record to `proj.invocations`.
--- @type EmissionProjection
local proj = duffle.project_emission(body, cbi, wc, corpus.components, {
reg_use_schema = schema,
reg_use_param = atom_record.reg_use_param_name,
@@ -172,11 +308,13 @@ local function project_atom(atom_record, src, corpus)
msg = string.format("RegUse schema %q is missing", atom_record.reg_use_schema_name),
}
end
--- @type integer, EmitError
for _, err in ipairs(corpus.reg_use_errors or {}) do
if err.schema_name == atom_record.reg_use_schema_name then
proj.errors[#proj.errors + 1] = err
end
end
--- @type AtomPaths
local paths = {
tokens = atom_record.body_tokens or {},
line_in_body = duffle.build_body_line_index(body),
@@ -199,23 +337,33 @@ end
--- @param ctx PassCtx -- { shared = { corpus = ... }, out_root, ... }
--- @return PassResult
function M.run(ctx)
--- @type PassOutputEntry[]
local outputs = {}
--- @type EmitError[]
local errors = {}
--- @type EmitWarning[]
local warnings = {}
--- @type Corpus|nil
local corpus = ctx and ctx.shared and ctx.shared.corpus
if type(corpus) ~= "table" then error("emission_model: ctx.shared.corpus is required (canonical projection)", 0) end
if type(corpus.source_order) ~= "table" then error("emission_model: ctx.shared.corpus.source_order is required", 0) end
-- Project once, collect errors + warnings for one atom.
-- Kind must be one of: atom | atom_proc | raw_atom | comp_bare | comp_proc.
--- @param atom AtomEntry
--- @param src SourceFile
--- @return nil
local function process_atom(atom, src)
if not (atom and atom.body) then return end
--- @type string
local kind = atom.kind
if kind ~= "atom" and kind ~= "atom_proc" and kind ~= "raw_atom" and kind ~= "comp_bare" and kind ~= "comp_proc" then
return
end
--- @type EmissionProjection
local proj = project_atom(atom, src, corpus)
--- @type integer, EmitError
for _, e in ipairs(proj.errors) do
-- Preserve `kind` (cycle / count_mismatch / unbalanced) so readers dispatch on the diagnostic class and leave the message string as display text.
errors[#errors + 1] = {
@@ -225,6 +373,7 @@ function M.run(ctx)
source = e.source or src.path,
}
end
--- @type integer, EmitWarning
for _, w in ipairs(proj.warnings) do
warnings[#warnings + 1] = {
kind = w.kind,
@@ -237,11 +386,15 @@ function M.run(ctx)
-- Walk `corpus.source_order`; within each source, visit atoms followed by raw_atoms.
-- Recognized kinds (atom | atom_proc | raw_atom | comp_bare | comp_proc) each receive the atom.paths projection via duffle.project_emission.
-- Components are macros inlined into atom bodies; focused tests and isolated component analyses consume atom.paths directly.
--- @type integer, SourceFile
for _, src in ipairs(corpus.source_order) do
--- @type SourceScan
local scan = src.scan or {}
--- @type integer, AtomEntry
for _, atom in ipairs(scan.atoms or {}) do
process_atom(atom, src)
end
--- @type integer, AtomEntry
for _, atom in ipairs(scan.raw_atoms or {}) do
process_atom(atom, src)
end
scripts/passes/offsets.lua
+88 -29
View File
@@ -20,7 +20,9 @@
-- Uses `debug.getinfo` to find this file's own directory, so it works both standalone and when require'd from the orchestrator.
-- Bootstrap: load `duffle_paths.lua` via `debug.getinfo(1, "S").source` (works both standalone + when require'd).
-- duffle_paths.lua sets package.path then returns `require("duffle")` at the bottom, so the dofile value IS the duffle module.
--- @type string
local _bootstrap_dir = debug.getinfo(1, "S").source:match("^@?(.*[/\\])") or "./"
--- @type DuffleExport
local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua")
-- ════════════════════════════════════════════════════════════════════════════
@@ -28,33 +30,20 @@ local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua")
-- ════════════════════════════════════════════════════════════════════════════
-- Offset macro/enum naming prefixes (the emitted header uses these).
--- @type string
local OFFSET_MACRO_PREFIX = "_atom_offset_"
--- @type string
local OFFSET_ENUM_PREFIX = "atom_offset_"
-- Column width for the `#define _atom_offset_F_T = N` alignment.
--- @type integer
local OFFSET_MACRO_COL = 44
-- ════════════════════════════════════════════════════════════════════════════
-- Type declarations
-- ════════════════════════════════════════════════════════════════════════════
--- @class SourceFile
--- @field path string -- Absolute path to the source file
--- @field text string -- Full source text
--- @field dir string -- Directory containing the source
--- @field basename string -- Filename without extension
--- @field scan table -- Pre-scanned SourceScan payload (from duffle.scan_source)
--- @class PassCtx
--- @field shared table -- Cross-pass shared state
--- @field shared.corpus table -- Corpus projection
--- @field shared.word_counts table
--- @field out_root string -- Output root (e.g. "build/gen")
--- @class PassResult
--- @field outputs table[] -- {kind=, path=} entries describing emit files
--- @field errors table[] -- {line=, msg=} entries; build-stops
--- @field warnings table[] -- {line=, msg=} entries; build-succeeds
-- SourceFile, PassCtx, PassResult: see ps1_meta.lua
--- @class BranchOffset
--- @field tag string -- Marker tag (e.g. "F" in `atom_offset(F, T)`)
@@ -69,6 +58,32 @@ local OFFSET_MACRO_COL = 44
--- @field total_words integer -- Total word count of the atom body
--- @field offsets BranchOffset[] -- Per-branch offset list
--- @class OffsetBranch
--- @field tag string
--- @field target string
--- @field branch_word integer
--- @field consuming_encoder string|nil
--- @field consuming_arg_pos integer|nil
--- @field line integer|nil
--- @class MarkerProjectState
--- @field labels table<string, integer> -- bag: label name -> word index
--- @field branches OffsetBranch[]
--- @class OffsetConst
--- @field macro_name string
--- @field enum_name string
--- @field value integer
--- @class OffsetOutput
--- @field offsets_h string
--- @class OffsetsPass
--- @field run fun(ctx: PassCtx): PassResult
--- @class AtomEntry
--- @field paths AtomPaths|nil
-- ════════════════════════════════════════════════════════════════════════════
-- Canonical marker projection
-- ════════════════════════════════════════════════════════════════════════════
@@ -76,10 +91,17 @@ local OFFSET_MACRO_COL = 44
-- MARKER_PROJECTORS is the marker-kind data table.
-- The emission-model pass already records marker word positions + consuming-instruction context;
-- this pass only projects those records into the label/branch lookup shape needed by offset computation.
--- @type table<string, fun(state: MarkerProjectState, marker: EmissionMarker): nil>
local MARKER_PROJECTORS = {
--- @param state MarkerProjectState
--- @param marker EmissionMarker
--- @return nil
label = function(state, marker)
state.labels[marker.name] = marker.word_index
end,
--- @param state MarkerProjectState
--- @param marker EmissionMarker
--- @return nil
offset = function(state, marker)
state.branches[#state.branches + 1] = {
tag = marker.name,
@@ -93,11 +115,15 @@ local MARKER_PROJECTORS = {
--- Project canonical marker records into the two lookup tables used by the offset renderer.
--- No source text, body text, or body token is inspected.
--- @param markers table[] -- atom.paths.markers
--- @return table<string, integer>, table[]
--- @param markers EmissionMarker[]
--- @return table<string, integer>
--- @return OffsetBranch[]
local function project_markers(markers)
--- @type MarkerProjectState
local state = { labels = {}, branches = {} }
--- @type integer, EmissionMarker
for _, marker in ipairs(markers or {}) do
--- @type (fun(state: MarkerProjectState, marker: EmissionMarker): nil)|nil
local project = MARKER_PROJECTORS[marker.kind]
if project then project(state, marker) end
end
@@ -119,12 +145,15 @@ end
--- `jump_reg` / `call_reg` / `jump_link` -> ERROR. Register-form jumps have no offset field; `atom_offset` is invalid.
--- missing `consuming_encoder` -> ERROR. A lone top-level `atom_offset` is not a branch.
--- @param labels table<string, integer>
--- @param branches table[]
--- @param errors table[]
--- @param branches OffsetBranch[]
--- @param errors PassFinding[]
--- @return BranchOffset[]
local function compute_offsets(labels, branches, errors)
--- @type BranchOffset[]
local results = {}
--- @type integer, OffsetBranch
for _, br in ipairs(branches) do
--- @type integer|nil
local target = labels[br.target]
if not target then
errors[#errors + 1] = {
@@ -132,6 +161,7 @@ local function compute_offsets(labels, branches, errors)
msg = "Branch target '" .. br.target .. "' has no atom_label (at word " .. br.branch_word .. ")",
}
else
--- @type string|nil
local consuming = br.consuming_encoder
if consuming == nil or consuming == "" then
errors[#errors + 1] = {
@@ -171,7 +201,7 @@ end
--- (internal) Build a constant-table entry `{macro_name, enum_name, value}` from a BranchOffset.
--- @param bo BranchOffset
--- @return table
--- @return OffsetConst
local function make_offset_const(bo)
return {
macro_name = OFFSET_MACRO_PREFIX .. bo.tag .. "_" .. bo.target,
@@ -183,19 +213,24 @@ end
--- (internal) Emit one atom's offset constants + enum into the lines buffer.
--- @param add fun(s: string)
--- @param atom AtomData
--- @return nil
local function emit_atom_offsets(add, atom)
if #atom.offsets == 0 then return end
add("// --- atom: " .. atom.name .. " (" .. atom.total_words .. " words) ---")
add("")
--- @type OffsetConst[]
local consts = {}
--- @type integer, BranchOffset
for _, r in ipairs(atom.offsets) do
consts[#consts + 1] = make_offset_const(r)
end
--- @type integer, OffsetConst
for _, c in ipairs(consts) do
add("#define " .. pad_right(c.macro_name, OFFSET_MACRO_COL) .. " " .. c.value)
end
add("")
add("enum {")
--- @type integer, OffsetConst
for _, c in ipairs(consts) do
add(" " .. c.enum_name .. " = " .. c.macro_name .. ",")
end
@@ -204,18 +239,23 @@ local function emit_atom_offsets(add, atom)
end
--- Generate the per-directory .offsets.h header.
--- @param dir string -- the absolute source directory
--- @param sources table[] -- sources contributing to this directory (for the header comment)
--- @param atoms_data AtomData[]
--- @param dir string
--- @param sources SourceFile[]
--- @param atoms_data AtomData[]
--- @return string
local function generate_header(dir, sources, atoms_data)
--- @type string
local dir_basename = duffle.basename_no_ext(dir)
--- @type string[]
local lines = {}
--- @param s string
--- @return nil
local function add(s) lines[#lines + 1] = s end
add("// Auto-generated by ps1_meta.lua (passes/offsets.lua) — DO NOT EDIT")
add("// Directory: " .. dir:gsub("/", "\\") .. "\\")
--- @type integer, SourceFile
for _, src in ipairs(sources) do
add("// source: " .. src.path:gsub("/", "\\"))
end
@@ -224,6 +264,7 @@ local function generate_header(dir, sources, atoms_data)
add("#pragma region " .. dir_basename)
add("")
add("")
--- @type integer, AtomData
for _, atom in ipairs(atoms_data) do
emit_atom_offsets(add, atom)
end
@@ -232,21 +273,27 @@ local function generate_header(dir, sources, atoms_data)
return table.concat(lines, "\n") .. "\n"
end
--- @type OffsetsPass
local M = {}
--- (internal) Aggregate atoms from every source in one directory, render the per-directory `offsets.h`.
--- Returns the offsets_h path if a header was written, or nil.
--- @param ctx PassCtx
--- @param dir string -- the absolute source directory
--- @param sources SourceFile[] -- sources in this directory
--- @param errors table[]
--- @return string|nil -- the offsets_h path
--- @param dir string
--- @param sources SourceFile[]
--- @param errors PassFinding[]
--- @return string|nil
local function process_directory(ctx, dir, sources, errors)
--- @type AtomData[]
local atoms_data = {}
--- @param atom AtomEntry
--- @return nil
local function append_atom(atom)
--- @type AtomPaths|nil
local paths = atom and atom.paths
if not paths then return end
--- @type table<string, integer>, OffsetBranch[]
local labels, branches = project_markers(paths.markers)
atoms_data[#atoms_data + 1] = {
name = atom.raw_name or atom.name,
@@ -255,13 +302,18 @@ local function process_directory(ctx, dir, sources, errors)
}
end
--- @type integer, SourceFile
for _, src in ipairs(sources) do
--- @type SourceScan
local scan = src.scan or {}
--- @type integer, AtomEntry
for _, atom in ipairs(scan.atoms or {}) do append_atom(atom) end
--- @type integer, AtomEntry
for _, atom in ipairs(scan.raw_atoms or {}) do append_atom(atom) end
end
if #atoms_data == 0 then return nil end
--- @type string
local out_path = dir .. "/gen/offsets.h"
duffle.ensure_dir(duffle.dirname(out_path))
duffle.write_file(out_path, generate_header(dir, sources, atoms_data))
@@ -274,10 +326,14 @@ end
--- @param ctx PassCtx
--- @return PassResult
function M.run(ctx)
--- @type OffsetOutput[]
local outputs = {}
--- @type PassFinding[]
local errors = {}
--- @type PassFinding[]
local warnings = {}
--- @type Corpus|nil
local corpus = ctx.shared and ctx.shared.corpus
if type(corpus) ~= "table" then
error("offsets.run requires ctx.shared.corpus", 0)
@@ -287,8 +343,11 @@ function M.run(ctx)
end
-- Per-directory aggregation: every source in the same directory contributes to one `gen/offsets.h`.
--- @type table<string, SourceFile[]>
local sources_by_dir = corpus.sources_by_dir or duffle.group_sources_by_dir(corpus.source_order)
--- @type string, SourceFile[]
for dir, sources in pairs(sources_by_dir) do
--- @type string|nil
local out_path = process_directory(ctx, dir, sources, errors)
if out_path then
outputs[#outputs + 1] = { offsets_h = out_path }
scripts/passes/report.lua
+401 -43
View File
File diff suppressed because it is too large Load Diff
scripts/passes/scan_source.lua
+922 -32
View File
File diff suppressed because it is too large Load Diff
scripts/passes/static_analysis.lua
+1195 -28
View File
File diff suppressed because it is too large Load Diff
scripts/passes/word_count_eval.lua
+14 -22
View File
@@ -23,7 +23,9 @@
-- Bootstrap: load `scripts/duffle_paths.lua` (sets package.path + package.cpath).
-- Uses `debug.getinfo` to find this file's own directory, so it works both standalone and when require'd from the orchestrator.
-- duffle_paths.lua sets package.path then returns `require("duffle")` at the bottom, so the dofile value IS the duffle module.
--- @type string
local _bootstrap_dir = debug.getinfo(1, "S").source:match("^@?(.*[/\\])") or "./"
--- @type DuffleExport
local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua")
-- ════════════════════════════════════════════════════════════════════════════
@@ -31,35 +33,20 @@ local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua")
-- ════════════════════════════════════════════════════════════════════════════
--- @class WordCounts
--- @field [string] integer -- macro name -> word count
--- @field [string] integer -- bag: macro name -> word count
--- @class SourceFile
--- @field path string -- absolute path to the source file
--- @field text string -- the full source text
--- @field dir string -- the directory containing the source
--- @field basename string -- filename without extension
--- @class WordCountEval
--- @field count_token_words fun(token: string, wc: WordCounts): integer
--- @field run fun(ctx: PassCtx): PassResult
--- @class PassCtx
--- @field sources SourceFile[] -- all source files in the build
--- @field metadata_path string -- path to word_count.metadata.h
--- @field shared table -- cross-pass shared state
--- @field shared.corpus table -- canonical corpus (required)
--- @field shared.corpus.word_counts WordCounts -- canonical count table (populated by this pass)
--- @field out_root string -- output root (e.g. "build/gen")
--- @field project_root string -- project root (e.g. "code/")
--- @field upstream table<string, table> -- per-pass upstream outputs
--- @field flags table -- CLI flags
--- @field verbose boolean -- if true, log diagnostic info
--- @class PassResult
--- @field outputs table[] -- {kind=, path=} entries describing emit files
--- @field errors table[] -- {line=, msg=} entries; build-stops
--- @field warnings table[] -- {line=, msg=} entries; build-succeeds
-- SourceFile, PassCtx, PassResult: see ps1_meta.lua
-- DuffleExport: see duffle.lua (facade returned by duffle_paths.lua)
-- ════════════════════════════════════════════════════════════════════════════
-- Module exports
-- ════════════════════════════════════════════════════════════════════════════
--- @type WordCountEval
local M = {}
-- ┌────────────────────────────────────────────────────────────────────┐
@@ -74,11 +61,14 @@ local M = {}
--- @param wc WordCounts -- the shared word-count table
--- @return integer
function M.count_token_words(token, wc)
--- @type string
local s = duffle.trim(token)
if s == "" then return 0 end
--- @type string|nil, integer
local name, after = duffle.read_ident(s, 1)
if not name then return 1 end
if wc[name] then return wc[name] end
--- @type integer
local paren_pos = duffle.skip_ws_and_cmt(s, after)
if s:sub(paren_pos, paren_pos) == "(" then
io.stderr:write(" warning: unknown macro '" .. name .. "', assuming 1 word\n")
@@ -105,6 +95,7 @@ end
--- @return PassResult
function M.run(ctx)
-- 1. Canonical-corpus ownership gate.
--- @type Corpus|nil
local corpus = ctx.shared and ctx.shared.corpus
if type(corpus) ~= "table" then
error("word_count_eval.run requires ctx.shared.corpus (canonical corpus). The fixture must install the corpus before running this pass.", 0)
@@ -117,6 +108,7 @@ function M.run(ctx)
-- 3. Load authored metadata. Generated .macs.h files are NOT scanned
-- (the pass computes their counts from the just-built bodies after disk emission; see passes/components.lua).
--- @type WordCounts
local wc = duffle.load_word_counts(ctx.metadata_path)
-- 4. Assign the count table. ONE assignment, no copy. The assignment creates no secondary alias.