diff --git a/scripts/duffle.lua b/scripts/duffle.lua index 364141f..b05e391 100644 --- a/scripts/duffle.lua +++ b/scripts/duffle.lua @@ -1,10 +1,52 @@ --- duffle.lua — facade over duffle_scan / duffle_isa / duffle_emit. + +--- @class DuffleExport +--- bag: open module-export keys from duffle_scan / duffle_isa / duffle_emit + +--- @type DuffleExport local scan = require("duffle_scan") +--- @type DuffleExport local isa = require("duffle_isa") +--- @type DuffleExport local emit = require("duffle_emit") +--- @type DuffleExport local M = {} +--- @alias Path string +--- @alias LineNum integer +--- @alias ByteOff integer +--- @alias MacroName string +--- @alias AtomName string +--- @alias Severity string + +--- @class SourceFile +--- @field path Path +--- @field text string +--- @field dir string +--- @field basename string +--- @field scan SourceScan|nil + +--- @class CorpusView +--- @field register_alias_registry table +--- @field type_name_registry table +--- @field atom_views table +--- @field atom_ctxs table +--- @field atom_phases table +--- @field binds_by_name table +--- @field atoms_by_name table +--- @field atom_infos AtomInfoEntry[] +--- @field components table +--- @field component_atom_infos AtomInfoEntry[]|nil +--- @field component_body_index table +--- @field tape_chains table|nil +--- @field source_order SourceFile[] +--- @field collisions CorpusCollision[] + +--- @param src DuffleExport +--- @param label string +--- @return nil local function merge(src, label) + --- @type string, any for k, v in pairs(src) do if M[k] ~= nil and M[k] ~= v then error("duffle facade name collision on " .. tostring(k) .. " from " .. label, 0) @@ -17,7 +59,10 @@ merge(scan, "duffle_scan") merge(isa, "duffle_isa") merge(emit, "duffle_emit") +--- @param ctx PassCtx +--- @return CorpusView function M.corpus_view(ctx) + --- @type Corpus local corpus = ctx and ctx.shared and ctx.shared.corpus if not corpus then error("requires ctx.shared.corpus", 0) end return { @@ -38,8 +83,16 @@ function M.corpus_view(ctx) } end +--- @param rules CheckRule[] +--- @param phase string +--- @param item AtomEntry|SourceFile +--- @param pipe_ctx PipeCtx +--- @param findings CheckFinding[] +--- @return nil function M.run_check_rules(rules, phase, item, pipe_ctx, findings) + --- @type integer, CheckRule for _, rule in ipairs(rules) do + --- @type (fun(item: AtomEntry|SourceFile, pipe_ctx: PipeCtx, findings: CheckFinding[]): nil)|nil local fn = rule[phase] if fn then fn(item, pipe_ctx, findings) end end diff --git a/scripts/duffle_emit.lua b/scripts/duffle_emit.lua index 5d558b4..b7c137d 100644 --- a/scripts/duffle_emit.lua +++ b/scripts/duffle_emit.lua @@ -1,8 +1,13 @@ --- duffle_emit.lua — project_emission + decl finders. +--- @type DuffleScan local scan = require("duffle_scan") +--- @type DuffleIsa local isa = require("duffle_isa") +--- @type DuffleEmit local M = {} +--- @type string, any for k, v in pairs(scan) do M[k] = v end +--- @type string, any for k, v in pairs(isa) do M[k] = v end -- Section 8: Cross-source component-body index + word-event expansion @@ -12,25 +17,55 @@ for k, v in pairs(isa) do M[k] = v end -- built once from the pre-tokenized bodies. --- @class ComponentBodyEntry ---- @field body_tokens table -- pre-tokenized {{tok=string, rel=integer}, ...} +--- @field body_tokens BodyToken[] --- @field body_off integer -- byte offset of body[1] in `source` --- @field line_of fun(pos:integer):integer -- byte-offset → 1-based line number in `source` --- @field source string -- absolute path of the source containing the declaration --- @field declaration integer -- 1-based line number of the MipsAtomComp_(ac_X) declaration --- @field kind string -- "comp_bare" | "comp_proc" +--- @field arg_names string[]|nil +--- @field sub_map table|nil -- bag: formal -> substituted operand + +--- @class EmissionWalkCtx +--- @field component_index table +--- @field word_counts WordCounts +--- @field components table +--- @field reg_use_schema RegUseSchema|nil +--- @field reg_use_param string|nil +--- @field atom_name AtomName|nil +--- @field schema_name string|nil +--- @field visiting table|nil -- bag: component name on DFS stack +--- @field root_call_path string|nil +--- @field root_call_line integer|nil + +--- @class RegUseCtx +--- @field reg_use_schema RegUseSchema|nil +--- @field reg_use_param string|nil +--- @field atom_name AtomName|nil +--- @field schema_name string|nil + +--- @class DuffleEmit + -- The cross-source component-body index is owned by the corpus (`corpus.component_body_index`, populated by `passes/components.lua`). -- Consumers (`passes/static_analysis.lua`, `passes/emission_model.lua`) read it directly; per-pass memoization helpers stay out of scope. -- ASCII byte constants used by split_call_args (kept local to keep Section 8 self-contained). +--- @type integer local E_BYTE_OPEN_PAREN = 0x28 +--- @type integer local E_BYTE_OPEN_BRACE = 0x7B +--- @type integer local E_BYTE_OPEN_BRACK = 0x5B +--- @type integer local E_BYTE_DQUOTE = 0x22 +--- @type integer local E_BYTE_SQUOTE = 0x27 +--- @type integer local E_BYTE_COMMA = 0x2C -- Map an open-delimiter byte to its matching close string for read_balanced. +--- @type table -- bag: open-delimiter byte -> close string local E_OPEN_CLOSE = { [E_BYTE_OPEN_PAREN] = ")", [E_BYTE_OPEN_BRACE] = "}", @@ -44,15 +79,22 @@ local E_OPEN_CLOSE = { --- @param inner string --- @return string[] local function split_call_args(inner) +--- @type string[] local args = {} if not inner or inner == "" then return args end +--- @type integer local pos = 1 +--- @type integer local len = #inner +--- @type integer local start = 1 while pos <= len do +--- @type integer local c = inner:byte(pos) +--- @type string|nil local close = E_OPEN_CLOSE[c] if close then +--- @type integer, integer local _, after = M.read_balanced(inner, string.char(c), close, pos) pos = after elseif c == E_BYTE_DQUOTE or c == E_BYTE_SQUOTE then @@ -74,17 +116,22 @@ end --- @param tok string --- @return string, string[] local function token_ident_and_args(tok) +--- @type string|nil, integer local ident, after = M.read_ident(tok, 1) if not ident then return "?", {} end +--- @type integer local paren_pos = M.skip_ws_and_cmt(tok, after) if tok:sub(paren_pos, paren_pos) ~= "(" then return ident, {} end +--- @type string|nil local inner = M.read_parens(tok, paren_pos) if not inner then return ident, {} end return ident, split_call_args(inner) end -- The macro-name prefix that marks a `mac_X(...)` component invocation. +--- @type string local E_MAC_PREFIX = "mac_" +--- @type integer local E_MAC_PREFIX_LEN = 4 --- Expand a body entry into the flat sequence of emitted machine-word events. @@ -104,10 +151,10 @@ local E_MAC_PREFIX_LEN = 4 --- --- Pure: reads `body_entry` / `component_index` / `word_counts`. Memoization is the caller's responsibility. --- Callers wanting `word_events` / `word_event_errors` precomputed for many atoms should memoize them per atom. ---- @param body_entry table -- `{body_tokens, body_off, line_of, source, declaration}` (declaration = root atom's atom.line) ---- @param component_index table -- the bare-name → ComponentBodyEntry map from M.get_component_body_index ---- @param word_counts table -- macro name → emitted-word count (from `ctx.shared.word_counts`) ---- @return WordEvent[], WordEventError[] +--- @param body_entry ComponentBodyEntry +--- @param component_index table +--- @param word_counts WordCounts +--- @return WordEvent[], EmitError[] -- ════════════════════════════════════════════════════════════════════════════ -- Section 11: project_emission (per-atom emission projection) @@ -122,12 +169,12 @@ local E_MAC_PREFIX_LEN = 4 -- word_counts table is authored-metadata + current-component count table. --- @class EmissionProjection ---- @field items table[] -- Ordered stream of word|label|offset|invoke_begin|invoke_end ---- @field word_events table[] -- Dense view of items where kind == "word" ---- @field markers table[] -- Dense view of items where kind == "label"|"offset" +--- @field items EmissionItem[] +--- @field word_events WordEvent[] +--- @field markers EmissionMarker[] --- @field invocations InvocationRecord[] -- dense view of items where kind == "invoke_begin"|"invoke_end" ---- @field errors table[] -- Token-resolution failures surfaced without fail-loud ---- @field warnings table[] -- Opaque warnings (e.g. unknown uncounted macro) +--- @field errors EmitError[] +--- @field warnings EmitWarning[] --- @class InvocationRecord --- Lives at `atom.paths.invocations[*]`. Constructed once at the single invocation-construction site @@ -148,7 +195,7 @@ local E_MAC_PREFIX_LEN = 4 --- @field end_word integer -- 1-based items index of the `invoke_end` item (set by `emit_invoke_end`) --- @field word_count integer -- Number of `word` items emitted between `start_word` and `end_word` (inclusive) --- @field debug_skip boolean -- `debug_skip` stamp; true iff `corpus.components[name].debug_skip` is true at construction. Always boolean (never `nil`). ---- @field errors table[] -- Per-invocation construction errors (cycle / count_mismatch); does not include pass-level errors +--- @field errors EmitError[] -- Internal recursive walker. The items stream holds every emitted event in order; `word_events`, `markers`, -- `invocations`, `errors`, `warnings` are dense views / side outputs appended alongside. @@ -166,35 +213,58 @@ local E_MAC_PREFIX_LEN = 4 -- then breaks out without recursing (the cycle entry still receives an invocation ID + paired `invoke_begin` / `invoke_end` items, so the boundary invariant is preserved). -- * Component declared-count mismatch (declared vs. measured) is a construction error (kind = "count_mismatch"); recorded on the invocation record and pass-level errors list. -- * Final boundary check: if any invocation is still open at end of walk, surface a "unbalanced" construction error. +--- @param root_body_entry ComponentBodyEntry +--- @param ctx_table EmissionWalkCtx +--- @return EmissionProjection local function _project_emission_inner(root_body_entry, ctx_table) +--- @type EmissionItem[] local items = {} +--- @type WordEvent[] local word_events = {} +--- @type EmissionMarker[] local markers = {} +--- @type InvocationRecord[] local invocations = {} +--- @type EmitError[] local errors = {} +--- @type EmitWarning[] local warnings = {} +--- @type integer local word_idx = 0 +--- @type InvocationRecord[] local invocation_stack = {} -- stack of currently-open invocation records +--- @type integer local next_inv_id = 0 +--- @type RegUseSchema|nil local reg_use_schema = ctx_table.reg_use_schema +--- @type string|nil local reg_use_param = ctx_table.reg_use_param +--- @type AtomName|nil local atom_name = ctx_table.atom_name +--- @type table -- bag: slot name -> readonly local slot_readonly = {} if reg_use_schema then +--- @type integer, RegUseSlot for _, slot in ipairs(reg_use_schema.slots or {}) do slot_readonly[slot.name] = slot.readonly == true end end +--- @param sub_map table|nil +--- @param operand string|any +--- @return any local function apply_sub(sub_map, operand) if not (sub_map and type(operand) == "string") then return operand end if sub_map[operand] then return sub_map[operand] end +--- @type integer|nil local dot = operand:find(".", 1, true) if dot then +--- @type string local head = operand:sub(1, dot - 1) +--- @type string|nil local mapped = sub_map[head] if type(mapped) == "string" then return mapped .. operand:sub(dot) @@ -203,39 +273,65 @@ local function _project_emission_inner(root_body_entry, ctx_table) return operand end +--- @param operand string|any +--- @return string|nil, string|nil, string|nil local function resolve_gpr_key(operand) if type(operand) ~= "string" then return nil end if operand:sub(1, 2) == "R_" then return operand end if not (reg_use_schema and reg_use_param) then return nil end +--- @type string local prefix = reg_use_param .. "." if operand:sub(1, #prefix) ~= prefix then return nil end +--- @type string local member_path = operand:sub(#prefix + 1) +--- @type string|nil local slot = reg_use_schema.alias_to_slot[member_path] if not slot then return nil, member_path end return "reguse:" .. atom_name .. ":" .. slot, nil, slot end +--- @return integer[] local function open_invocation_ids_snapshot() +--- @type integer[] local ids = {} +--- @type integer, InvocationRecord for _, inv in ipairs(invocation_stack) do ids[#ids + 1] = inv.id end return ids end +--- @param encoder string +--- @param args string[]|nil +--- @param line integer +--- @param word_call_text string|nil +--- @param def_source_now string|nil +--- @param def_line_now integer|nil +--- @param immediate_call_text string|nil +--- @param root_call_text_w string|nil +--- @param sub_map table|nil +--- @return nil local function emit_word(encoder, args, line, word_call_text, def_source_now, def_line_now, immediate_call_text, root_call_text_w, sub_map) +--- @type integer[] local inv_ids = open_invocation_ids_snapshot() +--- @type integer local outermost = inv_ids[1] or 0 -- For words emitted at the root atom body, `immediate_call_text` is nil and the walker's `word_call_text` (the word's own token, e.g. "nop") becomes the effective call_text. -- For words emitted inside a component expansion, `immediate_call_text` is the immediate outer `mac_X(...)` token text; -- The call that triggered the body expansion we're currently walking. +--- @type string|nil local eff_call_text = immediate_call_text or word_call_text +--- @type string|nil local eff_root_call_text = root_call_text_w +--- @type string[]|nil local gpr_keys = nil if reg_use_schema or sub_map then gpr_keys = {} +--- @type integer, string for pos, arg in ipairs(args or {}) do +--- @type any local effective = apply_sub(sub_map, arg) +--- @type string|nil, string|nil, string|nil local key, unresolved, slot = resolve_gpr_key(effective) gpr_keys[pos] = key if unresolved then @@ -247,8 +343,10 @@ local function _project_emission_inner(root_body_entry, ctx_table) } end if key and slot and slot_readonly[slot] then +--- @type InstructionRow|nil local row = M.instr(encoder) if row and row.writes then +--- @type integer, integer for _, wpos in ipairs(row.writes) do if wpos == pos then errors[#errors + 1] = { @@ -266,16 +364,25 @@ local function _project_emission_inner(root_body_entry, ctx_table) if not reg_use_schema then gpr_keys = nil end +--- @type InstructionRow|nil local isa = M.instr(encoder) +--- @type string local isa_kind = isa and isa.kind or "unknown" +--- @type integer local nop_words = (encoder == "nop" and 1) or (encoder == "nop2" and 2) or 0 +--- @type boolean local is_yield = (encoder == "mac_yield" or encoder == "mac_yield_tail") +--- @type string|nil local gp0_shape = type(encoder) == "string" and encoder:match("^mac_format_([%w_]+)_color$") or nil +--- @type boolean local is_load = (isa_kind == "load") +--- @type boolean local is_branch = (isa_kind == "branch") +--- @type boolean local is_unconditional_jump = (encoder == "jump" or encoder == "call_addr") +--- @type boolean local is_terminal_jump = (encoder == "jump_reg" or encoder == "call_reg" or encoder == "jump_link") items[#items + 1] = { kind = "word", @@ -324,10 +431,21 @@ local function _project_emission_inner(root_body_entry, ctx_table) word_idx = word_idx + 1 end +--- @param kind string +--- @param name string +--- @param target string|nil +--- @param line integer +--- @param immediate_call_text string|nil +--- @param root_call_text_w string|nil +--- @param consuming_encoder string|nil +--- @param consuming_arg_pos integer|nil +--- @return nil local function emit_marker(kind, name, target, line, immediate_call_text, root_call_text_w, consuming_encoder, consuming_arg_pos) +--- @type integer[] local inv_ids = open_invocation_ids_snapshot() +--- @type integer local outermost = inv_ids[1] or 0 -- Markers carry the open invocation stack snapshot. `call_text` / `root_call_text` belong to words, not markers — markers are zero-width and skip per-word call-site attribution. -- `consuming_encoder` + `consuming_arg_pos` carry the surrounding control-transfer instruction context @@ -338,6 +456,7 @@ local function _project_emission_inner(root_body_entry, ctx_table) if kind == "offset" and (consuming_encoder == nil or consuming_encoder == "") then return end +--- @type EmissionItem local it = { kind = kind, name = name, @@ -364,21 +483,32 @@ local function _project_emission_inner(root_body_entry, ctx_table) -- Count top-level commas in `tok` between position `from_pos` (inclusive) and `to_pos` (exclusive). -- Tracks paren depth so commas inside nested () don't count. Skips string literals + comments. -- Used by `emit_embedded_markers` to compute `consuming_arg_pos` for each embedded marker. +--- @param tok string +--- @param from_pos integer +--- @param to_pos integer +--- @return integer local function count_top_level_commas(tok, from_pos, to_pos) +--- @type integer local depth = 0 +--- @type integer local count = 0 +--- @type integer local i = from_pos while i < to_pos do +--- @type string local c = tok:sub(i, i) if c == "'" or c == '"' then +--- @type integer local next_pos = M.skip_str_or_cmt(tok, i) i = (next_pos > i) and next_pos or (i + 1) elseif c == "/" and tok:sub(i + 1, i + 1) == "/" then -- line comment: skip to end of line +--- @type integer|nil local nl = tok:find("\n", i, true) i = (nl and nl + 1) or (#tok + 1) elseif c == "/" and tok:sub(i + 1, i + 1) == "*" then -- block comment: skip to matching */ +--- @type integer|nil local close = tok:find("*/", i + 2, true) i = (close and close + 2) or (#tok + 1) elseif c == "(" then @@ -399,9 +529,13 @@ local function _project_emission_inner(root_body_entry, ctx_table) -- Find the position of the consuming instruction's open paren (the `(` that starts the consuming instruction's argument list). -- Returns nil if the token's leading text isn't an ident followed by `(` (e.g. the ident is at the start of a non-instruction token). +--- @param tok string +--- @return integer|nil local function find_consuming_paren(tok) +--- @type integer local i = 1 while i <= #tok do +--- @type string local c = tok:sub(i, i) if c == "(" then return i end if not c:match("[%w_]") and c ~= " " then return nil end @@ -410,24 +544,33 @@ local function _project_emission_inner(root_body_entry, ctx_table) return nil end +--- @param tok string +--- @param tok_line integer +--- @param consuming_encoder string|nil +--- @return nil local function emit_embedded_markers(tok, tok_line, consuming_encoder) -- When called with a non-nil `consuming_encoder`, the marker is nested inside that instruction's argument list. -- We compute each marker's arg position by counting top-level commas between the consuming instruction's `(` and the marker's start. +--- @type integer|nil local consuming_paren = nil if consuming_encoder then consuming_paren = find_consuming_paren(tok) end +--- @type integer local pos = 1 while pos <= #tok do -- Trim leading whitespace and comments before each scan. pos = M.skip_ws_and_cmt(tok, pos) if pos > #tok then break end +--- @type string|nil, integer local ident, after = M.read_ident(tok, pos) if not ident then -- Not an ident: token is a string or comment; skip or one-step. +--- @type integer local next_pos = M.skip_str_or_cmt(tok, pos) pos = (next_pos > pos) and next_pos or (pos + 1) goto continue_loop end if M.DELAY_MARKERS[ident] then +--- @type integer|nil local arg_pos = nil if consuming_encoder and consuming_paren then arg_pos = count_top_level_commas(tok, consuming_paren + 1, pos) + 1 @@ -442,7 +585,9 @@ local function _project_emission_inner(root_body_entry, ctx_table) goto continue_loop end -- Marker ident: parse the (...) arguments. +--- @type integer local open = M.skip_ws_and_cmt(tok, after) +--- @type string|nil, integer local inner, after_paren = M.read_parens(tok, open) if not inner then -- (...) Unreadable: fall back to non-marker behavior. @@ -453,10 +598,12 @@ local function _project_emission_inner(root_body_entry, ctx_table) -- For embedded markers, propagate the consuming_encoder + the marker's arg position -- (1-based) so `passes/offsets.lua` can dispatch per-consuming-instruction offset encoding. -- Offset markers are emitted only when a consuming encoder is present. +--- @type integer|nil local arg_pos = nil if consuming_encoder and consuming_paren then arg_pos = count_top_level_commas(tok, consuming_paren + 1, pos) + 1 end +--- @type string[] local args = split_call_args(inner) if ident == "atom_label" then emit_marker("label", args[1] or "", nil, tok_line, nil, nil, consuming_encoder, arg_pos) elseif consuming_encoder then emit_marker("offset", args[1] or "", args[2] or "", tok_line, nil, nil, consuming_encoder, arg_pos) @@ -466,6 +613,13 @@ local function _project_emission_inner(root_body_entry, ctx_table) end end +--- @param inv_kind string +--- @param component_name string +--- @param call_text string +--- @param root_call_text string|nil +--- @param call_path string +--- @param call_line integer +--- @return InvocationRecord local function emit_invoke_begin(inv_kind, component_name, call_text, root_call_text, call_path, call_line) next_inv_id = next_inv_id + 1 @@ -476,7 +630,9 @@ local function _project_emission_inner(root_body_entry, ctx_table) -- The walker has already found the component body in `ctx_table.component_index[component_name]`, so the matching entry MUST exist in `ctx_table.components[component_name]` -- (both registries are populated from the same source by the components pass). -- A missing entry is a corpus-plumbing bug; we fail loudly here rather than silently stamp `false` and mask the regression. +--- @type table|nil local components = ctx_table.components +--- @type ComponentDef|nil local component_def = components and components[component_name] or nil if not component_def then error("duffle.emit_invoke_begin: component " .. string.format("%q", component_name) @@ -486,7 +642,9 @@ local function _project_emission_inner(root_body_entry, ctx_table) , 0 ) end +--- @type boolean local debug_skip_stamp = component_def.debug_skip == true +--- @type InvocationRecord local inv = { id = next_inv_id, parent_id = 0, -- patched below by caller @@ -521,6 +679,8 @@ local function _project_emission_inner(root_body_entry, ctx_table) return inv end +--- @param inv InvocationRecord +--- @return nil local function emit_invoke_end(inv) -- 0-based emitted-word position of the LAST word inside this invocation. -- After the last body word was emitted, `word_idx` was incremented past it, so `word_idx - 1` is the 0-based position of the last word. @@ -532,6 +692,7 @@ local function _project_emission_inner(root_body_entry, ctx_table) word_index = word_idx, invocation_ids = open_invocation_ids_snapshot(), } +--- @type integer for i = #invocation_stack, 1, -1 do if invocation_stack[i] == inv then table.remove(invocation_stack, i) @@ -542,9 +703,14 @@ local function _project_emission_inner(root_body_entry, ctx_table) -- Resolve the per-token word count. If unresolved, surface ONE warning -- and fall back to 1 opaque word so the cycle budget still accounts for the slot. +--- @param ident string +--- @param tok_line integer +--- @return integer local function resolve_count(ident, tok_line) +--- @type WordCounts|nil local wc = ctx_table.word_counts if wc and wc[ident] then return wc[ident] end +--- @type string local canon = M.gte_canon(ident) if canon ~= ident and wc and wc[canon] then return wc[canon] end warnings[#warnings + 1] = { @@ -561,24 +727,40 @@ local function _project_emission_inner(root_body_entry, ctx_table) -- walk_root_call_text: Outermost `mac_X(...)` token text (preserved across recursion). -- walk_immediate_call_text: IMMEDIATE outer `mac_X(...)` token text for words emitted in this body — nil for the root atom body. -- Two trackers are propagated as separate parameters so words deep inside nested expansions correctly identify both their immediate call site and the outermost call site. +--- @param body_entry ComponentBodyEntry +--- @param walk_parent_inv_id integer +--- @param walk_root_call_text string|nil +--- @param walk_immediate_call_text string|nil +--- @return nil local function walk_body_entry(body_entry, walk_parent_inv_id, walk_root_call_text, walk_immediate_call_text) +--- @type BodyToken[] local tokens = body_entry.body_tokens or {} +--- @type integer local body_off = body_entry.body_off or 0 +--- @type LineIndexFn local line_of = body_entry.line_of or M.LineIndex("") +--- @type string local def_source = body_entry.source or "" +--- @type integer local def_line = body_entry.declaration or 0 +--- @type table|nil local sub_map = body_entry.sub_map -- Per-token dispatch: each matched branch returns; only the fall-through -- "opaque word" emit handles direct encoders + mac_X-without-component. +--- @param bt BodyToken +--- @return nil local function process_token(bt) +--- @type string local tok = M.trim(bt.tok or "") -- Substituted MipsCode args can carry // comments from the call site. while tok ~= "" do if tok:sub(1, 2) == "//" then +--- @type integer|nil local nl = tok:find("\n") tok = M.trim(nl and tok:sub(nl + 1) or "") elseif tok:sub(1, 2) == "/*" then +--- @type integer|nil local close = tok:find("*/", 3, true) tok = M.trim(close and tok:sub(close + 2) or "") else @@ -586,19 +768,25 @@ local function _project_emission_inner(root_body_entry, ctx_table) end end if tok == "" then return end +--- @type string|nil, integer local ident, after = M.read_ident(tok, 1) if not ident then ident = "?" end +--- @type string, string[] local _, args = token_ident_and_args(tok) +--- @type integer local tok_line = line_of(body_off + bt.rel) or 0 if M.DELAY_MARKERS[ident] then emit_marker("delay", ident, nil, tok_line) +--- @type string local rest = tok:sub(after or (#tok + 1)) while true do rest = M.trim(rest) if rest:sub(1, 2) == "//" then +--- @type integer|nil local nl = rest:find("\n") rest = nl and rest:sub(nl + 1) or "" elseif rest:sub(1, 2) == "/*" then +--- @type integer|nil local close = rest:find("*/", 3, true) if not close then rest = ""; break end rest = rest:sub(close + 2) @@ -615,6 +803,7 @@ local function _project_emission_inner(root_body_entry, ctx_table) -- Pass `ident` as the consuming instruction so `emit_embedded_markers` can compute each marker's arg position + record the consuming_encoder for the offsets pass. -- Canonicalize `jump_rel` to `branch_equal` (its preprocessor-expanded form) so the `consuming_encoder` metadata in marker records is canonical. -- `jump_rel`: unconditional jump alias from `code/duffle/mips.h`. +--- @type string local consuming_encoder_for_markers = (ident == "jump_rel") and "branch_equal" or ident if ident ~= "atom_label" and ident ~= "atom_offset" then emit_embedded_markers(tok, tok_line, consuming_encoder_for_markers) @@ -628,6 +817,7 @@ local function _project_emission_inner(root_body_entry, ctx_table) -- MipsCode formals (nop_slot1, …): the ident is a sub_map key. -- Re-process the replacement token so load_word(...) becomes a real encoder. if sub_map and type(sub_map[ident]) == "string" and sub_map[ident] ~= ident then +--- @type string local repl = M.trim(sub_map[ident]) if repl ~= "" then process_token({ tok = repl, rel = bt.rel }) @@ -635,15 +825,20 @@ local function _project_emission_inner(root_body_entry, ctx_table) end end if ident:sub(1, 4) == "mac_" then +--- @type string local bare = ident:sub(5) +--- @type ComponentBodyEntry|nil local comp = ctx_table.component_index[bare] if comp then +--- @type string local invocation_root_call_text = walk_root_call_text or tok if ctx_table.visiting[bare] then -- Cycle: still allocate inv_id, emit zero-width begin/end, record the cycle error; do NOT recurse. +--- @type InvocationRecord local inv = emit_invoke_begin(comp.kind or "comp_bare", bare, tok, invocation_root_call_text, def_source, tok_line) inv.parent_id = walk_parent_inv_id inv.call_text = tok +--- @type EmitError local err = { kind = "cycle", msg = string.format("project_emission: component cycle detected: %q", bare), @@ -657,6 +852,7 @@ local function _project_emission_inner(root_body_entry, ctx_table) end -- First visit: descend + count + count_mismatch-check below. ctx_table.visiting[bare] = true +--- @type InvocationRecord local inv = emit_invoke_begin(comp.kind or "comp_bare", bare, tok, invocation_root_call_text, def_source, tok_line) inv.parent_id = walk_parent_inv_id inv.call_text = tok @@ -665,11 +861,14 @@ local function _project_emission_inner(root_body_entry, ctx_table) -- Propagate trackers into the recursive walk: -- immediate_call_text = this call's tok (the IMMEDIATE outer call for words emitted in this body) -- root_call_text = the OUTERMOST call (immutable across the recursion) +--- @type string[]|nil local formal_names = ctx_table.component_index[bare] and ctx_table.component_index[bare].arg_names +--- @type table|nil local child_map = nil if formal_names then child_map = {} +--- @type integer, string for i, fname in ipairs(formal_names) do child_map[fname] = apply_sub(sub_map, args[i]) end @@ -688,8 +887,11 @@ local function _project_emission_inner(root_body_entry, ctx_table) ctx_table.visiting[bare] = nil emit_invoke_end(inv) -- Count `word` items inside [start_word, end_word]. +--- @type integer local wc_inside = 0 +--- @type integer for i = inv.start_word, inv.end_word do +--- @type EmissionItem|nil local it = items[i] if it and it.kind == "word" then wc_inside = wc_inside + 1 @@ -698,8 +900,10 @@ local function _project_emission_inner(root_body_entry, ctx_table) inv.word_count = wc_inside -- count_mismatch is a construction error: word_counts["mac_X"] is the declared count populated by the components pass; -- We compare against the measured word count. +--- @type integer|nil local declared = ctx_table.word_counts["mac_" .. bare] if declared and wc_inside ~= declared then +--- @type EmitError local err = { kind = "count_mismatch", msg = string.format("project_emission: mac_%s declared=%d measured=%d", bare, declared, wc_inside), @@ -715,13 +919,17 @@ local function _project_emission_inner(root_body_entry, ctx_table) end -- Direct encoder, or mac_X-without-component: resolve count + emit n words. -- Resolve_count may emit a warning if the count is unresolved. +--- @type integer local n = resolve_count(ident, tok_line) +--- @type string local out_ident = (ident == "nop2") and "nop" or ident +--- @type integer for _ = 1, n do emit_word(out_ident, args, tok_line, tok, def_source, def_line, walk_immediate_call_text, walk_root_call_text, sub_map) end end +--- @type integer, BodyToken for _, bt in ipairs(tokens) do process_token(bt) end @@ -780,11 +988,12 @@ end --- for the open invocation stack at that word. --- --- @param body_text string -- the raw atom body string ---- @param component_index table -- bare-name → component record (corpus.component_body_index) ---- @param word_counts table -- macro name → emitted word count ---- @param components table -- bare-name → component definition (corpus.components); REQUIRED — consumed at the invocation-construction site to stamp +--- @param component_index table +--- @param word_counts WordCounts +--- @param components table --- `invocation.debug_skip`. A missing or non-table `components` raises a fail-loud error rather than silently falling back. --- @return EmissionProjection +--- @param reg_use_ctx RegUseCtx|nil function M.project_emission(body_text, component_index, word_counts, components, reg_use_ctx) -- The recursive walk delegates to `_project_emission_inner` so component bodies (which arrive as -- `{body_tokens, body_off, line_of, source, declaration}` records from `corpus.component_body_index`) @@ -816,6 +1025,7 @@ function M.project_emission(body_text, component_index, word_counts, components, } end +--- @type BodyToken[] local tokens = M.tokenize_body(body_text) return _project_emission_inner({ body_tokens = tokens, @@ -849,10 +1059,17 @@ end -- (FI_, atom_dbg_skip, comments) until it finds an ident followed by "(". -- That ident is the function name; the parens contents are the args. ------------------------------------------------------------------------------- +--- @param source string +--- @param before_pos integer +--- @param slice_mips_code_len integer +--- @return string|nil, string|nil function M.find_function_decl_for(source, before_pos, slice_mips_code_len) +--- @type integer local search_pos = 1 +--- @type integer|nil local last_match = nil while true do +--- @type integer|nil local found = source:find("Slice_MipsCode", search_pos, true) if not found or found >= before_pos then break end last_match = found @@ -860,10 +1077,12 @@ function M.find_function_decl_for(source, before_pos, slice_mips_code_len) end if not last_match then return nil, nil end +--- @type integer local pos = last_match + slice_mips_code_len while pos < before_pos do -- skip whitespace while pos <= #source do +--- @type string local c = source:sub(pos, pos) if c == " " or c == "\t" or c == "\n" or c == "\r" then pos = pos + 1 @@ -880,17 +1099,21 @@ function M.find_function_decl_for(source, before_pos, slice_mips_code_len) end -- skip block comments if source:sub(pos, pos + 1) == "/*" then +--- @type integer|nil local close = source:find("*/", pos + 2, true) if not close then break end pos = close + 2 goto continue end -- try to read an ident +--- @type string|nil, integer local ident, ident_end = M.read_ident(source, pos) if not ident then break end -- check if the next non-ws char after ident is "(" +--- @type integer local next_pos = M.skip_ws_and_cmt(source, ident_end) if source:sub(next_pos, next_pos) == "(" then +--- @type string|nil local inner = M.read_parens(source, next_pos) if inner then return ident, inner @@ -917,11 +1140,18 @@ end -- The walk finds the LAST "MipsAtom*" before before_pos, then skips whitespace + qualifiers (internal, I_, FI_, comments) -- until it finds an ident followed by "(". ------------------------------------------------------------------------------- +--- @param source string +--- @param before_pos integer +--- @param mips_atom_ptr_len integer +--- @return string|nil, string|nil, string|nil, integer|nil function M.find_atom_proc_decl_for(source, before_pos, mips_atom_ptr_len) +--- @type integer local search_pos = 1 +--- @type integer|nil local last_match = nil while true do -- plain=true: "*" is literal, no escaping needed +--- @type integer|nil local found = source:find("MipsAtom*", search_pos, true) if not found or found >= before_pos then break end last_match = found @@ -929,10 +1159,12 @@ function M.find_atom_proc_decl_for(source, before_pos, mips_atom_ptr_len) end if not last_match then return nil, nil end +--- @type integer local pos = last_match + mips_atom_ptr_len while pos < before_pos do -- skip whitespace while pos <= #source do +--- @type string local c = source:sub(pos, pos) if c == " " or c == "\t" or c == "\n" or c == "\r" then pos = pos + 1 @@ -949,17 +1181,21 @@ function M.find_atom_proc_decl_for(source, before_pos, mips_atom_ptr_len) end -- skip block comments if source:sub(pos, pos + 1) == "/*" then +--- @type integer|nil local close = source:find("*/", pos + 2, true) if not close then break end pos = close + 2 goto continue end -- try to read an ident +--- @type string|nil, integer local ident, ident_end = M.read_ident(source, pos) if not ident then break end -- check if the next non-ws char after ident is "(" +--- @type integer local next_pos = M.skip_ws_and_cmt(source, ident_end) if source:sub(next_pos, next_pos) == "(" then +--- @type string|nil, integer local inner, after_paren = M.read_parens(source, next_pos) if inner then return ident, inner, ident, after_paren diff --git a/scripts/duffle_isa.lua b/scripts/duffle_isa.lua index 28ee37e..d9af32b 100644 --- a/scripts/duffle_isa.lua +++ b/scripts/duffle_isa.lua @@ -1,16 +1,126 @@ --- duffle_isa.lua — encoder / GTE / hardware tables. + +--- @class InstructionImm +--- @field arg integer +--- @field signed boolean|nil +--- @field width integer + +--- @class InstructionValue +--- @field dest integer +--- @field op string +--- @field sources integer[]|nil +--- @field immediate integer|nil +--- @field source integer|nil + +--- @class InstructionRow +--- @field cycles integer +--- @field kind string +--- @field reads integer[]|nil +--- @field writes integer[]|nil +--- @field imm InstructionImm[]|nil +--- @field value InstructionValue|nil +--- @field delay_slot boolean|nil +--- @field suppress_arg1 table|nil -- bag: GPR ident -> reason + +--- @class TapeAtomMacroRow +--- @field kind string +--- @field binds boolean + +--- @class GteCommandPort +--- @field register string +--- @field role string + +--- @class GteCommandLatch +--- @field register string +--- @field required integer + +--- @class GteCommandRow +--- @field aliases string[] +--- @field cycles integer +--- @field inputs string[] +--- @field outputs GteCommandPort[] +--- @field latch GteCommandLatch[] + +--- @class GteCrAliasGroup +--- @field [1] integer -- C2 control-register slot +--- @field [2] string[] -- aliases that share that slot + +--- @class GtePackedSlotRelation +--- @field slot integer +--- @field first string +--- @field second string + +--- @class HardwareRelationPort +--- @field domain string +--- @field arg integer + +--- @class HardwareRelationVisibility +--- @field kind string +--- @field required integer + +--- @class HardwareRelationEvidence +--- @field confidence string +--- @field source string + +--- @class HardwareRelationRow +--- @field id string +--- @field semantic string +--- @field consumer string +--- @field token string +--- @field direction string +--- @field reads HardwareRelationPort +--- @field writes HardwareRelationPort +--- @field visibility HardwareRelationVisibility|nil +--- @field evidence HardwareRelationEvidence +--- @field violation_kind string +--- @field destination_match string|nil +--- @field fanout_to string[]|nil +--- @field required integer|nil +--- @field clear_on_consumer boolean|nil +--- @field stage boolean|nil +--- @field cu2_transition boolean|nil +--- @field status_register integer|nil + +--- @class Cu2TransitionPolicy +--- @field status_register integer +--- @field enable_bit integer +--- @field required integer +--- @field visibility_kind string +--- @field evidence HardwareRelationEvidence + +--- @class DuffleIsa +--- @field TAPE_ATOM_MACROS table +--- @field DELAY_MARKERS table +--- @field INSTRUCTION table +--- @field GTE_COMMAND table +--- @field ALIAS_TO_CANONICAL table +--- @field instr fun(ident: string): InstructionRow|nil +--- @field gte_canon fun(ident: string): string +--- @field gte fun(ident: string): GteCommandRow|nil +--- @field GTE_CR_ALIAS_GROUPS GteCrAliasGroup[] +--- @field GTE_PACKED_SLOT_RELATIONS GtePackedSlotRelation[] +--- @field OPERAND_READ_POSITIONS table +--- @field GP0_CMD_SIZE table +--- @field GP0_CMD_BY_SHAPE table +--- @field UNKNOWN_INSTRUCTION_CYCLES integer +--- @field HARDWARE_RELATIONS HardwareRelationRow[] +--- @field CU2_TRANSITION_POLICY Cu2TransitionPolicy + +--- @type DuffleIsa local M = {} -- Section 7: domain tables -- ════════════════════════════════════════════════════════════════════════════ -- atom_info sub-calls: atom_bind, atom_reads, atom_writes, atom_view, atom_reg_types, atom_ctx, atom_phase. +--- @type table M.TAPE_ATOM_MACROS = { ["atom_info"] = { kind = "info", binds = false }, } -- Empty C macros that prefix the next encoder. Zero words. -- BdSlot_ nop is one nop word. The marker is not the BD instruction. +--- @type table -- bag: marker prefix -> true M.DELAY_MARKERS = { ["GteDelay_"] = true, ["LdSlot_"] = true, @@ -19,6 +129,7 @@ M.DELAY_MARKERS = { } -- One row per encoder. Read through duffle.instr. +--- @type table M.INSTRUCTION = { ["BdSlot_"] = { cycles = 0, kind = "marker", }, ["LdSlot_"] = { cycles = 0, kind = "marker", }, @@ -115,6 +226,7 @@ M.INSTRUCTION = { } -- One row per GTE command. Alias cycle numbers live here, not on INSTRUCTION. +--- @type table M.GTE_COMMAND = { ["gte_cmdw_avsz3"] = { aliases = { "gte_avg_sort_z3", "gte_avsz3", "gte_cmdw_avg_sort_z3" }, @@ -296,14 +408,24 @@ M.GTE_COMMAND = { }, } +--- @param ident string +--- @return InstructionRow|nil function M.instr (ident) return M.INSTRUCTION [ident] end +--- @param ident string +--- @return string function M.gte_canon(ident) return M.ALIAS_TO_CANONICAL [ident] or ident end +--- @param ident string +--- @return GteCommandRow|nil function M.gte (ident) return M.GTE_COMMAND[M.gte_canon(ident)] end +--- @return nil local function build_alias_map() + --- @type table -- bag: alias or canon -> canon M.ALIAS_TO_CANONICAL = {} + --- @type string, GteCommandRow for canon, row in pairs(M.GTE_COMMAND) do M.ALIAS_TO_CANONICAL[canon] = canon + --- @type integer, string for _, alias in ipairs(row.aliases or {}) do M.ALIAS_TO_CANONICAL[alias] = canon end @@ -319,6 +441,7 @@ build_alias_map() --- Cross-alias writes inside one atom body, or across the wave-context boundary, silently clobber each other. --- The `check_gte_cr_alias_writes` check warns about each pair per source. See `docs/gte_reference.md` §"Control-register alias table" --- for the HW rationale and the libgte outer-product convention. +--- @type GteCrAliasGroup[] M.GTE_CR_ALIAS_GROUPS = { { 24, { "gte_cr_RBK", "gte_cr_OFX" } }, -- background R vs screen offset X { 25, { "gte_cr_GBK", "gte_cr_OFY" } }, -- background G vs screen offset Y @@ -326,6 +449,7 @@ M.GTE_CR_ALIAS_GROUPS = { } -- Packed RT slots named by the gte.h packed-slot comment. First must be written before second. +--- @type GtePackedSlotRelation[] M.GTE_PACKED_SLOT_RELATIONS = { { slot = 2, first = "gte_cr_RT13", second = "gte_cr_RT22" }, } @@ -340,6 +464,7 @@ M.GTE_PACKED_SLOT_RELATIONS = { -- * The check tracks one entry per destination GPR per MFC2 / CFC2 event. -- A subsequent event counts as a "use" iff any of its read operand positions reference that destination GPR's ident (e.g. `R_T0`). -- * Branch delay slots are out of scope (MIPS control-flow; tracked separately). +--- @type table -- bag: encoder ident -> GPR operand positions M.OPERAND_READ_POSITIONS = { -- CPU ALU with one or two GPR operands. Reads every GPR operand. ["add_ui"] = {1, 2}, @@ -439,6 +564,7 @@ M.OPERAND_READ_POSITIONS = { -- set_poly_gt3(p) -> set_len(p, 9) -> 10 total GP0 0x34 -- set_poly_g4(p) -> set_len(p, 8) -> 9 total GP0 0x38 -- set_poly_gt4(p) -> set_len(p, 12) -> 13 total GP0 0x3C +--- @type table -- bag: GP0 cmd byte -> word count M.GP0_CMD_SIZE = { [0x20] = 5, -- Poly_F3 [0x24] = 8, -- Poly_FT3 @@ -452,6 +578,7 @@ M.GP0_CMD_SIZE = { -- Shape suffix (after `ac_format_` / `mac_format_` prefix) -> GP0 cmd byte. -- Lets the static-analysis check derive the cmd byte from a macro name like `mac_format_g4_color` -> `g4` -> 0x38 -> 9 expected words. +--- @type table -- bag: shape suffix -> GP0 cmd byte M.GP0_CMD_BY_SHAPE = { ["f3"] = 0x20, ["ft3"] = 0x24, ["f4"] = 0x28, ["ft4"] = 0x2C, @@ -459,6 +586,7 @@ M.GP0_CMD_BY_SHAPE = { ["g4"] = 0x38, ["gt4"] = 0x3C, } +--- @type integer M.UNKNOWN_INSTRUCTION_CYCLES = 1 -- Hardware-relation policy table. @@ -491,6 +619,7 @@ M.UNKNOWN_INSTRUCTION_CYCLES = 1 -- * passes/static_analysis.lua::analyze_hardware_relations (forward walker). -- * passes/static_analysis.lua::transfer_hazards CHECK_RULES reader (renders hazards onto `findings`). -- This table is consumed by the hardware-relation analyzer and hazard renderer. +--- @type HardwareRelationRow[] M.HARDWARE_RELATIONS = { -- CPU → COP2 data register (MTC2). The ordinary default is 2 cached words between producer and consumer (cpuspecifications.md:407-419). { @@ -681,6 +810,7 @@ M.HARDWARE_RELATIONS = { -- Bounded Status/SR.CU2 transition policy. -- The value lattice and the transition consumer both read this immutable row; no second value pass is permitted. -- The source says the enable/disable transition takes "2 clock cycles or so", so the boundary is conservative rather than exact. +--- @type Cu2TransitionPolicy M.CU2_TRANSITION_POLICY = { status_register = 12, enable_bit = 0x40000000, diff --git a/scripts/duffle_paths.lua b/scripts/duffle_paths.lua index 5aee365..43a45ee 100644 --- a/scripts/duffle_paths.lua +++ b/scripts/duffle_paths.lua @@ -16,9 +16,14 @@ --- Net effect: the caller gets the duffle module in one statement; no separate `dofile(...)` + `require("duffle")` dance. --- +--- @class DufflePaths +--- @field setup fun(): nil + +--- @type DufflePaths local M = {} -- Cache key for the repo root. Stored in `package.loaded` (process-global) so all 8 entry scripts + passes scripts share one resolution. +--- @type string local CACHE_KEY = "__duffle_repo_root__" --- Resolve the repo root from this script's own path. Zero shell spawn. @@ -30,13 +35,16 @@ local CACHE_KEY = "__duffle_repo_root__" local function find_repo_root() if package.loaded[CACHE_KEY] then return package.loaded[CACHE_KEY] end + --- @type string local source = debug.getinfo(1, "S").source -- Strip the leading `@` (Lua's dofile marker) and the trailing `/duffle_paths.lua` filename. -- What remains is the directory containing this script, i.e. `/scripts/`. + --- @type string|nil local scripts_dir = source and source:match("^@?(.*)[/\\]duffle_paths%.lua$") if not scripts_dir then return nil end -- The repo root is the parent of `scripts/`. Strip the trailing `scripts/` (with or without trailing slash). + --- @type string local root = scripts_dir:gsub("scripts[\\/]?$", "") root = root:gsub("\\", "/") if root == "" then root = "./" end @@ -50,7 +58,9 @@ end --- This script does NOT touch the OS environment: no `os.setenv`, no `os.putenv`, no `$PATH` mods. --- It just sets `package.path` and `package.cpath` (the standard Lua way to register module search dirs). --- lpeg is built by `update_deps.ps1` to `toolchain/lpeg/`, which we wire into `package.cpath` here (so `require("lpeg")` from `duffle.lua` resolves without any global state). +--- @return nil function M.setup() + --- @type string|nil local repo_root = find_repo_root() if not repo_root then -- Unreachable in practice: find_repo_root() derives the repo root from this script's own source path via debug.getinfo(1, "S").source (no subprocess, no git CLI, <1ms). @@ -59,7 +69,9 @@ function M.setup() os.exit(2) end + --- @type string local scripts_dir = repo_root .. "scripts/" + --- @type string local passes_dir = repo_root .. "scripts/passes/" package.path = scripts_dir .. "?.lua;" .. scripts_dir .. "?/init.lua;" @@ -70,7 +82,9 @@ function M.setup() -- lpeg: built by `update_deps.ps1` to `toolchain/lpeg/lpeg.dll`. -- lfs: compiled from pcsx-redux's vendored luafilesystem source to `toolchain/lfs/lfs.dll`. -- Wire both directories into cpath so `require("lpeg")` and `require("lfs")` resolve. + --- @type string local lpeg_dir = repo_root .. "toolchain/lpeg/" + --- @type string local lfs_dir = repo_root .. "toolchain/lfs/" package.cpath = lpeg_dir .. "?.dll;" .. lfs_dir .. "?.dll;" diff --git a/scripts/duffle_scan.lua b/scripts/duffle_scan.lua index 381f0ef..a794fb8 100644 --- a/scripts/duffle_scan.lua +++ b/scripts/duffle_scan.lua @@ -1,4 +1,4 @@ ---- duffle.lua — shared primitives + domain tables for the tape-atom metaprograms. +--- duffle_scan.lua — shared primitives + domain tables for the tape-atom metaprograms. --- * Character classification: `is_space`, `is_alpha`, `is_alnum`, `is_digit`, plus the byte-fast `_byte` variants. --- * String / path primitives: `trim`, `dirname`, `basename_no_ext`, `normalize_path`, `canonical_path_key`, `find_byte`. --- * I/O primitives: `read_file`, `write_file`, `ensure_dir`. @@ -7,61 +7,130 @@ --- * Word-count loader: `load_word_counts` for `WORD_COUNT(...)` metadata files. --- * Line lookup: `LineIndex` returns an O(log N) `line_of(pos)` closure for source-mapping. --- * Domain tables: `TAPE_ATOM_MACROS`, `GTE_PIPELINE_LATENCY`, `GP0_CMD_SIZE`, `GP0_CMD_BY_SHAPE`, `INSTRUCTION_LATENCY`. +--- +--- **Type aliases** (Path, LineNum, ByteOff, MacroName, AtomName, Severity) and **@class SourceFile** +--- live on `duffle.lua`. This file re-exports `M` only. + +--- @class LfsMod +--- @field currentdir fun(): string|nil +--- @field attributes fun(path: string, request: string|nil): string|nil +--- @field mkdir fun(path: string): boolean|nil + +--- @class LpegPattern + +--- @alias LpegCtor fun(...: any): LpegPattern + +--- @class LpegMod +--- @field P LpegCtor +--- @field S LpegCtor +--- @field R LpegCtor +--- @field C fun(p: LpegPattern): LpegPattern +--- @field match fun(p: LpegPattern, s: string, init: integer|nil): any + +--- @class PathRoot +--- @field kind string +--- @field prefix string +--- @field rest string +--- @field anchored boolean + +--- @class QuotedInclude +--- @field path string +--- @field include_path string +--- @field include_text string +--- @field line integer + +--- @class ResolverEvidence +--- @field include_path string|nil +--- @field include_text string|nil +--- @field root_source string +--- @field root_line integer +--- @field candidate_a string|nil +--- @field candidate_b string|nil +--- @field candidate_a_in_code_root boolean|nil +--- @field candidate_b_in_code_root boolean|nil +--- @field selected_path string|nil +--- @field disposition string|nil +--- @field reason string|nil +--- @field duplicate_of string|nil +--- @field alternate_path string|nil + +--- @class ResolveOptions +--- @field unity_root Path +--- @field project_root Path + +--- @alias LineIndexFn fun(query_pos: integer): integer + +--- @class DuffleScan + +--- @class SourceResolver +--- @field resolved ResolverEvidence[] +--- @field skipped ResolverEvidence[] +--- @field shadowed ResolverEvidence[] + +--- @type DuffleScan local M = {} -- Required native extension: lfs (LuaFileSystem). Built by `update_deps.ps1` to `toolchain/lfs/lfs.dll` and wired into package.cpath by `scripts/duffle_paths.lua`. -- If lfs is missing, `require` throws — fail loud per the build-tool convention. +--- @type LfsMod local lfs = require("lfs") --- ════════════════════════════════════════════════════════════════════════════ --- Cross-file type aliases --- ════════════════════════════════════════════════════════════════════════════ - ---- @alias Path string -- Absolute or CWD-relative file path ---- @alias LineNum integer -- 1-indexed source line number ---- @alias ByteOff integer -- 0-indexed byte offset within a source string ---- @alias MacroName string -- lower_snake_case macro identifier (e.g. "mac_yield") ---- @alias AtomName string -- lower_snake_case atom name (e.g. "cube_g4_face") ---- @alias Severity string -- "error" | "warning" | "info" - ---- @class SourceFile ---- @field path Path -- Absolute path to the source file ---- @field text string -- Full source text ---- @field dir string -- Directory containing the source ---- @field basename string -- Filename without extension - -- ════════════════════════════════════════════════════════════════════════════ -- ASCII byte constants -- ════════════════════════════════════════════════════════════════════════════ +--- @type integer local BYTE_SPACE = 0x20 -- ' ' +--- @type integer local BYTE_TAB = 0x09 -- '\t' +--- @type integer local BYTE_NEWLINE = 0x0A -- '\n' +--- @type integer local BYTE_CR = 0x0D -- '\r' +--- @type integer local BYTE_VT = 0x0B -- '\v' +--- @type integer local BYTE_FF = 0x0C -- '\f' +--- @type integer local BYTE_UNDERSCORE = 0x5F -- '_' +--- @type integer local BYTE_DOT = 0x2E -- '.' +--- @type integer local BYTE_SLASH = 0x2F -- '/' +--- @type integer local BYTE_BACKSLASH = 0x5C -- '\\' +--- @type integer local BYTE_STAR = 0x2A -- '*' +--- @type integer local BYTE_DQUOTE = 0x22 -- '"' +--- @type integer local BYTE_SQUOTE = 0x27 -- '\'' +--- @type integer local BYTE_COMMA = 0x2C -- ',' +--- @type integer local BYTE_SEMI = 0x3B -- ';' +--- @type integer local BYTE_OPEN_PAREN = 0x28 -- '(' +--- @type integer local BYTE_OPEN_BRACE = 0x7B -- '{' +--- @type integer local BYTE_OPEN_BRACK = 0x5B -- '[' +--- @type integer local BYTE_LOWER_A = 0x61 -- 'a' +--- @type integer local BYTE_LOWER_Z = 0x7A -- 'z' +--- @type integer local BYTE_UPPER_A = 0x41 -- 'A' +--- @type integer local BYTE_UPPER_Z = 0x5A -- 'Z' +--- @type integer local BYTE_DIGIT_0 = 0x30 -- '0' +--- @type integer local BYTE_DIGIT_9 = 0x39 -- '9' -- ════════════════════════════════════════════════════════════════════════════ @@ -76,6 +145,7 @@ local BYTE_DIGIT_9 = 0x39 -- '9' -- LPeg handles the high-level scanner; the byte-by-byte helpers in Section 1 handle classification primitives that LPeg's CPython-level cost would dominate. -- -- If the require fails, fail loud with an actionable message. The build script (`update_deps.ps1`) builds lpeg.dll into `toolchain/lpeg/`; run it when the dll is missing. +--- @type boolean, LpegMod|string local lpeg_ok, lpeg = pcall(require, "lpeg") if not lpeg_ok then io.stderr:write("[duffle] require('lpeg') failed: ", lpeg, "\n") @@ -83,30 +153,45 @@ if not lpeg_ok then io.stderr:write("[duffle] Run 'scripts/update_deps.ps1' to build it into toolchain/lpeg/.\n") os.exit(1) end +--- @type LpegCtor, LpegCtor, LpegCtor local P, S, R = lpeg.P, lpeg.S, lpeg.R -- Character class patterns +--- @type LpegPattern local alpha_pat = R("AZ", "az") + P("_") +--- @type LpegPattern local digit_pat = R("09") +--- @type LpegPattern local lpeg_alnum_pat = alpha_pat + digit_pat -- Identifier: alpha followed by zero+ alnum. Capture as a string. +--- @type LpegPattern local lpeg_alpha_pat = alpha_pat +--- @type LpegPattern local lpeg_ident_pat = lpeg.C(alpha_pat * lpeg_alnum_pat^0) +--- @type LpegPattern local lpeg_str_pat = P('"') * (P(1) - S('"\\') + P('\\') * P(1))^0 * P('"') -- String literal: "..." with backslash escapes. +--- @type LpegPattern local lpeg_chr_pat = P("'") * (P(1) - S("'\\") + P('\\') * P(1))^0 * P("'") -- Char literal: '...' with backslash escapes. +--- @type LpegPattern local lpeg_line_cmt_pat = P("//") * (P(1) - S("\n"))^0 -- Line comment: // ... to end-of-line. +--- @type LpegPattern local lpeg_block_cmt_pat = P("/*") * (P(1) - P("*/"))^0 * P("*/") -- Block comment: /* ... */ (no nesting per C standard). +--- @type LpegPattern local lpeg_str_or_cmt_pat = lpeg_str_pat + lpeg_chr_pat + lpeg_line_cmt_pat + lpeg_block_cmt_pat -- String or comment (any of the four forms). -- Whitespace + comment skipper: zero+ (whitespace run | string | comment). +--- @type LpegPattern local ws_pat = S(" \t\n\r\v\f") +--- @type LpegPattern local lpeg_ws_and_cmt_pat = (ws_pat + lpeg_str_or_cmt_pat)^0 -- Generic "skip until target, but step over balanced groups" matcher. -- Used by scan_to_char for non-ident / non-bracket chars. We accept any single char except the target. -- The balanced-group stepping is handled by the caller (via read_balanced). +--- @param target string +--- @return LpegPattern local lpeg_scan_to_target_pat = function(target) return (P(1) - P(target))^0 end -- ════════════════════════════════════════════════════════════════════════════ @@ -116,9 +201,13 @@ local lpeg_scan_to_target_pat = function(target) return (P(1) - P(target))^0 en -- Used in all hot loops because they avoid the string allocation per s:sub(pos, pos) call. -- Whitespace characters per C locale. +--- @param b integer +--- @return boolean function M.is_space_byte(b) return b == BYTE_SPACE or b == BYTE_TAB or b == BYTE_NEWLINE or b == BYTE_CR or b == BYTE_VT or b == BYTE_FF end -- Letters (a-z, A-Z) and underscore. +--- @param b integer|nil +--- @return boolean function M.is_alpha_byte(b) if not b then return false end if b >= BYTE_LOWER_A and b <= BYTE_LOWER_Z then return true end -- 'a'..'z' @@ -127,15 +216,23 @@ function M.is_alpha_byte(b) end -- Single digit. +--- @param b integer|nil +--- @return boolean function M.is_digit_byte(b) return b and b >= BYTE_DIGIT_0 and b <= BYTE_DIGIT_9 end -- Letter OR digit OR underscore. +--- @param b integer|nil +--- @return boolean function M.is_alnum_byte(b) return M.is_alpha_byte(b) or M.is_digit_byte(b) end -- String-based wrappers (kept for callers that already have a single-char string; the byte versions are what the hot loops should call). +--- @param c integer|string +--- @return boolean function M.is_space(c) if type(c) == "number" then return M.is_space_byte(c) end return c == " " or c == "\t" or c == "\n" or c == "\r" or c == "\v" or c == "\f" end +--- @param c integer|string|nil +--- @return boolean function M.is_alpha(c) if type(c) == "number" then return M.is_alpha_byte(c) end if not c or #c == 0 then return false end @@ -143,10 +240,14 @@ function M.is_alpha(c) if c >= "A" and c <= "Z" then return true end return c == "_" end +--- @param c integer|string|nil +--- @return boolean function M.is_digit(c) if type(c) == "number" then return M.is_digit_byte(c) end return c and c >= "0" and c <= "9" end +--- @param c integer|string|nil +--- @return boolean function M.is_alnum(c) return M.is_alpha(c) or M.is_digit(c) end -- ════════════════════════════════════════════════════════════════════════════ @@ -154,8 +255,12 @@ function M.is_alnum(c) return M.is_alpha(c) or M.is_digit(c) end -- ════════════════════════════════════════════════════════════════════════════ -- Trim leading and trailing whitespace from a string. +--- @param s string +--- @return string function M.trim(s) + --- @type integer local a = 1; while a <= #s and M.is_space_byte(s:byte(a)) do a = a + 1 end + --- @type integer local b = #s; while b >= a and M.is_space_byte(s:byte(b)) do b = b - 1 end return s:sub(a, b) end @@ -166,6 +271,7 @@ end --- @param start integer -- optional 1-indexed start (default 1) --- @return integer|nil function M.find_byte(haystack, target, start) + --- @type integer for pos = start or 1, #haystack do if haystack:byte(pos) == target then return pos end end @@ -173,9 +279,14 @@ function M.find_byte(haystack, target, start) end -- Returns the directory portion of a path. +--- @param path Path +--- @return Path function M.dirname(path) + --- @type integer local last_sep = 0 + --- @type integer for pos = 1, #path do + --- @type integer local b = path:byte(pos) if b == BYTE_SLASH or b == BYTE_BACKSLASH then last_sep = pos end end @@ -184,14 +295,22 @@ function M.dirname(path) end -- Returns the basename of a path, with the file extension stripped. +--- @param path Path +--- @return string function M.basename_no_ext(path) + --- @type integer local last_sep = 0 + --- @type integer for pos = 1, #path do + --- @type integer local b = path:byte(pos) if b == BYTE_SLASH or b == BYTE_BACKSLASH then last_sep = pos end end + --- @type integer local a = last_sep + 1 + --- @type integer local last_dot = #path + 1 + --- @type integer for pos = #path, a, -1 do if path:byte(pos) == BYTE_DOT then last_dot = pos; break end end @@ -200,10 +319,14 @@ end --- Parse the lexical root without changing display spelling. --- UNC server/share names are part of the immutable root; drive-relative paths remain distinct from drive-absolute paths. +--- @param input string +--- @return PathRoot local function parse_path_root(input) + --- @type string|nil local drive = input:match("^(%a:)") if drive then if input:sub(3, 3) == "/" then + --- @type string local rest = input:sub(4) while rest:sub(1, 1) == "/" do rest = rest:sub(2) end return { kind = "drive_absolute", prefix = drive .. "/", rest = rest, anchored = true } @@ -212,21 +335,28 @@ local function parse_path_root(input) end if input:sub(1, 2) == "//" then + --- @type integer local server_start = 3 + --- @type integer|nil local server_end = M.find_byte(input, BYTE_SLASH, server_start) if not server_end or server_end == server_start then error("UNC path requires //server/share: " .. input, 3) end + --- @type string local server = input:sub(server_start, server_end - 1) + --- @type integer local share_start = server_end + 1 while input:sub(share_start, share_start) == "/" do share_start = share_start + 1 end + --- @type integer local share_end = M.find_byte(input, BYTE_SLASH, share_start) or (#input + 1) if share_end == share_start then error("UNC path requires //server/share: " .. input, 3) end + --- @type string local share = input:sub(share_start, share_end - 1) + --- @type string local rest = input:sub(share_end + 1) while rest:sub(1, 1) == "/" do rest = rest:sub(2) end return { @@ -238,6 +368,7 @@ local function parse_path_root(input) end if input:sub(1, 1) == "/" then + --- @type string local rest = input:sub(2) while rest:sub(1, 1) == "/" do rest = rest:sub(2) end return { kind = "posix_absolute", prefix = "/", rest = rest, anchored = true } @@ -253,8 +384,11 @@ function M.normalize_path(path) if type(path) ~= "string" then error("normalize_path requires a string path", 2) end if path == "" then return "" end + --- @type PathRoot local root = parse_path_root(path:gsub("\\", "/")) + --- @type string[] local segments = {} + --- @type string for segment in root.rest:gmatch("[^/]+") do if segment == "." then -- no-op @@ -269,6 +403,7 @@ function M.normalize_path(path) end end + --- @type string local tail = table.concat(segments, "/") if root.kind == "relative" then return tail ~= "" and tail or "." end if root.kind == "drive_relative" then return root.prefix .. tail end @@ -276,8 +411,12 @@ function M.normalize_path(path) return root.prefix .. tail end +--- @param path Path +--- @return Path local function absolute_normalized_path(path) + --- @type Path local normalized = M.normalize_path(path) + --- @type PathRoot local root = parse_path_root(normalized) if root.kind == "drive_relative" then error("drive-relative path cannot be resolved without a per-drive cwd: " .. normalized, 3) @@ -291,11 +430,14 @@ end --- @param path Path --- @return string function M.canonical_path_key(path) + --- @type Path local normalized = M.normalize_path(path) + --- @type PathRoot local root = parse_path_root(normalized) if root.kind == "drive_relative" then error("canonical_path_key cannot compare drive-relative path: " .. normalized, 2) end + --- @type string local key = absolute_normalized_path(normalized):lower() if #key > 3 and key:sub(-1) == "/" then key = key:sub(1, -2) end return key @@ -308,14 +450,22 @@ end -- File contents intentionally use io.open below. -- LuaFileSystem handles path metadata, directory iteration, the current directory, and mkdir; -- it does not expose file-content read/write streams. +--- @param path Path +--- @return string function M.read_file(path) + --- @type file*|nil local f = io.open(path, "r") if not f then error("Cannot open " .. path) end + --- @type string local content = f:read("*a"); f:close() return content end +--- @param path Path +--- @param content string +--- @return nil function M.write_file(path, content) + --- @type file*|nil local f = io.open(path, "w") if not f then error("Cannot write " .. path) end f:write(content); f:close() @@ -325,12 +475,15 @@ end --- (text mode would convert LF -> CRLF, breaking byte-identical diffs against git-tracked gen/*.h files which are stored as LF). --- @param path string --- @param content string +--- @return nil function M.write_file_lf(path, content) + --- @type file*|nil local f = io.open(path, "wb") if not f then error("Cannot write " .. path) end f:write(content); f:close() end +--- @type table -- bag: input path -> absolute path local _absolute_path_cache = {} --- Convert a (possibly relative) path to an absolute path, using CWD if needed. @@ -343,22 +496,29 @@ function M.to_absolute_path(path) if _absolute_path_cache[path] then return _absolute_path_cache[path] end if #path >= 2 and path:sub(2, 2) == ":" then -- Already absolute; normalize slashes for consistency. + --- @type string local result = (path:gsub("/", "\\")) _absolute_path_cache[path] = result return result end + --- @type string|nil local cwd = lfs.currentdir() if not cwd then _absolute_path_cache[path] = path; return path end cwd = cwd:gsub("/", "\\") + --- @type string local tail = (path:gsub("/", "\\")) + --- @type string local result = cwd .. "\\" .. tail _absolute_path_cache[path] = result return result end -- Cache of directories already verified to exist in this process. +--- @type table -- bag: dir path -> already ensured local _ensured_dirs = {} +--- @param path Path +--- @return nil function M.ensure_dir(path) if _ensured_dirs[path] then return end _ensured_dirs[path] = true @@ -370,10 +530,12 @@ end --- Group a list of `SourceFile`-shaped records by their `dir` field. --- Used by the annotation / static-analysis / report passes to partition sources into per-DIRECTORY (per-module) buckets before emitting per-module reports. --- Insertion order is preserved within each bucket (matches source order in `corpus.source_order`). ---- @param sources table[] -- list of source records (each having a `dir` string field) ---- @return table -- map of `dir` -> sources in that dir +--- @param sources SourceFile[] +--- @return table function M.group_sources_by_dir(sources) + --- @type table local by_dir = {} + --- @type integer, SourceFile for _, src in ipairs(sources) do by_dir[src.dir] = by_dir[src.dir] or {} table.insert(by_dir[src.dir], src) @@ -387,15 +549,25 @@ end -- Skip a string or C-style comment starting at position `pos`. -- Returns the position just past the construct, or `pos` unchanged if no string/comment starts there. +--- @param s string +--- @param pos integer +--- @return integer function M.skip_str_or_cmt(s, pos) return lpeg.match(lpeg_str_or_cmt_pat, s, pos) or pos end -- Skip whitespace AND C-style comments starting at position `pos`. -- LPeg-backed; ~5-10x faster than a hand-rolled byte-by-byte walker. +--- @param s string +--- @param pos integer +--- @return integer function M.skip_ws_and_cmt(s, pos) return lpeg.match(lpeg_ws_and_cmt_pat, s, pos) or pos end -- Read a C-style identifier (alpha followed by zero+ alnum) starting at position `pos`. -- Returns the identifier string + the position just past it, or nil + pos if no identifier starts here. +--- @param s string +--- @param pos integer +--- @return string|nil, integer function M.read_ident(s, pos) + --- @type string|nil local result = lpeg.match(lpeg_ident_pat, s, pos) if result then return result, pos + #result end return nil, pos @@ -403,16 +575,26 @@ end -- Read a balanced-delimited group (parens, braces, or brackets) starting at position `pos`. -- Returns the inner content (between the delimiters) + the position just past the closing delimiter, or nil + pos if `s[pos]` isn't `open_char`. +--- @param s string +--- @param open_char string +--- @param close_char string +--- @param pos integer +--- @return string|nil, integer function M.read_balanced(s, open_char, close_char, pos) + --- @type integer local open_byte = open_char:byte() if s:byte(pos) ~= open_byte then return nil, pos end -- scan: pos = pos + 1 -- scan: + --- @type integer local len = #s + --- @type integer local depth = 1 + --- @type integer local a = pos while pos <= len and depth > 0 do + --- @type integer local c = s:byte(pos) if c == open_byte then depth = depth + 1 @@ -424,6 +606,7 @@ function M.read_balanced(s, open_char, close_char, pos) pos = pos + 1 -- scan: (depth=depth) else + --- @type integer local nx = M.skip_str_or_cmt(s, pos) if nx > pos then -- scan: @@ -438,22 +621,39 @@ function M.read_balanced(s, open_char, close_char, pos) end -- Convenience specializations of read_balanced. +--- @param s string +--- @param pos integer +--- @return string|nil, integer M.read_parens = function(s, pos) return M.read_balanced(s, "(", ")", pos) end +--- @param s string +--- @param pos integer +--- @return string|nil, integer M.read_braces = function(s, pos) return M.read_balanced(s, "{", "}", pos) end +--- @param s string +--- @param pos integer +--- @return string|nil, integer M.read_brackets = function(s, pos) return M.read_balanced(s, "[", "]", pos) end -- Scan forward from position `start` until we find a specific single byte `target`, transparently stepping over balanced parens/braces/brackets. -- Returns the position of `target`, or nil if not found. +--- @param s string +--- @param target string +--- @param start integer +--- @return integer|nil function M.scan_to_char(s, target, start) + --- @type integer local target_byte = target:byte() + --- @type integer local pos = start while pos <= #s do + --- @type integer local c = s:byte(pos) if c == target_byte then return pos end -- scan: ... | if c == BYTE_OPEN_PAREN then local _, a = M.read_balanced(s, "(", ")", pos); pos = a -- scan: ... ( ) ... elseif c == BYTE_OPEN_BRACE then local _, a = M.read_balanced(s, "{", "}", pos); pos = a -- scan: ... { } ... elseif c == BYTE_OPEN_BRACK then local _, a = M.read_balanced(s, "[", "]", pos); pos = a -- scan: ... [ ] ... else + --- @type integer local nx = M.skip_str_or_cmt(s, pos) pos = (nx > pos) and nx or (pos + 1) -- scan: ... ... @@ -465,31 +665,48 @@ end -- If `s[pos]` is `#`, skip to the end of the preprocessor directive line (past the newline). -- Returns the position past the newline, or nil if `s[pos]` is not `#`. -- scan: #\n -> past the newline +--- @param s string +--- @param pos integer +--- @return integer|nil function M.skip_preprocessor_line(s, pos) if s:byte(pos) ~= 35 then return nil end -- '#' + --- @type integer local scan = pos + --- @type integer local len = #s while scan <= len and s:byte(scan) ~= BYTE_NEWLINE do scan = scan + 1 end return scan + 1 end +--- @param byte integer +--- @return boolean local function is_horizontal_space(byte) return byte == BYTE_SPACE or byte == BYTE_TAB or byte == BYTE_CR or byte == BYTE_VT or byte == BYTE_FF end +--- @param source string +--- @param first integer +--- @param after_last integer +--- @return boolean local function segment_has_newline(source, first, after_last) + --- @type integer for pos = first, after_last - 1 do if source:byte(pos) == BYTE_NEWLINE then return true end end return false end +--- @param source string +--- @param pos integer +--- @return integer|nil local function skip_directive_space(source, pos) while pos <= #source do + --- @type integer local byte = source:byte(pos) if is_horizontal_space(byte) then pos = pos + 1 elseif byte == BYTE_SLASH and source:byte(pos + 1) == BYTE_STAR then + --- @type integer local after = M.skip_str_or_cmt(source, pos) if after == pos or segment_has_newline(source, pos, after) then return nil end pos = after @@ -504,14 +721,23 @@ end --- Apply C line splicing once for the include scanner. --- Every retained logical byte maps back to its original physical byte offset and one-based physical line so diagnostics preserve source-as-written evidence. +--- @param source string +--- @return string, integer[], integer[] local function splice_c_lines(source) + --- @type string[] local logical_bytes = {} + --- @type integer[] local physical_pos = {} + --- @type integer[] local physical_line = {} + --- @type integer local pos = 1 + --- @type integer local line = 1 while pos <= #source do + --- @type integer local byte = source:byte(pos) + --- @type integer|nil local splice_len = nil if byte == BYTE_BACKSLASH and source:byte(pos + 1) == BYTE_NEWLINE then splice_len = 2 @@ -523,6 +749,7 @@ local function splice_c_lines(source) pos = pos + splice_len line = line + 1 else + --- @type integer local logical_pos = #logical_bytes + 1 logical_bytes[logical_pos] = source:sub(pos, pos) physical_pos [logical_pos] = pos @@ -539,7 +766,7 @@ end --- Interpreted records retain original physical include text and line numbers. --- Angle includes and include-like text inside comments/strings are ignored. --- @param source_text string ---- @return table[] -- ordered `{path, include_path, include_text, line}` records +--- @return QuotedInclude[] function M.parse_direct_quoted_includes(source_text) if type(source_text) ~= "string" then error("parse_direct_quoted_includes requires source text", 2) @@ -547,11 +774,16 @@ function M.parse_direct_quoted_includes(source_text) -- Each arm's effect on (pos, line_leading) is annotated at the branch site. -- Arm order: newline / horiz-space / '//' / '/*' / '"' / '\'' / '#' / default. + --- @type string, integer[], integer[] local logical_text, physical_pos, physical_line = splice_c_lines(source_text) + --- @type QuotedInclude[] local includes = {} + --- @type integer local pos = 1 + --- @type boolean local line_leading = true while pos <= #logical_text do + --- @type integer local byte = logical_text:byte(pos) if byte == BYTE_NEWLINE then -- line break; refresh leading-whitespace state for next line. @@ -562,11 +794,13 @@ function M.parse_direct_quoted_includes(source_text) pos = pos + 1 elseif byte == BYTE_SLASH and logical_text:byte(pos + 1) == BYTE_SLASH then -- '//' line comment: skip_str_or_cmt walks to EOL on its own, so no separate newline scan is needed here. + --- @type integer local after = M.skip_str_or_cmt(logical_text, pos) -- pos := after when the skipper agrees, else single-byte advance. pos = (after > pos) and after or (pos + 1) elseif byte == BYTE_SLASH and logical_text:byte(pos + 1) == BYTE_STAR then -- '/*' block comment. + --- @type integer local after = M.skip_str_or_cmt(logical_text, pos) if after <= pos then -- skipper refused (unterminated /*). Treat this byte as ordinary content: step one, mark non-leading. @@ -574,6 +808,7 @@ function M.parse_direct_quoted_includes(source_text) pos = pos + 1 else -- jump past the closing '*/'. The span may cross lines, so rescan for embedded '\n' to refresh line_leading. + --- @type integer for scan = pos, after - 1 do if logical_text:byte(scan) == BYTE_NEWLINE then line_leading = true end end @@ -582,13 +817,16 @@ function M.parse_direct_quoted_includes(source_text) elseif byte == BYTE_DQUOTE or byte == BYTE_SQUOTE then -- enter + leave the string literal in one skip; literal bodies cannot contain a directive regardless of what they look like. line_leading = false + --- @type integer local after = M.skip_str_or_cmt(logical_text, pos) pos = (after > pos) and after or (pos + 1) elseif byte == 35 and line_leading then -- '#' at line head -- Sequential pre-checks; any one failing falls through to ::not_include:: (single-byte advance). -- Full success pushes the record and jumps to ::directive_done:: without ever entering the not-include path. -- (All locals are pre-declared at the top of this arm because Lua forbids a goto from crossing a local declaration into its scope.) + --- @type integer, integer, integer|nil, string|nil, integer, integer local hash_pos, directive_line, scan, ident, after_ident, after_quote + --- @type string, integer, integer local include_path, physical_first, physical_last hash_pos = pos directive_line = physical_line[hash_pos] or 1 @@ -633,20 +871,31 @@ function M.parse_direct_quoted_includes(source_text) return includes end +--- @param path Path +--- @param wanted string +--- @return boolean local function path_has_segment(path, wanted) + --- @type string for segment in M.normalize_path(path):gmatch("[^/]+") do if segment:lower() == wanted then return true end end return false end +--- @param candidate_key string +--- @param root_key string +--- @return boolean local function canonical_key_is_within(candidate_key, root_key) if candidate_key == root_key then return true end + --- @type string local prefix = root_key .. "/" return candidate_key:sub(1, #prefix) == prefix end +--- @param path Path +--- @return SourceFile local function load_source_record(path) + --- @type Path local normalized = absolute_normalized_path(path) return { path = normalized, @@ -659,8 +908,8 @@ end --- Resolve a unity source corpus without recursive discovery. --- The root is loaded first; only its direct quoted includes are considered, in source order. --- Candidate A is root-directory relative and candidate B is `/code` relative. ---- @param options table -- `{unity_root=Path, project_root=Path}` ---- @return table +--- @param options ResolveOptions +--- @return Corpus function M.resolve_source_corpus(options) if type(options) ~= "table" then error("resolve_source_corpus requires options", 2) end if type(options.unity_root) ~= "string" or options.unity_root == "" then @@ -670,12 +919,19 @@ function M.resolve_source_corpus(options) error("resolve_source_corpus requires options.project_root", 2) end + --- @type Path local project_root = absolute_normalized_path(options.project_root) + --- @type Path local code_root = M.normalize_path(project_root .. "/code") + --- @type string local code_root_key = M.canonical_path_key(code_root) + --- @type SourceFile local root = load_source_record(options.unity_root) + --- @type SourceFile[] local source_order = { root } + --- @type table local sources_by_path = { [M.canonical_path_key(root.path)] = root, } + --- @type SourceResolver local resolver = { resolved = { { @@ -693,13 +949,21 @@ function M.resolve_source_corpus(options) shadowed = {}, } + --- @type integer, QuotedInclude for _, include in ipairs(M.parse_direct_quoted_includes(root.text)) do + --- @type Path local candidate_a = absolute_normalized_path(root.dir .. "/" .. include.path) + --- @type Path local candidate_b = absolute_normalized_path(code_root .. "/" .. include.path) + --- @type string local key_a = M.canonical_path_key(candidate_a) + --- @type string local key_b = M.canonical_path_key(candidate_b) + --- @type boolean local inside_a = canonical_key_is_within(key_a, code_root_key) + --- @type boolean local inside_b = canonical_key_is_within(key_b, code_root_key) + --- @type ResolverEvidence local evidence = { include_path = include.path, include_text = include.include_text, @@ -724,10 +988,15 @@ function M.resolve_source_corpus(options) resolver.skipped[#resolver.skipped + 1] = evidence else -- Boundary checks above deliberately precede every filesystem probe. + --- @type boolean local exists_a = inside_a and lfs.attributes(candidate_a, "mode") == "file" + --- @type boolean local exists_b = inside_b and ((key_b == key_a and exists_a) or lfs.attributes(candidate_b, "mode") == "file") + --- @type Path|nil local selected = nil + --- @type string|nil local selected_key = nil + --- @type string|nil local disposition = nil if exists_a then selected = candidate_a @@ -765,6 +1034,7 @@ function M.resolve_source_corpus(options) evidence.duplicate_of = sources_by_path[selected_key].path resolver.skipped[#resolver.skipped + 1] = evidence else + --- @type SourceFile local source = load_source_record(selected) evidence.disposition = disposition source_order[#source_order + 1] = source @@ -789,21 +1059,32 @@ end -- Split a brace-body into top-level comma-separated tokens. Honors nested parens/braces/brackets and skips strings/comments. -- Splits at top-level NEWLINES and SEMICOLONS too, AND emits a token break after a top-level comment/string. -- Pure-comment / pure-string chunks contribute 0 words. +--- @param body string +--- @return string[] function M.split_top_level_commas(body) + --- @type string[] local tokens = {} + --- @type integer local pos = 1 + --- @type integer local body_len = #body + --- @type integer local token_start = 1 -- True iff `chunk` contains any non-whitespace, non-comment, non-string content (i.e., real token material). -- Walks through ws + comments individually so a chunk like " /* trailing */ shift_lleft(...)" is correctly classified as having real content (the macro call). + --- @param chunk string + --- @return boolean local function has_real_content(chunk) + --- @type integer local scan = 1 + --- @type integer local len = #chunk while scan <= len do if M.is_space_byte(chunk:byte(scan)) then scan = scan + 1 else + --- @type integer local nx = M.skip_str_or_cmt(chunk, scan) if nx > scan then scan = nx -- skipped a comment or string @@ -815,8 +1096,11 @@ function M.split_top_level_commas(body) return false end + --- @param end_pos integer + --- @return nil local function emit(end_pos) if end_pos >= token_start then + --- @type string local chunk = body:sub(token_start, end_pos) if M.trim(chunk) ~= "" then if has_real_content(chunk) then @@ -834,6 +1118,7 @@ function M.split_top_level_commas(body) end while pos <= body_len do + --- @type integer local c = body:byte(pos) if c == BYTE_OPEN_PAREN then local _, a = M.read_parens(body, pos); pos = a -- scan: ... ( ... elseif c == BYTE_OPEN_BRACE then local _, a = M.read_braces(body, pos); pos = a -- scan: ... { ... @@ -854,6 +1139,7 @@ function M.split_top_level_commas(body) pos = pos + 1 token_start = pos else + --- @type integer local nx = M.skip_str_or_cmt(body, pos) if nx > pos then -- scan: ... ... @@ -874,26 +1160,34 @@ end -- Section 4: tokenize_body + build_body_line_index (shared, memoized) -- ════════════════════════════════════════════════════════════════════════════ +--- @type table -- bag: body text -> tokens local _tokenize_body_cache = {} +--- @type table> -- bag: body text -> offset-to-line local _body_line_index_cache = {} --- Tokenize the body inner-text into a flat list of `{tok, rel}` pairs. --- `tok` is the trimmed token string; `rel` is the byte offset within `body`. --- Memoized on the body string — first call pays O(body_len), subsequent calls return cached. --- @param body string ---- @return table[] -- {{tok=string, rel=integer}, ...} +--- @return BodyToken[] function M.tokenize_body(body) if _tokenize_body_cache[body] ~= nil then return _tokenize_body_cache[body] end + --- @type BodyToken[] local out = {} + --- @type integer local len = #body + --- @type integer local rel = 1 while rel <= len do + --- @type integer local ws_end = M.skip_ws_and_cmt(body, rel) if ws_end > rel then rel = ws_end end if rel > len then break end + --- @type integer local scan = rel while scan <= len do + --- @type integer local c = body:byte(scan) -- Terminator bytes (delimit a token at the top level): ',' = 0x2C, '\n' = 0x0A, ';' = 0x3B. -- These also appear as separators between argument lists inside the parens/braces/brackets, so we stop the scan when we hit any of them. @@ -902,10 +1196,12 @@ function M.tokenize_body(body) if c == BYTE_SEMI then break end -- Line-comment '// ... \n' (0x2F 0x2F): skip to (and past) the next newline, or to end-of-body. if c == BYTE_SLASH and body:byte(scan + 1) == BYTE_SLASH then + --- @type integer|nil local nl = M.find_byte(body, BYTE_NEWLINE, scan) scan = nl and (nl + 1) or (len + 1) -- Block-comment '/* ... */' (0x2F 0x2A): skip to (and past) the matching '*/', or to end-of-body. elseif c == BYTE_SLASH and body:byte(scan + 1) == BYTE_STAR then + --- @type integer|nil local close = body:find("*/", scan + 2, true) scan = close and (close + 2) or (len + 1) -- Group opener bytes (consume the balanced group via the matching reader): '(' = 0x28, '{' = 0x7B, '[' = 0x5B. @@ -919,10 +1215,12 @@ function M.tokenize_body(body) scan = scan + 1 end end + --- @type string local tok = M.trim(body:sub(rel, scan - 1)) if tok ~= "" then out[#out + 1] = { tok = tok, rel = rel } end if scan <= len then scan = scan + 1 + --- @type integer local w = M.skip_ws_and_cmt(body, scan) if w > scan then scan = w end end @@ -935,12 +1233,16 @@ end --- Build a line-index: count `\n` chars from offset 1 up to the offset; that count + 1 is the line number (1-based). --- Memoized on the body string. --- @param body string ---- @return table -- index[pos] = line_number +--- @return table -- bag: byte offset -> 1-based line function M.build_body_line_index(body) if _body_line_index_cache[body] ~= nil then return _body_line_index_cache[body] end + --- @type table -- bag: byte offset -> 1-based line local index = {} + --- @type integer local len = #body + --- @type integer local newline_count = 0 + --- @type integer for pos = 1, len do if pos > 1 then index[pos] = newline_count + 1 @@ -960,20 +1262,33 @@ end -- Section 5: load_word_counts -- ════════════════════════════════════════════════════════════════════════════ +--- @param metadata_path Path +--- @return WordCounts function M.load_word_counts(metadata_path) + --- @type WordCounts local counts = {} + --- @type string local content = M.read_file(metadata_path) + --- @type integer local len = #content + --- @type integer local pos = 1 + --- @type string local prefix = "WORD_COUNT(" while pos <= len do + --- @type integer|nil local nl = M.find_byte(content, BYTE_NEWLINE, pos) + --- @type integer local line_end = nl or (len + 1) + --- @type string local line = content:sub(pos, line_end - 1) -- scan: WORD_COUNT(, ) + --- @type string local trimmed = M.trim(line) if trimmed:sub(1, #prefix) == prefix and trimmed:sub(-1) == ")" then + --- @type string local inner = trimmed:sub(#prefix + 1, #trimmed - 1) + --- @type integer|nil local comma = M.find_byte(inner, BYTE_COMMA, 1) if comma then counts[M.trim(inner:sub(1, comma - 1))] = @@ -989,9 +1304,14 @@ end -- Section 6: LineIndex (constant-time line lookup) -- ══════════════════════════════════════════════════ +--- @param source string +--- @return LineIndexFn function M.LineIndex(source) + --- @type integer[] local positions = {} + --- @type integer local n = 0 + --- @type integer for pos = 1, #source do if source:byte(pos) == BYTE_NEWLINE then n = n + 1 @@ -999,9 +1319,13 @@ function M.LineIndex(source) end end -- (internal) Binary-search for the line number containing query_pos. + --- @param query_pos integer + --- @return integer local function line_of(query_pos) + --- @type integer, integer local lo, hi = 1, n while lo <= hi do + --- @type integer local mid = math.floor((lo + hi) / 2) if positions[mid] <= query_pos then lo = mid + 1 else hi = mid - 1 end diff --git a/scripts/elf32.lua b/scripts/elf32.lua index a92144d..e5d75f6 100644 --- a/scripts/elf32.lua +++ b/scripts/elf32.lua @@ -22,6 +22,94 @@ -- spec: System V ABI gABI v1.2 §"ELF Header" (Table 1) + §"Section Header Table" -- spec: System V ABI gABI v1.2 §"Symbol Table" (Elf32_Sym layout) +--- @class Elf32Adapter +--- @field read_u8_at fun(off: integer): integer|nil +--- @field read_u16_at fun(off: integer): integer|nil +--- @field read_u32_at fun(off: integer): integer|nil +--- @field read_size fun(): integer + +--- @class Elf32Header +--- @field e_entry integer +--- @field e_shoff integer +--- @field e_shentsize integer +--- @field e_shnum integer +--- @field e_shstrndx integer +--- @field error string|nil + +--- @class Elf32Section +--- @field sh_name integer +--- @field sh_type integer +--- @field sh_flags integer +--- @field sh_addr integer +--- @field sh_offset integer +--- @field sh_size integer +--- @field sh_link integer +--- @field name string + +--- @class Elf32Sym +--- @field value integer +--- @field size integer +--- @field info integer +--- @field shndx integer + +--- @class Elf32HeaderLayout +--- @field magic_offset integer +--- @field magic string +--- @field class_offset integer +--- @field endian_offset integer +--- @field header_bytes integer +--- @field e_entry_offset integer +--- @field e_shoff_offset integer +--- @field e_shentsize_offset integer +--- @field e_shnum_offset integer +--- @field e_shstrndx_offset integer + +--- @class Elf32SectionLayout +--- @field sh_name_offset integer +--- @field sh_type_offset integer +--- @field sh_flags_offset integer +--- @field sh_addr_offset integer +--- @field sh_offset_offset integer +--- @field sh_size_offset integer +--- @field sh_link_offset integer +--- @field sh_entsize_bytes integer + +--- @class Elf32SymLayout +--- @field st_name integer +--- @field st_value integer +--- @field st_size integer +--- @field st_info integer +--- @field sym_entry_bytes integer + +--- @class Elf32Mod +--- @field ELFCLASS32 integer +--- @field ELFDATA2LSB integer +--- @field EM_MIPS integer +--- @field SHT_SYMTAB integer +--- @field SHT_STRTAB integer +--- @field SHT_NOBITS integer +--- @field SHF_WRITE integer +--- @field SHF_ALLOC integer +--- @field SHF_EXECINSTR integer +--- @field ELF32_HEADER Elf32HeaderLayout +--- @field ELF32_SECTION Elf32SectionLayout +--- @field ELF32_SYM Elf32SymLayout +--- @field dw_dwarf32_terminator integer +--- @field read_u32 fun(adapter: Elf32Adapter, off: integer): integer|nil +--- @field read_u16 fun(adapter: Elf32Adapter, off: integer): integer|nil +--- @field read_u8 fun(adapter: Elf32Adapter, off: integer): integer|nil +--- @field size fun(adapter: Elf32Adapter): integer +--- @field read_u32_le fun(buf: string, off: integer): integer +--- @field read_u16_le fun(buf: string, off: integer): integer +--- @field validate_adapter fun(adapter: any): boolean, string|nil +--- @field get_str fun(strtab: string, off: integer): string|nil +--- @field parse_elf32_headers fun(adapter: Elf32Adapter): Elf32Header|nil, string|nil +--- @field walk_sections fun(adapter: Elf32Adapter, hdr: Elf32Header): Elf32Section[]|nil, string|nil +--- @field read_section_bytes fun(adapter: Elf32Adapter, section: Elf32Section): string|nil +--- @field read_named_section fun(adapter: Elf32Adapter, sections: Elf32Section[], name: string): string|nil, string|nil +--- @field collect_symbols fun(adapter: Elf32Adapter, sections: Elf32Section[]): table|nil, string|nil + +--- @type Elf32Mod local M = {} -- ════════════════════════════════════════════════════════════════════════════ @@ -39,7 +127,7 @@ local M = {} --- **Call form:** explicit-pass. The reader receives `adapter` as the first positional argument and the offset as the second; no `self` is passed. --- Test fixtures declare `function(offset) ... end` and the parsers call them via dot syntax `adapter.read_u8_at(off)`. --- The colon form `adapter:read_u8_at(off)` would prepend the adapter table as `offset` and break the contract. ---- @param adapter table +--- @param adapter Elf32Adapter --- @param off integer -- zero-based wire offset --- @return integer|nil function M.read_u32(adapter, off) @@ -50,7 +138,7 @@ function M.read_u32(adapter, off) end --- Read a 2-byte little-endian unsigned integer from `adapter` at zero-based wire offset `off`. ---- @param adapter table +--- @param adapter Elf32Adapter --- @param off integer -- zero-based wire offset --- @return integer|nil function M.read_u16(adapter, off) @@ -59,7 +147,7 @@ function M.read_u16(adapter, off) end --- Read a 1-byte unsigned integer from `adapter` at zero-based wire offset `off`. ---- @param adapter table +--- @param adapter Elf32Adapter --- @param off integer -- zero-based wire offset --- @return integer|nil function M.read_u8(adapter, off) @@ -67,7 +155,7 @@ function M.read_u8(adapter, off) end --- Total adapter byte length. ---- @param adapter table +--- @param adapter Elf32Adapter --- @return integer function M.size(adapter) return adapter.read_size() @@ -76,7 +164,11 @@ end --- Forwarders kept for backward compat with scripts/elf_dwarf.lua. --- The metaprogram side keeps `read_u32_le` / `read_u16_le`; --- both layers now use the same byte-level helpers under the hood. +--- @param buf string +--- @param off integer +--- @return integer function M.read_u32_le(buf, off) + --- @type integer local byte_off = off + 1 return buf:byte(byte_off) + buf:byte(byte_off + 0x01) * 0x00000100 @@ -89,6 +181,7 @@ end --- @param off integer -- zero-based wire offset --- @return integer function M.read_u16_le(buf, off) + --- @type integer local byte_off = off + 1 return buf:byte(byte_off) + buf:byte(byte_off + 0x01) * 0x00000100 end @@ -116,6 +209,7 @@ M.SHF_EXECINSTR = 0x4 -- spec: gABI v1.2 §"Section Attributes" — executable -- ELF32 header layout (System V ABI gABI v1.2 §"ELF Header" Table 1) -- --------------------------------------------------------------------------- -- All offsets are zero-based wire offsets. The header is 52 bytes total (header_bytes = 0x34 = 52). +--- @type Elf32HeaderLayout M.ELF32_HEADER = { magic_offset = 0x00, -- 4 bytes; expected "\127ELF" magic = "\127ELF", @@ -134,6 +228,7 @@ M.ELF32_HEADER = { -- --------------------------------------------------------------------------- -- Each entry is 40 bytes (sh_entsize_bytes = 0x28 = 40); -- zero-based, field offsets relative to the start of the entry. +--- @type Elf32SectionLayout M.ELF32_SECTION = { sh_name_offset = 0x00, -- 4-byte LE; offset into .shstrtab sh_type_offset = 0x04, -- 4-byte LE; section type (SHT_*) @@ -150,6 +245,7 @@ M.ELF32_SECTION = { -- --------------------------------------------------------------------------- -- Each entry is 16 bytes (sym_entry_bytes = 0x10 = 16); -- zero-based, field offsets relative to the start of the entry. +--- @type Elf32SymLayout M.ELF32_SYM = { st_name = 0x00, -- 4-byte LE; offset into the linked string table st_value = 0x04, -- 4-byte LE; symbol value (address / absolute) @@ -190,6 +286,7 @@ end --- @return string|nil function M.get_str(strtab, off) if off < 0 or off >= #strtab then return nil end + --- @type integer|nil local end_pos = strtab:find("\0", off + 1, true) if not end_pos then return nil end return strtab:sub(off + 1, end_pos - 1) @@ -205,38 +302,50 @@ end --- On failure returns nil + a stable error code: --- bad_magic, unsupported_elf_class, unsupported_elf_data, truncated_header --- The header's machine field is NOT validated here — callers (e.g. the helper's prime path) decide whether to require EM_MIPS before symbol reads. ---- @param adapter table ---- @return table|nil, string|nil +--- @param adapter Elf32Adapter +--- @return Elf32Header|nil, string|nil function M.parse_elf32_headers(adapter) + --- @type boolean, string|nil local ok, err = M.validate_adapter(adapter) if not ok then return nil, err end -- 4-byte magic: 0x7F 'E' 'L' 'F'. -- The byte readers take the adapter explicitly. -- The production `Support.File` adapter is wrapped by the caller to drop its implicit `self` so the parser shape is flat pass-style. + --- @type integer|nil local b1 = M.read_u8(adapter, 0) + --- @type integer|nil local b2 = M.read_u8(adapter, 1) + --- @type integer|nil local b3 = M.read_u8(adapter, 2) + --- @type integer|nil local b4 = M.read_u8(adapter, 3) if not (b1 and b2 and b3 and b4) or not (b1 == 0x7f and b2 == 0x45 and b3 == 0x4c and b4 == 0x46) then return nil, "bad_magic" end + --- @type integer|nil local class = M.read_u8(adapter, M.ELF32_HEADER.class_offset) if class ~= M.ELFCLASS32 then return nil, "unsupported_elf_class" end + --- @type integer|nil local data = M.read_u8(adapter, M.ELF32_HEADER.endian_offset) if data ~= M.ELFDATA2LSB then return nil, "unsupported_elf_data" end + --- @type integer|nil local e_entry = M.read_u32(adapter, M.ELF32_HEADER.e_entry_offset) + --- @type integer|nil local e_shoff = M.read_u32(adapter, M.ELF32_HEADER.e_shoff_offset) + --- @type integer|nil local e_shentsize = M.read_u16(adapter, M.ELF32_HEADER.e_shentsize_offset) + --- @type integer|nil local e_shnum = M.read_u16(adapter, M.ELF32_HEADER.e_shnum_offset) + --- @type integer|nil local e_shstrndx = M.read_u16(adapter, M.ELF32_HEADER.e_shstrndx_offset) if not (e_entry and e_shoff and e_shentsize and e_shnum and e_shstrndx) then return nil, "truncated_header" @@ -254,10 +363,11 @@ end --- Read one section-header entry from `adapter` at `sh_off`. --- Returns a table with the wire fields plus a (yet-unresolved) `name` field. ---- @param adapter table +--- @param adapter Elf32Adapter --- @param sh_off integer ---- @return table|nil, string|nil -- entry, error +--- @return Elf32Section|nil, string|nil local function read_section_entry(adapter, sh_off) + --- @type Elf32Section local entry = { sh_name = M.read_u32(adapter, sh_off + M.ELF32_SECTION.sh_name_offset), sh_type = M.read_u32(adapter, sh_off + M.ELF32_SECTION.sh_type_offset), @@ -279,21 +389,26 @@ end --- (the section at logical index 0 is at array position 1, etc.). --- Each entry has the wire fields plus a resolved `name` derived from `.shstrtab`. --- Returns nil + a stable error code on failure: truncated_section_headers, missing_shstrtab, truncated_strtab ---- @param adapter table ---- @param hdr table -- the table returned by parse_elf32_headers ---- @return table|nil, string|nil +--- @param adapter Elf32Adapter +--- @param hdr Elf32Header +--- @return Elf32Section[]|nil, string|nil function M.walk_sections(adapter, hdr) if not hdr or hdr.error then return nil, hdr and hdr.error or "truncated_section_headers" end + --- @type integer local file_size = M.size(adapter) if hdr.e_shoff + hdr.e_shnum * hdr.e_shentsize > file_size then return nil, "truncated_section_headers" end -- Read every section header first; we need .shstrtab to resolve names. + --- @type Elf32Section[] local sections = {} + --- @type integer for i = 0, hdr.e_shnum - 1 do + --- @type integer local sh_off = hdr.e_shoff + i * hdr.e_shentsize + --- @type Elf32Section|nil, string|nil local entry, err = read_section_entry(adapter, sh_off) if not entry then return nil, err end sections[i + 1] = entry @@ -303,6 +418,7 @@ function M.walk_sections(adapter, hdr) return nil, "missing_shstrtab" end + --- @type Elf32Section|nil local shstrtab = sections[hdr.e_shstrndx + 1] if not shstrtab or shstrtab.sh_type ~= M.SHT_STRTAB then return nil, "missing_shstrtab" @@ -310,9 +426,11 @@ function M.walk_sections(adapter, hdr) if shstrtab.sh_offset + shstrtab.sh_size > file_size then return nil, "truncated_section_headers" end + --- @type string|nil local shstrtab_bytes = M.read_section_bytes(adapter, shstrtab) if not shstrtab_bytes then return nil, "truncated_section_headers" end + --- @type integer, Elf32Section for _, s in ipairs(sections) do s.name = M.get_str(shstrtab_bytes, s.sh_name) or "" end @@ -322,14 +440,18 @@ end --- Read the bytes of one section. Returns a string, or nil if the adapter returns nil for any byte (out-of-bounds). --- The caller is responsible fors sizing the buffer (the section's sh_offset + sh_size must fit in adapter.size). ---- @param adapter table ---- @param section table -- one entry from walk_sections +--- @param adapter Elf32Adapter +--- @param section Elf32Section --- @return string|nil function M.read_section_bytes(adapter, section) + --- @type integer local size = section.sh_size if size == 0 then return "" end + --- @type string[] local out = {} + --- @type integer for i = 0, size - 1 do + --- @type integer|nil local b = M.read_u8(adapter, section.sh_offset + i) if b == nil then return nil end out[#out + 1] = string.char(b) @@ -339,14 +461,16 @@ end --- Convenience: walk sections, then look up the named section, then read its bytes. --- Returns nil + a stable error code if the section is absent or out-of-bounds. ---- @param adapter table ---- @param sections table -- 1-based array from walk_sections +--- @param adapter Elf32Adapter +--- @param sections Elf32Section[] --- @param name string --- @return string|nil, string|nil function M.read_named_section(adapter, sections, name) if not sections then return nil, "missing_section" end + --- @type integer, Elf32Section for _, s in ipairs(sections) do if s.name == name then + --- @type string|nil local bytes = M.read_section_bytes(adapter, s) if not bytes then return nil, "truncated_section_data" end return bytes, nil @@ -359,15 +483,19 @@ end --- Each stored entry is `{ value = st_value, size = st_size, info = st_info, shndx = st_shndx }`. --- Both STB_LOCAL and STB_GLOBAL symbols are included; the live ELF stores `smem` as a local symbol. --- Returns nil + a stable error code on failure: missing_symtab_strtab, truncated_section_headers ---- @param adapter table ---- @param sections table ---- @return table|nil, string|nil +--- @param adapter Elf32Adapter +--- @param sections Elf32Section[] +--- @return table|nil, string|nil function M.collect_symbols(adapter, sections) if not sections then return nil, "missing_sections" end + --- @type table -- bag: symbol name -> Elf32Sym local symbols = {} + --- @type integer local file_size = M.size(adapter) + --- @type integer, Elf32Section for _, s in ipairs(sections) do if s.sh_type == M.SHT_SYMTAB then + --- @type Elf32Section|nil local strtab = sections[s.sh_link + 1] if not strtab or strtab.sh_type ~= M.SHT_STRTAB then return nil, "missing_symtab_strtab" @@ -375,30 +503,43 @@ function M.collect_symbols(adapter, sections) if strtab.sh_offset + strtab.sh_size > file_size then return nil, "truncated_section_headers" end + --- @type string|nil local strtab_bytes = M.read_section_bytes(adapter, strtab) if not strtab_bytes then return nil, "truncated_section_headers" end if s.sh_offset + s.sh_size > file_size then return nil, "truncated_section_headers" end + --- @type string|nil local symtab_bytes = M.read_section_bytes(adapter, s) if not symtab_bytes then return nil, "truncated_section_headers" end + --- @type number local n = #symtab_bytes / M.ELF32_SYM.sym_entry_bytes + --- @type integer for j = 0, n - 1 do + --- @type integer local e = s.sh_offset + j * M.ELF32_SYM.sym_entry_bytes + --- @type integer|nil local st_name = M.read_u32(adapter, e + M.ELF32_SYM.st_name) if st_name then + --- @type integer|nil local st_value = M.read_u32(adapter, e + M.ELF32_SYM.st_value) + --- @type integer|nil local st_size = M.read_u32(adapter, e + M.ELF32_SYM.st_size) + --- @type integer|nil local st_info = M.read_u8(adapter, e + M.ELF32_SYM.st_info) -- st_shndx is at offset 14 (2 bytes) — derived from the layout -- the metaprogram reads too. Inline the read to keep the -- adapter as the only I/O surface. + --- @type integer|nil local b1 = M.read_u8(adapter, e + 14) + --- @type integer|nil local b2 = M.read_u8(adapter, e + 15) if not (b1 and b2) then return nil, "truncated_section_headers" end + --- @type integer local st_shndx = b1 + b2 * 0x100 + --- @type string local name = M.get_str(strtab_bytes, st_name) or "" if name ~= "" then symbols[name] = { diff --git a/scripts/elf_dwarf.lua b/scripts/elf_dwarf.lua index beeb260..e61c794 100644 --- a/scripts/elf_dwarf.lua +++ b/scripts/elf_dwarf.lua @@ -8,14 +8,119 @@ -- Native dependencies -- ════════════════════════════════════════════════════════════════════════════ --- lfs is wired into package.cpath by `duffle_paths.lua` (vendored under `toolchain/lfs/lfs.dll`). +-- lfs is wired into package.cpath by `duffle_paths.lua` (vendored under `toolchain/lfs/lfs.dll`). +-- Elf32Adapter / Elf32Header / Elf32Section / Elf32Sym / Elf32Mod: see elf32.lua. +-- Path: see duffle.lua. NmAddr: see passes/atoms_source_map.lua. + +--- @class LfsMod +--- @field attributes fun(path: string, request: string): string|nil + +--- @class Dwarf4Aranges +--- @field unit_length_offset integer +--- @field version_offset integer +--- @field cu_offset_offset integer +--- @field addr_size_offset integer +--- @field seg_size_offset integer +--- @field entry_size integer +--- @field terminator_size integer +--- @field version_expected integer +--- @field addr_size_expected integer +--- @field seg_size_expected integer + +--- @class Dwarf5Rnglists +--- @field unit_length_offset integer +--- @field version_offset integer +--- @field addr_size_offset integer +--- @field seg_size_offset integer +--- @field offset_count_offset integer +--- @field first_entry_offset integer +--- @field end_of_list integer +--- @field start_length integer +--- @field version_expected integer +--- @field addr_size_expected integer +--- @field seg_size_expected integer +--- @field offset_count_expected integer + +--- @class DwarfLineOps +--- @field DW_LNS_extended integer +--- @field DW_LNS_copy integer +--- @field DW_LNS_advance_pc integer +--- @field DW_LNS_advance_line integer +--- @field DW_LNS_set_file integer +--- @field DW_LNS_negate_stmt integer +--- @field DW_LNE_end_sequence integer +--- @field DW_LNE_set_address integer +--- @field opcode_base integer +--- @field line_base integer +--- @field line_range integer +--- @field end_sequence_payload_size integer +--- @field set_address_payload_size integer + +--- @class Dwarf5DebugLine +--- @field version_offset_post_il integer +--- @field addr_size_offset integer +--- @field seg_size_offset integer +--- @field header_length_offset integer +--- @field program_header_start integer +--- @field form_addr_bytes integer +--- @field form_strp_bytes integer +--- @field form_data16_bytes integer +--- @field form_line_strp integer +--- @field form_string integer +--- @field form_udata integer +--- @field form_data16 integer +--- @field lnct_path integer +--- @field lnct_directory_index integer +--- @field lnct_md5 integer + +--- @class AbbrevAttr +--- @field name integer +--- @field form integer + +--- @class AbbrevDecl +--- @field code integer +--- @field tag integer +--- @field has_children integer +--- @field attrs AbbrevAttr[] + +--- @class ElfDwarf +--- @field DW_TAG table -- bag: tag name -> encoding +--- @field DW_AT table -- bag: attr name -> encoding +--- @field DW_FORM table -- bag: form name -> encoding +--- @field DW_ATE table -- bag: base-type encoding name -> code +--- @field MIPS_BYTES_PER_WORD integer +--- @field dw_dwarf32_terminator integer +--- @field DWARF4_ARANGES Dwarf4Aranges +--- @field DWARF5_RNGLISTS Dwarf5Rnglists +--- @field DWARF_LINE_OPS DwarfLineOps +--- @field DWARF5_DEBUG_LINE Dwarf5DebugLine +--- @field read_u32_le fun(buf: string, off: integer): integer +--- @field read_u16_le fun(buf: string, off: integer): integer +--- @field read_uleb128_at fun(buf: string, pos: integer): integer|nil, integer +--- @field read_sleb128_at fun(buf: string, pos: integer): integer|nil, integer +--- @field find_abbrev_table_end fun(table_bytes: string, table_start: integer): integer|nil +--- @field read_ref_sig8 fun(buf: string, pos: integer): integer, integer, integer +--- @field find_type_unit_by_signature fun(info: string, target_sig_lo: integer, target_sig_hi: integer): integer|nil, integer|nil +--- @field write_u32_le fun(value: integer): string +--- @field write_u16_le fun(value: integer): string +--- @field read_elf_sections fun(elf_path: Path, section_names: string[]): table +--- @field read_nm fun(elf_path: Path): table +--- @field uleb128 fun(n: integer): string +--- @field sleb128 fun(n: integer): string +--- @field uleb128_size fun(n: integer): integer +--- @field sleb128_size fun(n: integer): integer +--- @field read_line_unit_file_table fun(elf_path: string): table|nil, table|nil, table|nil + +--- @type LfsMod local lfs = require("lfs") -- scripts/elf32.lua contains format-constant tables + the byte-level walker. -- The this file re-exports `read_u32_le` / `read_u16_le` (and the DWARF32 terminator). -- read_u32_le is this module's reader; implementation in elf32.lua. +--- @type Elf32Mod local E = require("elf32") +--- @type ElfDwarf local M = {} -- ════════════════════════════════════════════════════════════════════════════ @@ -23,6 +128,8 @@ local M = {} -- ════════════════════════════════════════════ -- (DWARF5 §7.5.5 "Tag Encodings" + Table 7.1; gcc emits these exact values for the DWARF3-extension and DWARF5 line units.) +--- bag: DWARF tag name -> encoding +--- @type table M.DW_TAG = { compile_unit = 0x11, subprogram = 0x2E, @@ -38,6 +145,8 @@ M.DW_TAG = { -- We index the canonical gcc-emitted tags. Anything else falls through. } +--- bag: DWARF attr name -> encoding +--- @type table M.DW_AT = { name = 0x03, low_pc = 0x11, @@ -59,6 +168,8 @@ M.DW_AT = { decl_line = 0x3B, } +--- bag: DWARF form name -> encoding +--- @type table M.DW_FORM = { addr = 0x01, data1 = 0x0B, @@ -75,6 +186,8 @@ M.DW_FORM = { sec_offset = 0x17, } +--- bag: DWARF base-type encoding name -> code +--- @type table M.DW_ATE = { address = 0x01, boolean = 0x02, @@ -87,6 +200,7 @@ M.DW_ATE = { } -- DWARF5 §7.5.6 DW_FORM_implicit_const +--- @type integer local DW_FORM_implicit_const = 0x21 -- ════════════════════════════════════════════════════════════════════════════ @@ -120,6 +234,7 @@ M.dw_dwarf32_terminator = E.dw_dwarf32_terminator -- All offsets are zero-based wire offsets. --- spec: DWARF5 spec §7.4 (Address Range Table) — 32-bit DWARF form +--- @type Dwarf4Aranges M.DWARF4_ARANGES = { unit_length_offset = 0x00, -- 4-byte LE; length of unit body (excludes these 4 bytes) version_offset = 0x04, -- 2-byte LE; expected = 2 @@ -139,6 +254,7 @@ M.DWARF4_ARANGES = { -- All offsets are zero-based wire offsets. --- spec: DWARF5 spec §2.17 + §7.21 (Range List Table) — 32-bit DWARF form +--- @type Dwarf5Rnglists M.DWARF5_RNGLISTS = { unit_length_offset = 0x00, -- 4-byte LE version_offset = 0x04, -- 2-byte LE; expected = 5 @@ -161,6 +277,7 @@ M.DWARF5_RNGLISTS = { -- Compare to the *_offset fields above which are hex. --- spec: DWARF5 spec §6.2.5 (Line Number Program Opcodes) +--- @type DwarfLineOps M.DWARF_LINE_OPS = { -- Standard opcodes (§6.2.5.2) DW_LNS_extended = 0, -- spec: §6.2.5.2 — extended opcode marker byte @@ -197,6 +314,7 @@ M.DWARF_LINE_OPS = { -- These are documented inline at each parse site in read_line_unit_file_table below. --- spec: DWARF5 spec §6.2.4 (Line Number Program Header — version >= 5) +--- @type Dwarf5DebugLine M.DWARF5_DEBUG_LINE = { -- Header fields (zero-based, AFTER unit_length has been read). version_offset_post_il = 0x00, -- 2-byte LE; expected = 5 @@ -255,10 +373,17 @@ end -- Offsets are 0-based; returns (value, next_pos). -- Promoted from `local function` to M.* exports so passes/dwarf_injection.lua can import them as file-scope locals per the 2nd-caller lift precedent -- (the uleb128 + sleb128 encoders were promoted the same way). +--- @param buf string +--- @param pos integer +--- @return integer|nil +--- @return integer function M.read_uleb128_at(buf, pos) + --- @type integer, integer local value, shift = 0, 0 + --- @type integer local len = #buf while pos < len do + --- @type integer local b = buf:byte(pos + 1) value = value + (b % 0x80) * (2 ^ shift) shift = shift + 7 @@ -268,10 +393,17 @@ function M.read_uleb128_at(buf, pos) return nil, pos end +--- @param buf string +--- @param pos integer +--- @return integer|nil +--- @return integer function M.read_sleb128_at(buf, pos) + --- @type integer, integer local value, shift = 0, 0 + --- @type integer local len = #buf while pos < len do + --- @type integer local b = buf:byte(pos + 1) value = value + (b % 0x80) * (2 ^ shift) shift = shift + 7 @@ -287,27 +419,36 @@ end -- Find the 0-based offset of the table-terminator byte (a single 0) for the abbrev table starting at `table_start`. -- Returns nil on truncated input. Walks declaration headers -- (code, tag, has_children, attr/form pairs, DW_FORM_implicit_const constant) until it finds a 0 byte that follows a complete declaration. +--- @param table_bytes string +--- @param table_start integer +--- @return integer|nil function M.find_abbrev_table_end(table_bytes, table_start) + --- @type integer, integer local pos, len = table_start, #table_bytes if pos >= len or table_bytes:byte(pos + 1) == 0 then return pos end while pos < len do + --- @type integer|nil, integer local _code, code_end = M.read_uleb128_at(table_bytes, pos) if not _code then return nil end pos = code_end + --- @type integer|nil, integer local _tag, tag_end = M.read_uleb128_at(table_bytes, pos) if not _tag then return nil end pos = tag_end if pos >= len then return nil end pos = pos + 1 -- has_children byte while pos < len do + --- @type integer|nil, integer local attr, attr_end = M.read_uleb128_at(table_bytes, pos) if not attr then return nil end pos = attr_end + --- @type integer|nil, integer local form, form_end = M.read_uleb128_at(table_bytes, pos) if not form then return nil end pos = form_end if attr == 0 and form == 0 then break end if form == DW_FORM_implicit_const then + --- @type integer|nil, integer local _c, ce = M.read_sleb128_at(table_bytes, pos) if not _c then return nil end pos = ce @@ -321,8 +462,13 @@ end -- Read the null-terminated C string at 0-based offset `off` in `buf`. -- Stops at the first 0 byte or end of buffer. +--- @param buf string +--- @param off integer +--- @return string local function read_c_string_at(buf, off) + --- @type integer local len = #buf + --- @type integer local start = off while off < len and buf:byte(off + 1) ~= 0 do off = off + 1 end return buf:sub(start + 1, off) @@ -331,31 +477,45 @@ end -- Walk the .debug_abbrev table starting at 0-based offset `table_start` and return a list of declarations: -- {code, tag, has_children, attrs={ {name, form}, ... }}. -- Stops at the table terminator. +--- @param table_bytes string +--- @param table_start integer +--- @return AbbrevDecl[]|nil +--- @return string|nil local function parse_abbrev_table(table_bytes, table_start) + --- @type integer|nil local table_end = M.find_abbrev_table_end(table_bytes, table_start) if not table_end then return nil, "no terminator" end + --- @type AbbrevDecl[] local decls = {} + --- @type integer local pos = table_start while pos < table_end do + --- @type integer|nil, integer local code, code_end = M.read_uleb128_at(table_bytes, pos) if not code then return nil, "truncated code" end pos = code_end + --- @type integer|nil, integer local tag, tag_end = M.read_uleb128_at(table_bytes, pos) if not tag then return nil, "truncated tag" end pos = tag_end + --- @type integer local has_children = table_bytes:byte(pos + 1) pos = pos + 1 + --- @type AbbrevAttr[] local attrs = {} while true do + --- @type integer|nil, integer local attr, attr_end = M.read_uleb128_at(table_bytes, pos) if not attr then return nil, "truncated attr" end pos = attr_end + --- @type integer|nil, integer local form, form_end = M.read_uleb128_at(table_bytes, pos) if not form then return nil, "truncated form" end pos = form_end if attr == 0 and form == 0 then break end attrs[#attrs + 1] = { name = attr, form = form } if form == DW_FORM_implicit_const then + --- @type integer|nil, integer local _c, ce = M.read_sleb128_at(table_bytes, pos) if not _c then return nil, "truncated const" end pos = ce @@ -371,65 +531,142 @@ end -- For DW_FORM_strp we return the inline string resolved from `str_buf`. -- For DW_FORM_ref4 we return the absolute CU-relative offset. -- The caller decides whether to interpret that as a section offset. +--- @type table local FORM_READERS = { + --- @param buf string + --- @param _ string + --- @param pos integer + --- @return integer + --- @return integer [M.DW_FORM.addr] = function(buf, _, pos) return M.read_u32_le(buf, pos), pos + 4 end, + --- @param buf string + --- @param _ string + --- @param pos integer + --- @return string + --- @return integer [M.DW_FORM.string] = function(buf, _, pos) + --- @type string local s = read_c_string_at(buf, pos) return s, pos + #s + 1 end, + --- @param buf string + --- @param str_buf string + --- @param pos integer + --- @return string + --- @return integer [M.DW_FORM.strp] = function(buf, str_buf, pos) -- DW_FORM_strp: 4-byte offset into .debug_str. + --- @type integer local strp_off = M.read_u32_le(buf, pos) return read_c_string_at(str_buf, strp_off), pos + 4 end, + --- @param buf string + --- @param _ string + --- @param pos integer + --- @return integer|nil + --- @return integer [M.DW_FORM.udata] = function(buf, _, pos) return M.read_uleb128_at(buf, pos) end, + --- @param buf string + --- @param _ string + --- @param pos integer + --- @return integer + --- @return integer [M.DW_FORM.data1] = function(buf, _, pos) return buf:byte(pos + 1), pos + 1 end, + --- @param buf string + --- @param _ string + --- @param pos integer + --- @return integer + --- @return integer [M.DW_FORM.data2] = function(buf, _, pos) return M.read_u16_le(buf, pos), pos + 2 end, + --- @param buf string + --- @param _ string + --- @param pos integer + --- @return integer + --- @return integer [M.DW_FORM.data4] = function(buf, _, pos) return M.read_u32_le(buf, pos), pos + 4 end, + --- @param buf string + --- @param _ string + --- @param pos integer + --- @return integer + --- @return integer [M.DW_FORM.ref4] = function(buf, _, pos) return M.read_u32_le(buf, pos), pos + 4 end, + --- @param buf string + --- @param _ string + --- @param pos integer + --- @return integer + --- @return integer [M.DW_FORM.sec_offset] = function(buf, _, pos) -- DW_FORM_sec_offset: 4-byte offset (size depends on DWARF version; -- on DWARF5 32-bit it's always 4 bytes). return M.read_u32_le(buf, pos), pos + 4 end, + --- @param _ string + --- @param _ string + --- @param pos integer + --- @return integer + --- @return integer [M.DW_FORM.flag_present] = function(_, _, pos) return 1, pos end, + --- @param buf string + --- @param _ string + --- @param pos integer + --- @return nil + --- @return integer [M.DW_FORM.exprloc] = function(buf, _, pos) -- DW_FORM_exprloc: ULEB byte count + that many bytes of DW_OP_*. + --- @type integer|nil, integer local len, ne = M.read_uleb128_at(buf, pos) if not len then return nil, pos end return nil, ne + len end, + --- @param _ string + --- @param _ string + --- @param pos integer + --- @return nil + --- @return integer [DW_FORM_implicit_const] = function(_, _, pos) -- The constant is declared in the abbrev; no value bytes in the DIE. return nil, pos end, + --- @param buf string + --- @param _ string + --- @param pos integer + --- @return integer + --- @return integer [M.DW_FORM.ref_sig8] = function(buf, _, pos) -- DW_FORM_ref_sig8 (DWARF5 §7.4.2): An 8-byte value identifying a type by signature. -- The low 4 bytes (LE) are the type signature (content hash); -- The high 4 bytes (LE) are a CU-relative offset into the matching type unit. - -- Consumers use the low 4 to look up the type unit (see M.find_type_unit_by_signature) + -- Consumers use the low 4 to look up the type unit (see M.find_type_unit_by_signature) -- then the high 4 to resolve the specific type within it. -- Return the low 4 as the primary value to preserve the (value, next_pos) shape; -- the high 4 is exposed via M.read_ref_sig8 (which returns both halves). + --- @type integer, integer, integer local _, _, next_pos = M.read_ref_sig8(buf, pos) return M.read_u32_le(buf, pos), next_pos end, } +--- @param buf string +--- @param str_buf string +--- @param pos integer +--- @param form integer +--- @return string|integer|nil +--- @return integer local function read_form_value(buf, str_buf, pos, form) + --- @type (fun(buf: string, str_buf: string, pos: integer): (string|integer|nil, integer))|nil local r = FORM_READERS[form] if not r then return nil, pos @@ -464,15 +701,20 @@ function M.read_ref_sig8(buf, pos) return M.read_u32_le(buf, pos), M.read_u32_le --- @param target_sig_hi integer -- high 4 bytes (LE) of the desired signature --- @return integer|nil, integer|nil -- unit offset, type_offset within the unit function M.find_type_unit_by_signature(info, target_sig_lo, target_sig_hi) + --- @type integer local pos = 0 + --- @type integer local section_len = #info while pos + 4 < section_len do + --- @type integer local unit_length = M.read_u32_le(info, pos) if unit_length == 0xFFFFFFFF then return nil, nil -- DWARF64 not supported end -- unit_length is the body size, NOT including the 4-byte unit_length field itself. + --- @type integer local body_start = pos + 4 + --- @type integer local body_end = body_start + unit_length if body_end > section_len then return nil, nil -- malformed @@ -495,10 +737,13 @@ function M.find_type_unit_by_signature(info, target_sig_lo, target_sig_hi) -- byte 4-7: debug_abbrev_offset (4) -- byte 8-15: type_signature (8) -- byte 16-19: type_offset (4) + --- @type integer local unit_type = info:byte(body_start + 2 + 1) -- 0-based +2 = unit_type in 1-indexed if unit_type == 0x02 then -- DW_UT_type + --- @type integer, integer, integer local sig_lo, sig_hi, _ = M.read_ref_sig8(info, body_start + 8) -- 0-based +8 = type_signature in 1-indexed if sig_lo == target_sig_lo and sig_hi == target_sig_hi then + --- @type integer local type_offset = M.read_u32_le(info, body_start + 16) -- 0-based +16 = type_offset in 1-indexed return pos, type_offset end @@ -551,11 +796,15 @@ end function M.read_elf_sections(elf_path, section_names) -- Initialize result with all requested names set to "" so callers can do `sections[X] -- or ""` for missing sections without nil-checks. + --- @type table local result = {} + --- @type integer, string for _, name in ipairs(section_names) do result[name] = "" end -- O(1) lookup set. + --- @type table -- bag: requested section name -> true local wanted = {} + --- @type integer, string for _, name in ipairs(section_names) do wanted[name] = true end -- Existence check (lfs.attributes avoids an io.open-vs-fail race). @@ -564,45 +813,63 @@ function M.read_elf_sections(elf_path, section_names) return result end + --- @type file*|nil local f = io.open(elf_path, "rb") if not f then io.stderr:write(string.format("[elf_dwarf.read_elf_sections] io.open failed: %s\n", elf_path)) return result end + --- @type integer local file_size do f:seek("end", 0) file_size = f:seek("cur", 0) end + --- @type Elf32Adapter local adapter = { + --- @param offset integer + --- @return integer|nil read_u8_at = function(offset) f:seek("set", offset) + --- @type string|nil local b = f:read(1) if not b then return nil end return b:byte() end, + --- @param offset integer + --- @return integer|nil read_u16_at = function(offset) f:seek("set", offset) + --- @type string|nil local b1 = f:read(1) + --- @type string|nil local b2 = f:read(1) if not b1 or not b2 then return nil end return b1:byte() + b2:byte() * 0x100 end, + --- @param offset integer + --- @return integer|nil read_u32_at = function(offset) f:seek("set", offset) + --- @type string|nil local b1 = f:read(1) + --- @type string|nil local b2 = f:read(1) + --- @type string|nil local b3 = f:read(1) + --- @type string|nil local b4 = f:read(1) if not b1 or not b2 or not b3 or not b4 then return nil end return b1:byte() + b2:byte() * 0x100 + b3:byte() * 0x10000 + b4:byte() * 0x1000000 end, + --- @return integer read_size = function() return file_size end, } -- Delegate the header parse + section walk to E.*. + --- @type Elf32Header|nil, string|nil local hdr, hdr_err = E.parse_elf32_headers(adapter) if not hdr then io.stderr:write(string.format("[elf_dwarf.read_elf_sections] header parse failed: %s\n", tostring(hdr_err))) @@ -610,6 +877,7 @@ function M.read_elf_sections(elf_path, section_names) return result end + --- @type Elf32Section[]|nil, string|nil local sections, walk_err = E.walk_sections(adapter, hdr) if not sections then io.stderr:write(string.format("[elf_dwarf.read_elf_sections] section walk failed: %s\n", tostring(walk_err))) @@ -618,8 +886,10 @@ function M.read_elf_sections(elf_path, section_names) end -- Resolve the requested sections. + --- @type integer, Elf32Section for _, s in ipairs(sections) do if wanted[s.name] then + --- @type string|nil local bytes = E.read_section_bytes(adapter, s) if bytes then result[s.name] = bytes end end @@ -640,50 +910,69 @@ end --- - `st_size > 0` filter excludes undefined/imported symbols. --- --- @param elf_path Path ---- @return table +--- @return table function M.read_nm(elf_path) + --- @type table local addrs = {} -- Existence check first; an empty or missing ELF returns an empty map. if lfs.attributes(elf_path, "mode") ~= "file" then return addrs end + --- @type file*|nil local f = io.open(elf_path, "rb") if not f then return addrs end -- Build the file adapter for E.*. + --- @type integer local file_size do f:seek("end", 0) file_size = f:seek("cur", 0) end + --- @type Elf32Adapter local adapter = { + --- @param offset integer + --- @return integer|nil read_u8_at = function(offset) f:seek("set", offset) + --- @type string|nil local b = f:read(1) if not b then return nil end return b:byte() end, + --- @param offset integer + --- @return integer|nil read_u16_at = function(offset) f:seek("set", offset) + --- @type string|nil local b1 = f:read(1) + --- @type string|nil local b2 = f:read(1) if not b1 or not b2 then return nil end return b1:byte() + b2:byte() * 0x100 end, + --- @param offset integer + --- @return integer|nil read_u32_at = function(offset) f:seek("set", offset) + --- @type string|nil local b1 = f:read(1) + --- @type string|nil local b2 = f:read(1) + --- @type string|nil local b3 = f:read(1) + --- @type string|nil local b4 = f:read(1) if not b1 or not b2 or not b3 or not b4 then return nil end return b1:byte() + b2:byte() * 0x100 + b3:byte() * 0x10000 + b4:byte() * 0x1000000 end, + --- @return integer read_size = function() return file_size end, } -- Delegate the header + section walk to E.*. + --- @type Elf32Header|nil, string|nil local hdr, hdr_err = E.parse_elf32_headers(adapter) if not hdr then io.stderr:write(string.format("[elf_dwarf.read_nm] header parse failed: %s\n", tostring(hdr_err))) @@ -691,6 +980,7 @@ function M.read_nm(elf_path) return addrs end + --- @type Elf32Section[]|nil, string|nil local sections, walk_err = E.walk_sections(adapter, hdr) if not sections then io.stderr:write(string.format("[elf_dwarf.read_nm] section walk failed: %s\n", tostring(walk_err))) @@ -700,6 +990,7 @@ function M.read_nm(elf_path) -- E.collect_symbols returns every defined symbol (no binding filter). -- The metaprogram then applies its STB_LOCAL / STB_GLOBAL + size>0 filter, matching `nm`'s default (external symbols only). + --- @type table|nil, string|nil local symbols, sym_err = E.collect_symbols(adapter, sections) if not symbols then io.stderr:write(string.format("[elf_dwarf.read_nm] symbol collection failed: %s\n", tostring(sym_err))) @@ -709,9 +1000,11 @@ function M.read_nm(elf_path) f:close() + --- @type string, Elf32Sym for name, entry in pairs(symbols) do -- High nibble of st_info = binding (STB_LOCAL=0, STB_GLOBAL=1, STB_WEAK=2). -- math.floor(/16) is portable across LuaJIT 2.0/2.1 and plain Lua 5.x. + --- @type integer local binding = math.floor(entry.info / 16) if (binding == 0 or binding == 1) and entry.size > 0 then addrs[name] = { entry.value, entry.size } @@ -744,13 +1037,16 @@ end -- Spec: DWARF5 §7.6 "Variable-Length Data" / Appendix C. -- Top bit of each LEB128 byte. Set if more bytes follow in the encoding. +--- @type integer local LEB_CONT_BIT = 0x80 -- Low 7 bits of each LEB128 byte. The actual data payload. +--- @type integer local LEB_DATA_MASK = 0x7F -- Bit 6 of the 7-bit data (i.e. 0x40). For SLEB128: the sign-bit position used by the decoder for sign extension. -- Encoders MUST stop when the next byte would be redundant AND the sign bit in the last byte matches the value's sign. +--- @type integer local SLEB_SIGN_BIT = 0x40 --- ULEB128 (Unsigned Little-Endian Base 128) encoder. Returns the byte string for the non-negative integer `n`. @@ -768,8 +1064,10 @@ function M.uleb128(n) error("uleb128 requires non-negative number") end assert(n >= 0, "uleb128 requires non-negative input") + --- @type string[] local bytes = {} repeat + --- @type integer local b = n % (LEB_DATA_MASK + 1) -- extract low 7 bits n = (n - b) / (LEB_DATA_MASK + 1) -- shift right by 7 bits if n > 0 then b = b + LEB_CONT_BIT end -- set continuation bit if more bytes follow @@ -789,9 +1087,12 @@ end --- @param n integer -- any integer (negative allowed) --- @return string function M.sleb128(n) + --- @type string[] local bytes = {} + --- @type boolean local more = true while more do + --- @type integer local b = n % (LEB_DATA_MASK + 1) -- extract low 7 bits n = (n - b) / (LEB_DATA_MASK + 1) -- arithmetic shift right by 7 -- Termination: remaining value bits fit in the sign bit of the last byte. @@ -811,6 +1112,7 @@ end function M.uleb128_size(n) assert(n >= 0, "uleb128_size requires non-negative input") if n == 0 then return 1 end + --- @type integer local bytes = 1 while n >= 0x80 do n = (n - (n % (LEB_DATA_MASK + 1))) / (LEB_DATA_MASK + 1) -- arithmetic shift right by 7 @@ -827,10 +1129,14 @@ end --- @param n integer -- any integer (negative allowed) --- @return integer function M.sleb128_size(n) + --- @type boolean local more = true + --- @type integer local bytes = 0 + --- @type integer local v = n while more do + --- @type integer local b = v % (LEB_DATA_MASK + 1) -- extract low 7 bits v = (v - b) / (LEB_DATA_MASK + 1) -- arithmetic shift right by 7 if v == 0 and b < SLEB_SIGN_BIT then more = false end -- positive terminator @@ -869,21 +1175,26 @@ end --- downstream `resolve_provenance_file_index(path)` consumers consult the map directly. --- --- @param elf_path string -- absolute path to the post-link ELF (typically the gcc-emitted `.elf` BEFORE dwarf_injector's splice; both shapes work since the splice preserves `.debug_line`) ---- @return table|nil, table|nil, table|nil ---- basename_to_index: { [basename] = 1-based-per-unit-file-index, ... } ---- basenames: { [1-based-per-unit-file-index] = basename, ... } ---- paths: { [1-based-per-unit-file-index] = full path (mixed slashes), ... } +--- @return table|nil +--- @return table|nil +--- @return table|nil function M.read_line_unit_file_table(elf_path) + --- @type table local sections = M.read_elf_sections(elf_path, { ".debug_line", ".debug_line_str" }) + --- @type string local line = sections[".debug_line"] + --- @type string local lstr = sections[".debug_line_str"] or "" if not line or line == "" then io.stderr:write("[elf_dwarf.read_line_unit_file_table] no .debug_line section in: " .. tostring(elf_path) .. "\n") return nil end + --- @type table -- bag: 1-based file index -> basename local basenames = {} + --- @type table -- bag: basename -> 1-based file index local basename_to_index = {} + --- @type table -- bag: 1-based file index -> full path local paths = {} --- Read one form-code's bytes from `buf` at position `p` according to `form`. @@ -891,15 +1202,25 @@ function M.read_line_unit_file_table(elf_path) --- * the resolved string (DW_FORM_line_strp / DW_FORM_string) --- * the ULEB128 number (DW_FORM_udata) --- * nil + skip-bytes (DW_FORM_data16; we don't surface the MD5) + --- @param buf string + --- @param lstr_buf string + --- @param p integer + --- @param form integer + --- @return string|integer|nil + --- @return integer local function read_form(buf, lstr_buf, p, form) if form == M.DWARF5_DEBUG_LINE.form_line_strp then + --- @type integer local strp = M.read_u32_le(buf, p) + --- @type integer local end_pos = lstr_buf:find("\0", strp + 1, true) or (#lstr_buf + 1) return lstr_buf:sub(strp + 1, end_pos - 1), p + M.DWARF5_DEBUG_LINE.form_strp_bytes elseif form == M.DWARF5_DEBUG_LINE.form_string then + --- @type integer local nul = buf:find("\0", p + 1, true) or (#buf + 1) return buf:sub(p + 1, nul - 1), nul elseif form == M.DWARF5_DEBUG_LINE.form_udata then + --- @type integer|nil, integer local v, after = M.read_uleb128_at(buf, p) return v, after elseif form == M.DWARF5_DEBUG_LINE.form_data16 then @@ -916,34 +1237,51 @@ function M.read_line_unit_file_table(elf_path) --- Parse one DWARF-version-3-style unit (DWARF3/4 line program; gcc default in the PS1 toolchain still emits DWARF3 for line programs in `-g` mode). --- Layout: null-terminated directory list, then path(null) + dir_idx(ULEB) + time(ULEB) + size(ULEB) file entries terminated by an empty null. --- `content_start` = zero-based wire offset of the first byte of program-header content (after version + header_length fields). - --- @return unit_basenames { [idx_in_unit_1_based] = basename } - --- @return unit_paths { [idx_in_unit_1_based] = full path } + --- @param buf string + --- @param content_start integer + --- @param body_end integer + --- @return table + --- @return table local function parse_dwarf3_unit(buf, content_start, body_end) + --- @type integer local up = content_start -- 5 fixed bytes: min_insn, default_is, line_base (signed), line_range, opcode_base up = up + 5 + --- @type integer local opcode_base = buf:byte(content_start + 5) up = up + (opcode_base - 1) -- std_opcode_lengths + --- @type string[] local dirs = {} while up < body_end do + --- @type integer local nul = buf:find("\0", up + 1, true) or (body_end + 1) if nul > body_end then break end + --- @type integer local len = nul - up - 1 if len == 0 then up = nul break end dirs[#dirs + 1] = buf:sub(up + 1, nul - 1) up = nul end + --- @type table -- bag: 1-based unit file index -> basename local unit_basenames = {} + --- @type table -- bag: 1-based unit file index -> full path local unit_paths = {} while up < body_end do + --- @type integer local nul = buf:find("\0", up + 1, true) or (body_end + 1) if nul > body_end or nul == up + 1 then up = nul break end + --- @type string local path = buf:sub(up + 1, nul - 1) up = nul + --- @type integer|nil, integer local didx, up_next = M.read_uleb128_at(buf, up); up = up_next + --- @type integer|nil, integer local _time, up_next2 = M.read_uleb128_at(buf, up); up = up_next2 + --- @type integer|nil, integer local _size, up_next3 = M.read_uleb128_at(buf, up); up = up_next3 + --- @type integer local idx = #unit_basenames + 1 + --- @type string local bs = path:match("[^/\\]+$") or path unit_paths[idx] = path unit_basenames[idx] = bs @@ -957,25 +1295,42 @@ function M.read_line_unit_file_table(elf_path) --- Parse one DWARF-version-5-style unit (DWARF5 line program; used by modern gcc with `-gdwarf-5`). --- `content_start` is the first byte of program-header content (after the 8 fixed bytes version+addr_size+seg_size+header_length). - --- @return same shape as parse_dwarf3_unit + --- @param buf string + --- @param lstr_buf string + --- @param content_start integer + --- @param body_end integer + --- @return table + --- @return table local function parse_dwarf5_unit(buf, lstr_buf, content_start, body_end) + --- @type integer local up = content_start -- 6 fixed bytes: min_insn, max_ops_per_insn, default_is, line_base, line_range, opcode_base up = up + 6 + --- @type integer local opcode_base = buf:byte(content_start + 6) up = up + (opcode_base - 1) -- std_opcode_lengths -- directories + --- @type integer|nil, integer local dir_format_count, after = M.read_uleb128_at(buf, up); up = after + --- @type integer[] local dir_formats = {} + --- @type integer for i = 1, dir_format_count do + --- @type integer|nil, integer local f, a2 = M.read_uleb128_at(buf, up); up = a2 dir_formats[i] = f end + --- @type integer|nil, integer local dir_count, a3 = M.read_uleb128_at(buf, up); up = a3 + --- @type string[] local dirs = {} + --- @type integer for i = 1, dir_count do + --- @type string local combined = "" + --- @type integer for j = 1, dir_format_count do + --- @type string|integer|nil, integer local v, a4 = read_form(buf, lstr_buf, up, dir_formats[j]) up = a4 if j == 1 and type(v) == "string" then combined = v end @@ -983,25 +1338,39 @@ function M.read_line_unit_file_table(elf_path) dirs[i] = combined end -- file names + --- @type integer|nil, integer local file_format_count, after2 = M.read_uleb128_at(buf, up); up = after2 + --- @type integer[] local file_formats = {} + --- @type integer for i = 1, file_format_count do + --- @type integer|nil, integer local f, a2 = M.read_uleb128_at(buf, up); up = a2 file_formats[i] = f end + --- @type integer|nil, integer local file_count, a3 = M.read_uleb128_at(buf, up); up = a3 + --- @type table -- bag: 1-based unit file index -> basename local unit_basenames = {} + --- @type table -- bag: 1-based unit file index -> full path local unit_paths = {} + --- @type integer for i = 1, file_count do + --- @type string local combined = "" + --- @type integer local didx = 0 + --- @type integer for j = 1, file_format_count do + --- @type string|integer|nil, integer local v, a4 = read_form(buf, lstr_buf, up, file_formats[j]) up = a4 if j == 1 and type(v) == "string" then combined = v end if j == 2 and type(v) == "number" then didx = v end end + --- @type integer local idx = #unit_basenames + 1 + --- @type string local bs = combined:match("[^/\\]+$") or combined unit_paths[idx] = combined unit_basenames[idx] = bs @@ -1013,26 +1382,36 @@ function M.read_line_unit_file_table(elf_path) end --- Walk every line-program unit in the section. + --- @type integer local p = 0 + --- @type integer local section_end = #line while p + 4 <= section_end do + --- @type integer local unit_length = M.read_u32_le(line, p) if unit_length == 0xFFFFFFFF then io.stderr:write("[elf_dwarf.read_line_unit_file_table] 64-bit DWARF (initial-length 0xFFFFFFFF); not supported\n") return nil end + --- @type integer local body_start = p + 4 + --- @type integer local body_end = p + 4 + unit_length if body_end > section_end then break end + --- @type integer local version = M.read_u16_le(line, body_start) + --- @type table|nil, table|nil local unit_basenames, unit_paths if version >= 5 then -- DWARF5 header: version(2) + addr_size(1) + seg_size(1) + header_length(4) + content + --- @type integer local header_length_offset = body_start + 6 -- past version(2) + addr_size(1) + seg_size(1) - wait that's wrong; past hdr len is at +6 + --- @type integer local content_start = body_start + 8 -- past version(2) + addr_size(1) + seg_size(1) + header_length(4) unit_basenames, unit_paths = parse_dwarf5_unit(line, lstr, content_start, body_end) elseif version >= 2 then -- DWARF2/3/4 header: version(2) + header_length(4) + content + --- @type integer local content_start = body_start + 6 -- past version(2) + header_length(4) unit_basenames, unit_paths = parse_dwarf3_unit(line, content_start, body_end) else @@ -1047,6 +1426,7 @@ function M.read_line_unit_file_table(elf_path) -- For DWARF5 (crt0.s + C unit), each carries its own per-unit file-table map; -- the atom-side DW_LNS_set_file(N) refers to the C unit's indices, NOT crt0.s's. -- Since the C unit is the one with full include_directories + 12 entries, we can use it directly. + --- @type integer, string for idx, bs in pairs(unit_basenames) do basenames[idx] = bs paths[idx] = unit_paths[idx] diff --git a/scripts/passes/annotation.lua b/scripts/passes/annotation.lua index 8f2bba9..8cb0a50 100644 --- a/scripts/passes/annotation.lua +++ b/scripts/passes/annotation.lua @@ -10,7 +10,9 @@ -- Bootstrap follows the entry scripts; `scripts/duffle_paths.lua` sets package.path and package.cpath. See `ps1_meta.lua` for the rationale. -- `debug.getinfo(1, "S").source` locates this file for standalone and orchestrated runs, then `duffle_paths.lua` returns the loaded `duffle` module. +--- @type string local _bootstrap_dir = debug.getinfo(1, "S").source:match("^@?(.*[/\\])") or "./" +--- @type DuffleExport local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua") -- The annotation pass reads the source-derived registries from scan_source: @@ -21,28 +23,8 @@ local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua") -- Type declarations -- ════════════════════════════════════════════════════════════════════════════ ---- @class SourceFile ---- @field path string -- Absolute path to the source file ---- @field text string -- Full source text ---- @field dir string -- Directory containing the source ---- @field basename string -- Filename without extension ---- @field scan table -- Pre-scanned SourceScan payload (from duffle.scan_source) - ---- @class PassCtx ---- @field sources SourceFile[] ---- @field metadata_path string ---- @field shared table ---- @field shared.word_counts table ---- @field out_root string ---- @field project_root string ---- @field upstream table ---- @field flags table ---- @field verbose boolean - ---- @class PassResult ---- @field outputs table[] ---- @field errors table[] ---- @field warnings table[] +-- SourceFile, PassCtx, PassResult, PassShared, Corpus: see ps1_meta.lua +-- SourceScan, AtomEntry, BindsEntry, RegTypeDefault, AtomViewEntry: see scan_source.lua --- @class AtomAnnotation --- @field atom_name string -- Atom name (scan.atom_infos row) @@ -52,43 +34,51 @@ local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua") --- @field writes string[] -- R_* names (write targets) --- @field errors string[]|nil -- Parse-time errors from scan_source (atom_info body malformed) ---- @class DebugSkipMarker -- Sub-shape of scan_source.lua's @class DebugSkipMarker ---- @field marker_kind string -- Exact marker ident read from source. Only "atom_dbg_skip" (bare) is positive. ---- @field marker_line integer ---- @field args string|nil -- Trimmed text inside the parens (nil when has_parens is false) ---- @field has_parens boolean ---- @field is_bare boolean -- true iff marker_kind == "atom_dbg_skip" AND has_parens == false (the only positive form) ---- @field pending boolean -- true while awaiting the following declaration ---- @field superseded_by_marker_line integer|nil -- Set on a marker that was bumped out of the pending slot ---- @field target_kind string|nil -- "atom" | "comp_bare" | "comp_proc" | "unrelated" once observed - ---- @class Finding ---- @field line integer -- Source line (or 0 for pass-level) ---- @field msg string -- Finding message +--- @class RegTypeOccurrence +--- @field reg string +--- @field type_name string +--- @field source_line integer --- @class Findings ---- @field errors Finding[] ---- @field warnings Finding[] ---- @field info Finding[] +--- @field errors PassFinding[] +--- @field warnings PassFinding[] +--- @field info PassFinding[] --- @class PipeCtx ---- @field atom_index table -- raw_name or name -> scan.atoms row (kind atom/atom_proc) ---- @field binds_index table -- Name -> BindsStruct ---- @field annot_counts table -- Name -> annotation count (for unique_annotation check) ---- @field types table -- From scan_source ---- @field atom_views table -- From scan_source ---- @field seen_defaults table -- Duplicate atom_dbg_reg_default detection ---- @field seen_field table -- Binds_* -> count of fields (set/checked by check_binds_no_duplicate_fields) ---- @field _scan SourceScan -- Full scan payload (typed-view sub-calls live here) +--- @field atom_index table -- raw_name or name -> scan.atoms row (kind atom/atom_proc) +--- @field binds_index table +--- @field annot_counts table -- bag: atom name -> annotation count +--- @field types table +--- @field atom_views table +--- @field seen_defaults table -- bag: register ident -> occurrence count +--- @field seen_field table -- bag: leftover field-count slot +--- @field _scan SourceScan +--- @field word_counts WordCounts|nil +--- @field register_alias_registry table|nil +--- @field type_name_registry table|nil +--- @field type_occurrences RegTypeOccurrence[]|nil +--- @field atom_infos_list AtomInfoEntry[]|nil +--- @field binds_list BindsEntry[]|nil --- @class AnnotatedResult --- @field atoms AtomEntry[] --- @field annots AtomAnnotation[] --- @field macros MacroEntry[] --- @field binds BindsEntry[] ---- @field errors Finding[] ---- @field warnings Finding[] ---- @field info Finding[] +--- @field errors PassFinding[] +--- @field warnings PassFinding[] +--- @field info PassFinding[] +--- @field source string|nil + +--- @class CheckRule +--- @field per_annot (fun(item: AtomAnnotation, pipe_ctx: PipeCtx, findings: Findings): nil)|nil + +--- @class SourceScan +--- @field type_occurrences RegTypeOccurrence[]|nil + +--- @class AnnotationPass +--- @field validate fun(ctx: PassCtx, src: SourceFile, corpus_pipe_ctx: PipeCtx|nil): AnnotatedResult +--- @field run fun(ctx: PassCtx): PassResult -- ════════════════════════════════════════════════════════════════════════════ -- Per-check functions (the CHECK_RULES table's payload) @@ -100,6 +90,7 @@ local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua") --- @param info AtomAnnotation --- @param pipe_ctx PipeCtx --- @param findings Findings +--- @return nil local function check_atom_decl_exists(info, pipe_ctx, findings) if not pipe_ctx.atom_index[info.atom_name] then findings.errors[#findings.errors + 1] = { @@ -111,9 +102,12 @@ end --- Check: Every atom may have AT MOST ONE annotation. --- Post-loop: Needs full-corpus `annot_counts` from pipe_ctx. +--- @param _item AtomAnnotation|nil --- @param pipe_ctx PipeCtx --- @param findings Findings +--- @return nil local function check_unique_annotation(_item, pipe_ctx, findings) + --- @type string, integer for name, n in pairs(pipe_ctx.annot_counts) do if n > 1 then findings.errors[#findings.errors + 1] = { @@ -129,6 +123,7 @@ end --- @param info AtomAnnotation --- @param pipe_ctx PipeCtx --- @param findings Findings +--- @return nil local function check_binds_struct_exists(info, pipe_ctx, findings) if not info.binds then return end if pipe_ctx.binds_index[info.binds] then return end @@ -142,11 +137,14 @@ end --- Check: TAPE_WORDS(mac_X, N) ↔ WORD_COUNT(mac_X, N) drift. --- Three outcomes: missing (error), mismatch (error), match (info). ---- @param m MacroEntry ---- @param wc table -- Shared word-count table (from ctx.shared.word_counts) +--- @param m MacroEntry +--- @param pipe_ctx PipeCtx --- @param findings Findings +--- @return nil local function check_macro_word_drift(m, pipe_ctx, findings) + --- @type WordCounts local wc = (pipe_ctx and pipe_ctx.word_counts) or {} + --- @type integer|nil local declared = wc[m.name] if not declared then findings.errors[#findings.errors + 1] = { @@ -173,9 +171,12 @@ end --- @param _src SourceFile -- unused (kept for the per_source shape) --- @param pipe_ctx PipeCtx --- @param findings Findings +--- @return nil local function check_semantic_reg_defaults(_src, pipe_ctx, findings) -- Detect duplicate defaults using the ordered occurrence list (the out.types hash only retains the last declaration). + --- @type table -- bag: register ident -> first source line local seen_first_line = {} + --- @type integer, RegTypeOccurrence for _, occ in ipairs(pipe_ctx.type_occurrences or {}) do if seen_first_line[occ.reg] == nil then seen_first_line[occ.reg] = occ.source_line @@ -188,8 +189,11 @@ local function check_semantic_reg_defaults(_src, pipe_ctx, findings) } end end + --- @type table local reg_registry = pipe_ctx.register_alias_registry or {} + --- @type table local type_registry = pipe_ctx.type_name_registry or {} + --- @type string, RegTypeDefault for reg, def in pairs(pipe_ctx.types or {}) do if not reg_registry[reg] then findings.errors[#findings.errors + 1] = { @@ -223,11 +227,16 @@ end --- @param _src SourceFile --- @param pipe_ctx PipeCtx --- @param findings Findings +--- @return nil local function check_atom_reg_types(_src, pipe_ctx, findings) + --- @type table local reg_registry = pipe_ctx.register_alias_registry or {} + --- @type table local type_registry = pipe_ctx.type_name_registry or {} + --- @type integer, AtomInfoEntry for _, ai in ipairs(pipe_ctx.atom_infos_list or {}) do if ai.reg_type_overrides then + --- @type string, RegTypeOverride for reg, ov in pairs(ai.reg_type_overrides) do if not reg_registry[reg] then findings.errors[#findings.errors + 1] = { @@ -254,11 +263,14 @@ end --- @param _src SourceFile --- @param pipe_ctx PipeCtx --- @param findings Findings +--- @return nil local function check_atom_view_layout(_src, pipe_ctx, findings) + --- @type string, AtomViewEntry for atom_name, view in pairs(pipe_ctx.atom_views or {}) do if not view.binds_name then -- The atom had atom_reg_types but no atom_view; no layout check needed. else + --- @type BindsEntry|nil local bs = pipe_ctx.binds_index[view.binds_name] if not bs then findings.errors[#findings.errors + 1] = { @@ -280,15 +292,20 @@ local function check_atom_view_layout(_src, pipe_ctx, findings) end --- Check: Binds_* structs require unique field names because atom_view uses those names for typed-field lookup in gdb. ---- @param _src SourceFile +--- @param _src SourceFile --- @param pipe_ctx PipeCtx --- @param findings Findings +--- @return nil local function check_binds_no_duplicate_fields(_src, pipe_ctx, findings) + --- @type integer, BindsEntry for _, bs in ipairs(pipe_ctx.binds_list or {}) do + --- @type table -- bag: field name -> occurrence count local seen = {} + --- @type integer, TypeField for _, f in ipairs(bs.fields or {}) do seen[f.name] = (seen[f.name] or 0) + 1 end + --- @type string, integer for name, count in pairs(seen) do if count > 1 then findings.errors[#findings.errors + 1] = { @@ -312,11 +329,14 @@ end --- 5. pending + no target_kind -> dangling (no following declaration) --- 6. unsupported target_kind -> marker precedes an unrelated declaration --- Valid markers stamp `debug_skip` on whole-atom, bare-component, and proc-component declaration records in scan_source.lua. ---- @param marker DebugSkipMarker +--- @param marker DebugSkipMarker --- @param _pipe_ctx PipeCtx -- Unused; kept for consistency with per_annot ---- @param findings Findings +--- @param findings Findings +--- @return nil local function check_skip_marker(marker, _pipe_ctx, findings) + --- @type string local kind = marker.marker_kind + --- @type integer local line = marker.marker_line -- Left `scan.debug_skip_markers` with production records for `atom_dbg_skip` only; other identifiers take the walker's unrelated branch. @@ -372,12 +392,16 @@ end --- @param _src SourceFile --- @param pipe_ctx PipeCtx --- @param findings Findings +--- @return nil local function check_wave_context_migration(_src, pipe_ctx, findings) if not (pipe_ctx.types and next(pipe_ctx.types)) then return end if not (pipe_ctx.atom_infos_list) then return end + --- @type table local reg_registry = pipe_ctx.register_alias_registry or {} + --- @type integer, AtomInfoEntry for _, ai in ipairs(pipe_ctx.atom_infos_list) do if ai.reg_type_overrides then + --- @type string, RegTypeOverride for reg, _ in pairs(ai.reg_type_overrides) do if not reg_registry[reg] then findings.warnings[#findings.warnings + 1] = { @@ -405,6 +429,7 @@ end -- -- Adding a new check = 1 row here + 1 function above. The `validate()` dispatch loop never needs editing. +--- @type CheckRule[] local CHECK_RULES = { { name = "atom_decl_exists", per_annot = check_atom_decl_exists }, { name = "binds_struct_exists", per_annot = check_binds_struct_exists }, @@ -428,8 +453,11 @@ local CHECK_RULES = { --- @param ctx PassCtx --- @return PipeCtx local function build_corpus_pipe_ctx(ctx) + --- @type PipeCtx local view = duffle.corpus_view(ctx) + --- @type table -- bag: atom name -> annotation count local annot_counts = {} + --- @type integer, AtomInfoEntry for _, info in ipairs(view.atom_infos) do if info and info.atom_name then annot_counts[info.atom_name] = (annot_counts[info.atom_name] or 0) + 1 @@ -448,12 +476,16 @@ end --- @return AnnotatedResult local function validate(ctx, src, corpus_pipe_ctx) corpus_pipe_ctx = corpus_pipe_ctx or build_corpus_pipe_ctx(ctx) + --- @type SourceScan local scan = src.scan -- Build a per-source pipe_ctx: shared lookups come from `corpus_pipe_ctx`, while declarations, bodies, types, views, defaults, and occurrences come from `src.scan`. + --- @type table -- bag: register ident -> occurrence count local seen_defaults = {}; for reg, _ in pairs (scan.types or {}) do seen_defaults[reg] = (seen_defaults[reg] or 0) + 1 end + --- @type AtomInfoEntry[] local atom_infos_list = {}; for _, ai in ipairs(scan.atom_infos or {}) do atom_infos_list[#atom_infos_list + 1] = ai end + --- @type PipeCtx local pipe_ctx = { atom_index = {}, binds_index = {}, @@ -468,22 +500,28 @@ local function validate(ctx, src, corpus_pipe_ctx) register_alias_registry = corpus_pipe_ctx.register_alias_registry, type_name_registry = corpus_pipe_ctx.type_name_registry, } + --- @type AtomEntry[] local atoms = {} + --- @type integer, AtomEntry for _, a in ipairs(scan.atoms) do if a.kind == "atom" or a.kind == "atom_proc" then atoms[#atoms + 1] = a pipe_ctx.atom_index[a.raw_name or a.name] = a end end + --- @type integer, BindsEntry for _, b in ipairs(scan.binds) do pipe_ctx.binds_index[b.name] = b end -- Findings live in a single struct with three lists (errors / warnings / info). -- Each check writes to the list appropriate for its severity. + --- @type Findings local findings = { errors = {}, warnings = {}, info = {} } -- Lift parse-time errors already recorded in scan_source's atom_info payload into this pass's findings list. + --- @type integer, AtomInfoEntry for _, info in ipairs(scan.atom_infos) do if info.errors then + --- @type integer, string for _, msg in ipairs(info.errors) do findings.errors[#findings.errors + 1] = { line = info.info_line, @@ -494,6 +532,7 @@ local function validate(ctx, src, corpus_pipe_ctx) end -- THE per-annotation pipeline. ONE loop. CHECK_RULES dispatches per_annot rules. + --- @type integer, AtomInfoEntry for _, info in ipairs(scan.atom_infos) do duffle.run_check_rules(CHECK_RULES, "per_annot", info, pipe_ctx, findings) end @@ -503,13 +542,16 @@ local function validate(ctx, src, corpus_pipe_ctx) -- scan_source records each marker in scan.debug_skip_markers; this loop validates each record independently and emits at most one error per marker. -- Valid markers stamp `debug_skip = true` on the following atom or component declaration, which downstream consumers read directly. + --- @type DebugSkipMarker[] local skip_markers = scan.debug_skip_markers or {} + --- @type integer, DebugSkipMarker for _, marker in ipairs(skip_markers) do duffle.run_check_rules(CHECK_RULES, "per_skip_marker", marker, pipe_ctx, findings) end -- Per-macro rules (TAPE_WORDS vs WORD_COUNT drift). pipe_ctx.word_counts = corpus_pipe_ctx.word_counts + --- @type integer, MacroEntry for _, m in ipairs(scan.macros) do duffle.run_check_rules(CHECK_RULES, "per_macro", m, pipe_ctx, findings) end @@ -540,8 +582,7 @@ end -- M.run — orchestrator entry -- ════════════════════════════════════════════════════════════════════════════ ---- @class M - +--- @type AnnotationPass local M = {} -- Expose `validate` for downstream passes (e.g. report.lua) that need to re-render the per-source results into a per-MODULE report. @@ -550,31 +591,46 @@ M.validate = validate --- @param ctx PassCtx --- @return PassResult function M.run(ctx) + --- @type PassOutputEntry[] local outputs = {} + --- @type PassFinding[] local errors = {} + --- @type PassFinding[] local warnings = {} -- Build the shared pipe_ctx once for this run; every validate() call sees the same cross-source registries. -- The corpus owns the canonical cross-source registries; per-source scans retain body / declaration ownership. + --- @type PipeCtx local corpus_pipe_ctx = build_corpus_pipe_ctx(ctx) + --- @type Corpus local corpus = ctx.shared.corpus -- Group `corpus.sources_by_dir` by module, validate every source in each bucket, and emit one errors.h per directory. + --- @type table local by_dir = (corpus and corpus.sources_by_dir) or {} + --- @type string, SourceFile[] for dir, dir_sources in pairs(by_dir) do + --- @type string local dir_basename = dir:match("([^/\\]+)$") or dir + --- @type integer local dir_atoms = 0 + --- @type PassFinding[] local dir_errors = {} + --- @type PassFinding[] local dir_warnings = {} + --- @type integer, SourceFile for _, src in ipairs(dir_sources) do + --- @type AnnotatedResult local result = validate(ctx, src, corpus_pipe_ctx) result.source = src.path -- tag for downstream rendering dir_atoms = dir_atoms + #result.atoms + --- @type integer, PassFinding for _, e in ipairs(result.errors) do dir_errors[#dir_errors + 1] = { line = e.line, msg = e.msg, source = src.path } errors [#errors + 1] = { line = e.line, msg = e.msg } end + --- @type integer, PassFinding for _, w in ipairs(result.warnings) do dir_warnings[#dir_warnings + 1] = { line = w.line, msg = w.msg } warnings [#warnings + 1] = { line = w.line, msg = w.msg } diff --git a/scripts/passes/atoms_source_map.lua b/scripts/passes/atoms_source_map.lua index d1246a4..4dc4d5a 100644 --- a/scripts/passes/atoms_source_map.lua +++ b/scripts/passes/atoms_source_map.lua @@ -37,8 +37,11 @@ -- Bootstrap: load `duffle_paths.lua` via `debug.getinfo(1, "S").source` -- (works both standalone + when require'd). `duffle_paths.lua` sets package.path then returns `require("duffle")` -- at the bottom, so the dofile value IS the duffle module. +--- @type string local _bootstrap_dir = debug.getinfo(1, "S").source:match("^@?(.*[/\\])") or "./" +--- @type DuffleExport local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua") +--- @type ElfDwarfMod local elf_dwarf = require("elf_dwarf") -- ════════════════════════════════════════════════════════════════════════════ @@ -47,6 +50,7 @@ local elf_dwarf = require("elf_dwarf") -- Format version emitted as the first line. Bump + add a migration test if the format changes; -- the gdb runtime loader rejects mismatches (E2). +--- @type integer local FORMAT_VERSION = 1 -- ════════════════════════════════════════════════════════════════════════════ @@ -54,29 +58,71 @@ local FORMAT_VERSION = 1 -- ════════════════════════════════════════════════════════════════════════════ --- @class AtomSourceMapCtx ---- @field shared table -- `ctx.shared` ---- @field shared.corpus table -- source-order registry; single writer is build_ctx ---- @field shared.word_counts table ---- @field out_root string -- output root (e.g. "build/gen") ---- @field flags table -- `ctx.flags`; reads `flags.gdb_runtime` + `flags.elf_path` +--- @field shared PassShared +--- @field out_root string +--- @field flags PassFlags +--- @field project_root string|nil + +--- @class WordMapEntry +--- @field pos integer +--- @field line integer +--- @field text string +--- @field body_line integer +--- @field gpr_keys string[]|nil +--- @field invocation InvocationRecord|nil + +--- @class NmAddr +--- @field [1] integer -- st_value +--- @field [2] integer -- st_size + +--- @class GdbAtomRecord +--- @field idx integer|nil +--- @field name string +--- @field src_path string +--- @field file_base string +--- @field addr integer +--- @field size_bytes integer +--- @field words integer +--- @field entries WordMapEntry[] + +--- @class ElfDwarfMod +--- @field read_nm fun(elf_path: Path): table + +--- @class AtomSourceMapPass +--- @field render_source_map fun(src: SourceFile): string +--- @field render_provenance fun(src: SourceFile, wc: WordCounts): string +--- @field render_atom_source_map fun(atom: AtomEntry): string +--- @field render_atom_provenance fun(atom: AtomEntry, wc: WordCounts, rel_path: string): string +--- @field run fun(ctx: PassCtx): PassResult + +--- @class AtomEntry +--- @field paths AtomPaths|nil -- ════════════════════════════════════════════════════════════════════════════ -- Atom-path renderers -- ════════════════════════════════════════════════════════════════════════════ --- Join word boundaries (from `items`) to per-word call text + source lines (from `word_events`). ---- @param atom table ---- @return table[], integer +--- @param atom AtomEntry +--- @return WordMapEntry[] +--- @return integer local function canonical_word_entries(atom) + --- @type AtomPaths local paths = atom.paths or {} + --- @type WordEvent[] local events = paths.word_events or {} + --- @type EmissionItem[] local word_items = {} + --- @type integer, EmissionItem for _, item in ipairs(paths.items or {}) do if item.kind == "word" then word_items[#word_items + 1] = item end end + --- @type WordMapEntry[] local entries = {} + --- @type integer, WordEvent for index, event in ipairs(events) do + --- @type EmissionItem local item = word_items[index] or {} entries[#entries + 1] = { pos = event.i or (index - 1), @@ -97,18 +143,25 @@ end --- `WORD N CALL : RAW` (raw `.word` outside any mac_* component) --- Component identity comes from the outermost invocation record; the count-table lookup confirms the component was declared in `corpus.word_counts` --- (populated by word_count_eval + components passes). ---- @param src table ---- @param atom table ---- @param wc table -- identity alias of corpus.word_counts ---- @return string[], integer +--- @param src SourceFile +--- @param atom AtomEntry +--- @param wc WordCounts +--- @return string[] +--- @return integer local function emit_provenance_stanza(src, atom, wc) + --- @type string[] local lines = {} + --- @type string local rel_path = src.path:gsub("\\\\", "/") + --- @type WordMapEntry[], integer local entries, total = canonical_word_entries(atom) lines[#lines + 1] = string.format('ATOM %s "%s" 0', atom.raw_name or atom.name, rel_path) + --- @type integer, WordMapEntry for _, entry in ipairs(entries) do + --- @type InvocationRecord|nil local inv = entry.invocation + --- @type integer|nil local macro_count = inv and wc["mac_" .. inv.component_name] if inv and macro_count ~= nil then lines[#lines + 1] = string.format('WORD %d CALL %s:%d MACRO %s "%s:%d" BODY %d' @@ -125,10 +178,11 @@ local function emit_provenance_stanza(src, atom, wc) end --- Render the full provenance file content for one source. ---- @param src table ---- @param wc table +--- @param src SourceFile +--- @param wc WordCounts --- @return string local function render_provenance(src, wc) + --- @type string[] local lines = {} lines[#lines + 1] = "# FORMAT_VERSION 1" lines[#lines + 1] = "# auto-generated by ps1_meta.lua (passes/atoms_source_map.lua) — DO NOT EDIT" @@ -138,13 +192,19 @@ local function render_provenance(src, wc) lines[#lines + 1] = "# dwarf_injection to synthesize DW_TAG_inlined_subroutine instances + per-word" lines[#lines + 1] = "# line program rows for native source-level step into component bodies." + --- @param atom AtomEntry + --- @return nil local function append(atom) + --- @type string[] local stanza = emit_provenance_stanza(src, atom, wc) + --- @type integer, string for _, line in ipairs(stanza) do lines[#lines + 1] = line end end + --- @type integer, AtomEntry for _, atom in ipairs(src.scan.atoms or {}) do if atom.paths then append(atom) end end + --- @type integer, AtomEntry for _, atom in ipairs(src.scan.raw_atoms or {}) do if atom.paths then append(atom) end end @@ -154,16 +214,20 @@ end --- Render one atom's stanza for the sourcemap.txt form (ATOM header line, N WORD lines, ENDATOM marker). --- Returns (lines, total_words). ---- @param src table ---- @param atom table ---- @param wc table ---- @return string[], integer +--- @param src SourceFile +--- @param atom AtomEntry +--- @return string[] +--- @return integer local function emit_atom_stanza(src, atom) + --- @type string[] local lines = {} + --- @type string local rel_path = src.path:gsub("\\\\", "/") + --- @type WordMapEntry[], integer local entries, total = canonical_word_entries(atom) lines[#lines + 1] = string.format('ATOM %s "%s" 0', atom.raw_name or atom.name, rel_path) + --- @type integer, WordMapEntry for _, entry in ipairs(entries) do lines[#lines + 1] = string.format("WORD %d LINE %d TEXT %s", entry.pos, entry.line, entry.text) @@ -175,21 +239,27 @@ end --- Render the full source map file content for one source (one .atoms.sourcemap.txt per source). Mirrors offsets.lua's --- `project_atoms` shape: scan.atoms + scan.raw_atoms, no kind filter. ---- @param src table ---- @param wc table +--- @param src SourceFile --- @return string local function render_source_map(src) + --- @type string[] local lines = {} lines[#lines + 1] = "# FORMAT_VERSION " .. FORMAT_VERSION lines[#lines + 1] = "# auto-generated by ps1_meta.lua (passes/atoms_source_map.lua) — DO NOT EDIT" + --- @param atom AtomEntry + --- @return nil local function append(atom) + --- @type string[] local stanza = emit_atom_stanza(src, atom) + --- @type integer, string for _, line in ipairs(stanza) do lines[#lines + 1] = line end end + --- @type integer, AtomEntry for _, atom in ipairs(src.scan.atoms or {}) do if atom.paths then append(atom) end end + --- @type integer, AtomEntry for _, atom in ipairs(src.scan.raw_atoms or {}) do if atom.paths then append(atom) end end @@ -211,19 +281,29 @@ end --- Build the list of atoms with addresses + word entries. Shared helper for the gdb-runtime file emission. --- @param ctx PassCtx ---- @return table[] -- list of {idx, name, src_path, file_base, addr, size_bytes, words, entries} +--- @return GdbAtomRecord[] local function build_atom_table(ctx) + --- @type table local addrs = elf_dwarf.read_nm(ctx.flags.elf_path) + --- @type Corpus|nil local corpus = ctx.shared and ctx.shared.corpus + --- @type GdbAtomRecord[] local matched = {} + --- @type integer, SourceFile for _, src in ipairs(corpus.source_order or {}) do + --- @type string local file_base = src.path:match("([^/\\\\]+)$") or src.path + --- @param atom AtomEntry + --- @return nil local function append(atom) if not atom.paths then return end + --- @type string local name = atom.raw_name or atom.name + --- @type NmAddr|nil local info = addrs[name] if not info then return end + --- @type WordMapEntry[], integer local entries, total = canonical_word_entries(atom) matched[#matched + 1] = { name = name, @@ -235,12 +315,18 @@ local function build_atom_table(ctx) entries = entries, } end + --- @type integer, AtomEntry for _, atom in ipairs((src.scan or {}).atoms or {}) do append(atom) end + --- @type integer, AtomEntry for _, atom in ipairs((src.scan or {}).raw_atoms or {}) do append(atom) end end -- Deterministic order: sort by address (matches `nm` output ordering). + --- @param a GdbAtomRecord + --- @param b GdbAtomRecord + --- @return boolean table.sort(matched, function(a, b) return a.addr < b.addr end) + --- @type integer, GdbAtomRecord for i, a in ipairs(matched) do a.idx = i - 1 end return matched end @@ -252,12 +338,14 @@ end --- --- Why hardcoded per-atom: gdb's `$` substitution doesn't concat inside var names — `$__atom_name_$__i` in a `while` --- loop resolves to one literal identifier, not `name_i`. Compile-time emission is the only path. ---- @param lines table -- output line buffer (mutated in place) ---- @param matched table -- list of atom records from `build_atom_table` +--- @param lines string[] +--- @param matched GdbAtomRecord[] +--- @return nil local function append_gdb_commands(lines, matched) -- ── tape_atoms ── -- Hardcoded one printf per atom. No loop. lines[#lines + 1] = "define tape_atoms" + --- @type integer, GdbAtomRecord for _, a in ipairs(matched) do -- gdb 12.1 quirk: literals in printf args require an attached target. -- Use the per-atom convenience vars set above as printf args. @@ -273,6 +361,7 @@ local function append_gdb_commands(lines, matched) -- ── break_atom (generic) + per-atom break_atom_X ── lines[#lines + 1] = "define break_atom" lines[#lines + 1] = ' echo "Usage: break_atom_ (pick from the list below)"' + --- @type integer, GdbAtomRecord for _, a in ipairs(matched) do lines[#lines + 1] = string.format(' printf " break_atom_%%-32s\\n", $__atom_name_%d', a.idx) end @@ -282,6 +371,7 @@ local function append_gdb_commands(lines, matched) lines[#lines + 1] = "end" lines[#lines + 1] = "" + --- @type integer, GdbAtomRecord for _, a in ipairs(matched) do lines[#lines + 1] = string.format("define break_atom_%s", a.name) lines[#lines + 1] = string.format(" break *$__atom_addr_%d", a.idx) @@ -296,6 +386,7 @@ local function append_gdb_commands(lines, matched) -- ── step_atom / next_atom ── -- Hardcoded one tbreak per atom. No loop. lines[#lines + 1] = "define step_atom" + --- @type integer, GdbAtomRecord for _, a in ipairs(matched) do lines[#lines + 1] = string.format(" tbreak *$__atom_addr_%d", a.idx) end @@ -319,6 +410,7 @@ local function append_gdb_commands(lines, matched) lines[#lines + 1] = "define where_in_atom" lines[#lines + 1] = " set $__pc = (unsigned int)$pc" lines[#lines + 1] = " set $__matched = 0" + --- @type integer, GdbAtomRecord for _, a in ipairs(matched) do -- Precompute end_addr (gdb 12.1's expression evaluator chokes on `addr + words*4`). lines[#lines + 1] = string.format(" set $__end_%d = $__atom_addr_%d + $__atom_words_%d * 4", a.idx, a.idx, a.idx) @@ -328,14 +420,17 @@ local function append_gdb_commands(lines, matched) lines[#lines + 1] = string.format(" set $__word = ($__pc - $__atom_addr_%d) / 4", a.idx) lines[#lines + 1] = string.format(' printf "word: %%d/%%d\\n", $__word, $__atom_words_%d', a.idx) -- One inner-if per WORD entry. Each word's line + text hardcoded. + --- @type integer, WordMapEntry for _, we in ipairs(a.entries) do lines[#lines + 1] = string.format(" if $__word == %d", we.pos) -- Escape TEXT for printf format string. + --- @type string local escaped_text = we.text:gsub("%%", "%%%%"):gsub('"', '\\"') lines[#lines + 1] = string.format(' printf "source: %%s:%%d %%s\\n", $__atom_file_%d, %d, "%s"', a.idx, we.line, escaped_text) lines[#lines + 1] = " end" end -- Fallback for words beyond the source map (shouldn't happen if nm matches). + --- @type integer local max_word = 0 if #a.entries > 0 then max_word = a.entries[#a.entries].pos end lines[#lines + 1] = string.format(' if $__word > %d', max_word) @@ -361,6 +456,7 @@ local function append_gdb_commands(lines, matched) lines[#lines + 1] = " set $__in_atom = 0" lines[#lines + 1] = " set $__did_step = 0" lines[#lines + 1] = " set $__pc = (unsigned int)$pc" + --- @type integer, GdbAtomRecord for _, a in ipairs(matched) do -- Precompute end_addr in the convenience var (single expression gdb handles). lines[#lines + 1] = string.format(" set $__end_%d = $__atom_addr_%d + $__atom_words_%d * 4", a.idx, a.idx, a.idx) @@ -395,8 +491,10 @@ end --- Emit the gdb-runtime file (post-link). Pure gdb scripting — addresses come from `mipsel-none-elf-nm -S`, get embedded --- in `/gdb_tape_atoms_runtime.gdb`, and load via `set $var = ...` + `define ... end` blocks at gdb source-time. --- @param ctx PassCtx +--- @return nil local function emit_gdb_runtime(ctx) if not (ctx.flags and ctx.flags.gdb_runtime) then return end + --- @type string|nil local elf_path = ctx.flags.elf_path if not elf_path or elf_path == "" then io.stderr:write("[atoms_source_map] --gdb-runtime requires --elf \n") @@ -408,12 +506,14 @@ local function emit_gdb_runtime(ctx) return end + --- @type GdbAtomRecord[] local matched = build_atom_table(ctx) if #matched == 0 then io.stderr:write("[atoms_source_map] --gdb-runtime: no atoms matched against nm symbols (stale scan?).\n") return end + --- @type string[] local lines = {} lines[#lines + 1] = "# Auto-generated by ps1_meta.lua (passes/atoms_source_map.lua)" lines[#lines + 1] = "# DO NOT EDIT — re-run ps1_meta.lua --atoms-source-map --gdb-runtime to regenerate" @@ -435,6 +535,7 @@ local function emit_gdb_runtime(ctx) -- Per-atom convenience vars (used as printf args; literals aren't accepted -- without an attached target on gdb 12.1). + --- @type integer, GdbAtomRecord for _, a in ipairs(matched) do lines[#lines + 1] = string.format('set $__atom_name_%d = "%s"', a.idx, gdb_escape(a.name)) lines[#lines + 1] = string.format("set $__atom_addr_%d = 0x%x", a.idx, a.addr) @@ -451,10 +552,13 @@ local function emit_gdb_runtime(ctx) -- Confirmation line for the source operator. lines[#lines + 1] = 'printf "[gdb_tape_atoms] runtime loaded %d atoms from %s\\n", $__atom_count, $__elf_path' + --- @type string local out_path -- Move out of `/gdb_tape_atoms_runtime.gdb` to `/../gdb_tape_atoms_runtime.gdb` when the conventional `` is `/gen` -- (any equivalent spelling — relative, absolute backslash, absolute forward-slash, trailing-separator variants). -- This puts the gdb runtime alongside the ELF at `build/` rather than under the report subdir. + --- @param p string + --- @return boolean local function ends_with_gen_dir(p) if type(p) ~= "string" then return false end return p:match("[/\\]gen[/\\]?$") ~= nil or p == "build/gen" or p == "build\\gen" @@ -462,6 +566,7 @@ local function emit_gdb_runtime(ctx) if ends_with_gen_dir(ctx.out_root) then -- Strip the trailing `/gen` segment, then write the runtime script under `build/`. -- e.g. "C:/projects/Pikuma/ps1/build/gen" -> "C:/projects/Pikuma/ps1/build". + --- @type string local parent = ctx.out_root:gsub("[/\\]gen[/\\]?$", "") out_path = parent .. "/gdb_tape_atoms_runtime.gdb" else @@ -476,6 +581,7 @@ end -- M — module exports -- ════════════════════════════════════════════════════════════════════════════ +--- @type AtomSourceMapPass local M = {} -- Expose the pure render functions so `report.lua` and the focused tests can call them directly without triggering the file-emit path. @@ -483,19 +589,26 @@ M.render_source_map = render_source_map M.render_provenance = render_provenance --- Render ONE atom's sourcemap stanza. ---- @param atom table -- atom record (must have `atom.paths` populated) +--- @param atom AtomEntry --- @return string function M.render_atom_source_map(atom) assert(type(atom) == "table", "render_atom_source_map: atom must be a table") assert(type(atom.paths) == "table", "render_atom_source_map: atom.paths must be a table") + --- @type WordMapEntry[], integer local entries, total = canonical_word_entries(atom) + --- @type string[] local lines = {} lines[#lines + 1] = string.format("ATOM %s %d", (atom.raw_name or atom.name), total) + --- @type integer, WordMapEntry for _, entry in ipairs(entries) do + --- @type string local word_line = string.format("WORD %d LINE %d TEXT %s", entry.pos, entry.line, entry.text) + --- @type string[] local keys = {} + --- @type integer for pos = 1, 16 do + --- @type string|nil local k = entry.gpr_keys and entry.gpr_keys[pos] if type(k) == "string" and k:sub(1, 7) == "reguse:" then keys[#keys + 1] = k @@ -514,19 +627,24 @@ end --- --- `rel_path` is the source path (forward-slashes) embedded in every `CALL` line. --- The .md caller (report.lua) is expected to derive this once per `## ` heading and pass it down for each atom in that source. ---- @param atom table -- atom record (must have `atom.paths` populated) ---- @param wc table -- identity alias of `corpus.word_counts` ---- @param rel_path string -- source path (forward-slashes) for `CALL` fields +--- @param atom AtomEntry +--- @param wc WordCounts +--- @param rel_path string --- @return string function M.render_atom_provenance(atom, wc, rel_path) assert(type(atom) == "table", "render_atom_provenance: atom must be a table") assert(type(atom.paths) == "table", "render_atom_provenance: atom.paths must be a table") assert(type(rel_path) == "string", "render_atom_provenance: rel_path must be a string") + --- @type WordMapEntry[], integer local entries, total = canonical_word_entries(atom) + --- @type string[] local lines = {} lines[#lines + 1] = string.format("ATOM %s %d", (atom.raw_name or atom.name), total) + --- @type integer, WordMapEntry for _, entry in ipairs(entries) do + --- @type InvocationRecord|nil local inv = entry.invocation + --- @type integer|nil local macro_count = inv and wc and wc["mac_" .. inv.component_name] if inv and macro_count ~= nil then lines[#lines + 1] = string.format('WORD %d CALL %s:%d MACRO %s "%s:%d" BODY %d' @@ -546,16 +664,21 @@ end --- @param ctx PassCtx --- @return PassResult function M.run(ctx) + --- @type PassOutputEntry[] local outputs = {} + --- @type PassFinding[] local errors = {} + --- @type PassFinding[] local warnings = {} + --- @type Corpus|nil local corpus = ctx.shared and ctx.shared.corpus if type(corpus) ~= "table" or type(corpus.source_order) ~= "table" then error("atoms_source_map.run requires ctx.shared.corpus.source_order (canonical corpus).", 0) end -- Word counts come from `corpus.word_counts` (populated by word_count_eval + components passes). + --- @type WordCounts local wc = corpus.word_counts or {} if not next(wc) then warnings[#warnings + 1] = { diff --git a/scripts/passes/auto_reg.lua b/scripts/passes/auto_reg.lua index 507e54c..6bd2c0c 100644 --- a/scripts/passes/auto_reg.lua +++ b/scripts/passes/auto_reg.lua @@ -18,12 +18,26 @@ --- Pool exhaustion: If a phase declares more `R_` mappings than the 10-register pool can hold, --- emit `phase_register_pool_exhausted` as a build-stopping error. ---- @class AutoRegResult ---- @field outputs table[] -- {kind=, path=} entries ---- @field errors table[] -- {line=, msg=} entries (build-stops) ---- @field warnings table[] -- {line=, msg=} entries (build-continues) +--- @alias GprIdent string +--- @class GprAllocMap +--- @field [string] GprIdent -- bag: auto-reg symbol -> physical GPR + +--- @class AutoRegOutput +--- @field auto_reg_h string + +--- @class AutoRegResult +--- @field outputs AutoRegOutput[] +--- @field errors PassFinding[] +--- @field warnings PassFinding[] + +--- @class AutoRegPass +--- @field run fun(ctx: PassCtx): AutoRegResult +--- @field POOL GprIdent[] + +--- @type string local _bootstrap_dir = debug.getinfo(1, "S").source:match("^@?(.*[/\\])") or "./" +--- @type DuffleExport local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua") --- ════════════════════════════════════════════════════════════════════════════ @@ -42,6 +56,7 @@ local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua") --- R_K0/K1 (codes 26-27) — Kernel / interrupt handler reserves. Never touched by user code. --- R_GP/SP/FP/RA (codes 28-31) — R_SP/R_FP/R_RA are tape-runtime carriers between tape_enter and tape_exit; R_GP stays the host global pointer. --- +--- @type GprIdent[] local POOL = { "R_V0", "R_V1", "R_T0", "R_T1", "R_T2", "R_T3", @@ -57,6 +72,7 @@ local POOL = { -- Only the POOL entries matter for auto_reg — non-pool aliases -- (R_AT=1, R_A0..A3=4..7, R_T8=24, R_T9=25, R_K0/K1=26..27, R_GP/SP/FP/RA=28..31) -- are deliberately omitted — see the comment block above for the WHY of each exclusion. +--- @type table -- bag: MIPS GPR code -> POOL ident local INT_CODE_TO_POOL_GPR = { [2] = "R_V0", [3] = "R_V1", [4] = "R_A0", [5] = "R_A1", [6] = "R_A2", [7] = "R_A3", @@ -68,8 +84,12 @@ local INT_CODE_TO_POOL_GPR = { } -- Stable sort for deterministic allocation order. +--- @param tbl table -- bag: key set only; values unused +--- @return string[] local function stable_sort_keys(tbl) + --- @type string[] local keys = {} + --- @type string for k in pairs(tbl) do keys[#keys + 1] = k end table.sort(keys) return keys @@ -77,15 +97,25 @@ end -- Allocate one phase's auto-reg mappings. -- Returns (allocated_map, errors). On pool exhaustion, errors is populated and the function halts. +--- @param phase_label string +--- @param decls table -- bag: auto-reg symbol -> decl payload +--- @return GprAllocMap +--- @return PassFinding[] local function allocate_phase(phase_label, decls) -- Deep-copy POOL into a fresh sequence table. The original `table.unpack and table.unpack(POOL) or { unpack(POOL) }` -- idiom wraps the unpacked values in a single inner table under LuaJIT 5.1 (`table.unpack` is nil; the `or` returns one value), -- which corrupts the pool into `{ {R_T0, R_T1, ...} }` — making `table.remove(pool, 1)` return the inner table on iteration. + --- @type GprIdent[] local pool = {} + --- @type integer for i = 1, #POOL do pool[i] = POOL[i] end + --- @type GprAllocMap local result = {} + --- @type PassFinding[] local errors = {} + --- @type integer, string for _, sym in ipairs(stable_sort_keys(decls)) do + --- @type GprIdent|nil local next_gpr = table.remove(pool, 1) if not next_gpr then errors[#errors + 1] = { @@ -110,12 +140,19 @@ end -- Each entry's `code` is the integer MIPS GPR number (0..31); INT_CODE_TO_POOL_GPR translates it back to the physical GPR ident. -- Aliases whose `code` points to a non-POOL GPR (e.g. R_S0, R_T8, R_K1) are ignored — -- they don't affect the auto_reg pool, and they're already excluded from POOL above. +--- @param corpus Corpus +--- @return table +--- @return table local function build_user_pins(corpus) + --- @type table -- bag: pinned physical GPR -> true local user_pinned = {} + --- @type table -- bag: alias ident -> physical GPR local alias_to_gpr = {} if not corpus.register_alias_registry then return user_pinned, alias_to_gpr end + --- @type string, AliasEntry for alias_name, alias_entry in pairs(corpus.register_alias_registry) do if alias_entry.has_atom_reg and alias_entry.code then + --- @type GprIdent|nil local gpr = INT_CODE_TO_POOL_GPR[alias_entry.code] if gpr then user_pinned[gpr] = true @@ -133,22 +170,32 @@ end --- Clash-detection and source-pool-exclusion logic only needs the presence of each GPR (boolean test), --- but keeping count preserves the original find_hardcoded_rn shape so callers can switch without churn. --- The alias pattern is sorted lexicographically to keep the regex deterministic. +--- @param body_text string +--- @param alias_to_gpr table -- bag: alias ident -> physical GPR +--- @return table local function find_used_gprs(body_text, alias_to_gpr) + --- @type table -- bag: physical GPR -> hit count local found = {} -- (a) Hardcoded physical GPRs (R_T0..R_T7, R_V0..R_V1, R_A0..R_A3, R_S0..R_S7). + --- @type GprIdent for gpr in body_text:gmatch("(R_T%d+|R_V%d+|R_A%d+|R_S%d+)") do found[gpr] = (found[gpr] or 0) + 1 end -- (b) Alias references (R_) resolved to physical GPRs via the registry. -- Sorted by name so the regex is byte-stable across runs. if alias_to_gpr and next(alias_to_gpr) then + --- @type string[] local aliases = {} + --- @type string for alias_name in pairs(alias_to_gpr) do aliases[#aliases + 1] = alias_name end table.sort(aliases) + --- @type string local pattern = "(" .. table.concat(aliases, "|") .. ")" + --- @type string for alias_name in body_text:gmatch(pattern) do + --- @type GprIdent|nil local gpr = alias_to_gpr[alias_name] if gpr and not found[gpr] then found[gpr] = 1 @@ -159,10 +206,17 @@ local function find_used_gprs(body_text, alias_to_gpr) end -- Emit one gen/auto_reg.h header per directory. +--- @param out_dir string +--- @param dir string +--- @param sources SourceFile[] +--- @param mappings GprAllocMap +--- @return string|nil local function emit_auto_reg_h(out_dir, dir, sources, mappings) if not mappings or next(mappings) == nil then return end + --- @type string local out_path = out_dir .. "/" .. "auto_reg.h" duffle.ensure_dir(out_dir) + --- @type string[] local lines = { "#ifdef INTELLISENSE_DIRECTIVES", "#pragma once", @@ -170,14 +224,18 @@ local function emit_auto_reg_h(out_dir, dir, sources, mappings) "// Auto-generated by ps1_meta.lua (passes/auto_reg.lua) — DO NOT EDIT", "// Directory: " .. dir:gsub("/", "\\"), } + --- @type integer, SourceFile for _, src in ipairs(sources) do lines[#lines + 1] = "// source: " .. src.path end lines[#lines + 1] = "// Per-phase register allocations resolved by the lua pass." lines[#lines + 1] = "// R__Code = for every marker in this directory." lines[#lines + 1] = "" + --- @type integer, string for _, sym in ipairs(stable_sort_keys(mappings)) do + --- @type GprIdent local gpr = mappings[sym] + --- @type string local gpr_code = gpr .. "_Code" lines[#lines + 1] = "#define " .. sym .. "_Code " .. gpr_code end @@ -191,15 +249,20 @@ end -- Pass entry -- ════════════════════════════════════════════════════════════════════════════ +--- @type AutoRegPass local M = {} --- @param ctx PassCtx --- @return AutoRegResult function M.run(ctx) + --- @type AutoRegOutput[] local outputs = {} + --- @type PassFinding[] local errors = {} + --- @type PassFinding[] local warnings = {} + --- @type Corpus|nil local corpus = ctx.shared and ctx.shared.corpus if type(corpus) ~= "table" then error("auto_reg.run requires ctx.shared.corpus", 0) @@ -210,16 +273,22 @@ function M.run(ctx) -- MUST NOT be allocated to any auto-reg marker — they're preserved across atoms by the wave-context discipline. -- The corpus's register_alias_registry is the source of truth for these opt-in pins. -- Body references to those aliases (via alias_to_gpr) are also excluded on a per-atom basis in step 2 below. + --- @type table, table local user_pinned, alias_to_gpr = build_user_pins(corpus) -- 1. Allocate phase pools first (phase declarations take precedence over per-atom declarations). + --- @type table -- bag: phase_label -> alloc map local phase_allocations = {} + --- @type string, table for phase_label, decls in pairs(corpus.phase_auto_regs or {}) do + --- @type GprAllocMap, PassFinding[] local mapping, errs = allocate_phase(phase_label, decls) + --- @type string, GprIdent for sym, gpr in pairs(mapping) do phase_allocations[phase_label] = phase_allocations[phase_label] or {} phase_allocations[phase_label][sym] = gpr end + --- @type integer, PassFinding for _, e in ipairs(errs) do errors[#errors + 1] = e end @@ -229,15 +298,21 @@ function M.run(ctx) -- Otherwise, allocate a private pool for the atom. -- The phase membership is in `corpus.atom_phases[phase_label].atoms` (an array of atom names declared via `atom_phase()` -- in the atom's `atom_info` line). Build a reverse map `atom_name -> phase_label` so the lookup is O(1) per atom scope. + --- @type table -- bag: atom name -> phase label local atom_name_to_phase = {} + --- @type string, AtomPhaseGroup for phase_label, entry in pairs(corpus.atom_phases or {}) do + --- @type integer, AtomName for _, atom_name in ipairs(entry.atoms or {}) do atom_name_to_phase[atom_name] = phase_label end end + --- @type table -- bag: atom scope -> alloc map local atom_allocations = {} + --- @type AtomName, table for atom_scope, decls in pairs(corpus.atom_auto_regs or {}) do + --- @type string|nil local phase_label = atom_name_to_phase[atom_scope] -- Build the atom's source pool: start with the full POOL, subtract: -- (a) every GPR already committed (phase allocations + prior atom allocations) @@ -248,25 +323,36 @@ function M.run(ctx) -- the original `source_pool = phase_allocations[phase_label]` form used the phase -- allocation MAP as a pool, but that map has no array part, so `table.remove(source_pool, 1)` -- returned nil and every atom-with-phase marker errored with `phase_register_pool_exhausted`. + --- @type table -- bag: committed or body-referenced GPR -> true local used = {} + --- @type integer, GprAllocMap for _, m in pairs(phase_allocations) do for _, gpr in pairs(m) do used[gpr] = true end end + --- @type integer, GprAllocMap for _, m in pairs(atom_allocations) do for _, gpr in pairs(m) do used[gpr] = true end end -- (c) Body references — scan the atom body for hardcoded + alias-resolved GPRs. -- Folded into `used` so the source_pool exclusion is a single check. + --- @type AtomEntry|nil local atom = corpus.atoms_by_name and corpus.atoms_by_name[atom_scope] if atom and atom.body then + --- @type table local body_used = find_used_gprs(atom.body, alias_to_gpr) + --- @type GprIdent for gpr in pairs(body_used) do used[gpr] = true end end + --- @type GprIdent[] local source_pool = {} + --- @type integer, GprIdent for _, gpr in ipairs(POOL) do -- Exclude (a) prior commitments, (b) USER-PINNED GPRs (wave-context carriers declared via atom_reg + _Code defs, preserved across atoms globally). if not used[gpr] and not user_pinned[gpr] then source_pool[#source_pool + 1] = gpr end end + --- @type GprAllocMap local result = {} + --- @type integer, string for _, sym in ipairs(stable_sort_keys(decls)) do + --- @type GprIdent|nil local next_gpr = table.remove(source_pool, 1) if not next_gpr then errors[#errors + 1] = { @@ -289,10 +375,14 @@ function M.run(ctx) -- This warning is kept as a defensive safety net for cases the body scanner might miss -- (e.g. macros that expand to register references the scanner cannot resolve). -- For each resolved (scope, sym) -> R_Tn mapping, scan the atom body source for used GPRs. + --- @type AtomName, GprAllocMap for atom_scope, decls in pairs(atom_allocations) do + --- @type AtomEntry|nil local atom = corpus.atoms_by_name and corpus.atoms_by_name[atom_scope] if atom and atom.body then + --- @type table local used_in_body = find_used_gprs(atom.body, alias_to_gpr) + --- @type string, GprIdent for sym, allocated_gpr in pairs(decls) do if used_in_body[allocated_gpr] and used_in_body[allocated_gpr] > 0 then warnings[#warnings + 1] = { @@ -308,26 +398,36 @@ function M.run(ctx) -- 4. Emit per-directory gen/auto_reg.h. -- For each source directory that has atom_auto_regs or phase_auto_regs entries, emit one header. + --- @type table local sources_by_dir = corpus.sources_by_dir or {} + --- @type string, SourceFile[] for dir, sources in pairs(sources_by_dir) do + --- @type GprAllocMap local per_dir_mappings = {} + --- @type integer, SourceFile for _, src in ipairs(sources) do -- Collect every (sym -> gpr) entry that originated from a source in this directory. -- `src.scan.atom_auto_regs` is keyed by ATOM SCOPE NAME; `pairs(t)` iterates KEYS so `scope_name` here is the scope ident (e.g. "cube_g4_face"). -- The previous `for _, scan_atom_auto` form silently assigned the VALUE (a `{sym = sym}` table) to the variable, -- which made `atom_allocations[scan_atom_auto]` a table-indexed lookup that never resolved. + --- @type string for scope_name in pairs(src.scan and src.scan.atom_auto_regs or {}) do + --- @type string, GprIdent for sym, gpr in pairs(atom_allocations[scope_name] or {}) do per_dir_mappings[sym] = gpr end end + --- @type string for scope_name in pairs(src.scan and src.scan.phase_auto_regs or {}) do + --- @type string, GprIdent for sym, gpr in pairs(phase_allocations[scope_name] or {}) do per_dir_mappings[sym] = gpr end end end + --- @type string local out_dir = dir .. "/gen" + --- @type string|nil local out_path = emit_auto_reg_h(out_dir, dir, sources, per_dir_mappings) if out_path then outputs[#outputs + 1] = { auto_reg_h = out_path } end end diff --git a/scripts/passes/components.lua b/scripts/passes/components.lua index 3c0cd4d..d98380e 100644 --- a/scripts/passes/components.lua +++ b/scripts/passes/components.lua @@ -22,7 +22,9 @@ -- Uses `debug.getinfo` to find this file's own directory, so it works both standalone and when require'd from the orchestrator. -- Bootstrap: load `duffle_paths.lua` via `debug.getinfo(1, "S").source` (works both standalone + when require'd). -- duffle_paths.lua sets package.path then returns `require("duffle")` at the bottom, so the dofile value IS the duffle module. +--- @type string local _bootstrap_dir = debug.getinfo(1, "S").source:match("^@?(.*[/\\])") or "./" +--- @type DuffleExport local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua") -- ════════════════════════════════════════════════════════════════════════════ @@ -30,62 +32,75 @@ local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua") -- ════════════════════════════════════════════════════════════════════════════ -- Atom component declaration identifiers. +--- @type string local ATOM_COMP_PROC = "MipsAtomComp_Proc_" +--- @type string local MIPS_ATOM = "Slice_MipsCode" -- prefix on the function declaration that wraps an AtomComp_Proc_ -- Component-name prefixes. +--- @type string local AC_PREFIX = "ac_" -- arg to MipsAtomComp_(ac_X); the X is the atom name +--- @type integer local AC_PREFIX_LEN = 3 +--- @type string local MAC_PREFIX = "mac_" -- prefix on generated macros; the rest is the atom name +--- @type integer local MAC_PREFIX_LEN = 4 -- ASCII byte values used in tokenization. +--- @type integer local BYTE_NEWLINE = 10 +--- @type integer local BYTE_SLASH = 47 -- Output gen subdirectory + filename (per-directory aggregation; the directory name is the namespace). +--- @type string local GEN_SUBDIR = "gen" +--- @type string local MACS_FILENAME = "macs.h" -- ════════════════════════════════════════════════════════════════════════════ -- Type declarations -- ════════════════════════════════════════════════════════════════════════════ ---- @class SourceFile ---- @field path string -- Absolute path to the source file ---- @field text string -- Full source text ---- @field dir string -- Directory containing the source ---- @field basename string -- Filename without extension ---- @field scan table -- Pre-scanned SourceScan payload (from duffle.scan_source) - ---- @class PassCtx ---- @field sources SourceFile[] -- All source files in the build ---- @field metadata_path string -- Path to word_count.metadata.h ---- @field shared table -- Cross-pass shared state ---- @field out_root string -- Output root (e.g. "build/gen") ---- @field project_root string -- Project root (e.g. "code/") ---- @field upstream table -- Per-pass upstream outputs ---- @field flags table -- CLI flags ---- @field verbose boolean -- Log diagnostic info - ---- @class PassResult ---- @field outputs table[] -- {kind=, path=} entries describing emit files ---- @field errors table[] -- {line=, msg=} entries; build-stops ---- @field warnings table[] -- {line=, msg=} entries; build-succeeds +-- SourceFile, PassCtx, PassResult: see ps1_meta.lua +-- DuffleExport: see duffle.lua +-- SourceScan, AtomEntry, CorpusCollision, CollisionSite: see scan_source.lua +-- BodyToken: see emission_model.lua +-- WordCounts: see word_count_eval.lua +-- ComponentBodyEntry: see duffle_emit.lua +-- InstructionRow, GteCommandRow: see duffle_isa.lua --- @class Component ---- @field name string -- Atom name (without `ac_` prefix) ---- @field body string -- Brace-delimited body (without the braces) ---- @field args string|nil -- Function-args string (function form only) ---- @field line integer -- Source line of the declaration ---- @field comment string|nil -- Scanner-owned `declaration_comment`; the components pass reads it from the scanner record ---- @field kind string -- "comp_bare" | "comp_proc" (atom_proc is NOT a component — see `project_components`) ---- @field debug_skip boolean -- Mirror of `a.debug_skip` (scanner-owned); true iff a bare `atom_dbg_skip` marker immediately preceded the declaration +--- @field name string -- Atom name (without `ac_` prefix) +--- @field body string -- Brace-delimited body (without the braces) +--- @field body_off integer|nil -- Byte offset of body[1] in source +--- @field body_tokens BodyToken[]|nil +--- @field args string|nil -- Function-args string (function form only) +--- @field arg_names string[]|nil -- Formal names with leading `ab` dropped +--- @field line integer -- Source line of the declaration +--- @field comment string|nil -- Scanner-owned `declaration_comment`; the components pass reads it from the scanner record +--- @field kind string -- "comp_bare" | "comp_proc" (atom_proc is NOT a component — see `project_components`) +--- @field debug_skip boolean -- Mirror of `a.debug_skip` (scanner-owned); true iff a bare `atom_dbg_skip` marker immediately preceded the declaration + +--- @class ComponentMeta +--- @field cycle_cost integer +--- @field gp0_contrib integer + +--- @class ComponentMetaMap +--- @field [string] ComponentMeta -- bag: bare component name -> meta + +--- @class MacsOutput +--- @field macs_h string + +--- @class ComponentsPass +--- @field run fun(ctx: PassCtx): PassResult -- ════════════════════════════════════════════════════════════════════════════ -- Local helpers (file I/O + path normalization) -- ════════════════════════════════════════════════════════════════════════════ +--- @type ComponentsPass local M = {} -- ════════════════════════════════════════════════════════════════════════════ @@ -108,6 +123,7 @@ local M = {} --- @param before_pos integer --- @return string|nil local function find_function_args_for(source, name, before_pos) + --- @type string|nil, string|nil local _, args_inner = duffle.find_function_decl_for(source, before_pos, #MIPS_ATOM) return args_inner end @@ -124,9 +140,13 @@ end --- @return string[]|nil local function extract_arg_names(args_str) if not args_str or args_str == "" then return nil end + --- @type string[] local names = {} + --- @type string[] local tokens = duffle.split_top_level_commas(args_str) + --- @type integer, string for _, tok in ipairs(tokens) do + --- @type string local trimmed = duffle.trim(tok) if trimmed ~= "" then -- Strip trailing block comment (/* ... */) from the token, if present. @@ -134,13 +154,16 @@ local function extract_arg_names(args_str) -- not block comments embedded WITHIN a token between a parameter and a trailing comma. -- Without this strip, the identifier-walk below stops at the `/` of `*/` and returns -- the wrong name (or nothing). See `test_extract_arg_names_handles_trailing_block_comments`. + --- @type integer local trimmed_end = #trimmed if trimmed_end >= 2 and trimmed:sub(trimmed_end - 1, trimmed_end) == "*/" then -- Find the matching `/*` that opens the trailing comment. -- Walk back from the `*/` looking for `/*` (whitespace + `/*`). + --- @type integer local close_pos = trimmed_end - 1 -- position of the second-to-last char -- Walk back: skip trailing whitespace, then look for the `/*` opener. while close_pos > 1 do + --- @type string local ch = trimmed:sub(close_pos, close_pos) if ch == " " or ch == "\t" or ch == "\n" or ch == "\r" then close_pos = close_pos - 1 @@ -149,7 +172,9 @@ local function extract_arg_names(args_str) end end -- Now scan back from close_pos for the `/*` opener (slashes are at close_pos-1 and close_pos-2). + --- @type integer|nil local opener_pos = nil + --- @type integer local scan = close_pos - 3 while scan >= 1 do if trimmed:sub(scan, scan + 1) == "/*" then @@ -169,8 +194,10 @@ local function extract_arg_names(args_str) trimmed_end = #trimmed if trimmed_end >= 4 and trimmed:sub(trimmed_end, trimmed_end) == "]" then -- Walk back: skip digits, expect `[`. + --- @type integer local bracket_pos = trimmed_end - 1 while bracket_pos > 1 do + --- @type string local ch = trimmed:sub(bracket_pos, bracket_pos) if ch >= "0" and ch <= "9" then bracket_pos = bracket_pos - 1 @@ -185,8 +212,10 @@ local function extract_arg_names(args_str) if trimmed == "" then goto continue end -- Find the identifier at the end: walk back over trailers (whitespace + `*` + `[]`), -- then walk back over the identifier chars (alnum + `_`). + --- @type integer local ident_end = #trimmed while ident_end > 0 do + --- @type string local ch = trimmed:sub(ident_end, ident_end) if ch == " " or ch == "\t" or ch == "*" or ch == "]" or ch == "[" then ident_end = ident_end - 1 @@ -194,8 +223,10 @@ local function extract_arg_names(args_str) break end end + --- @type integer local ident_start = ident_end while ident_start > 0 do + --- @type string local ch = trimmed:sub(ident_start, ident_start) if duffle.is_alnum_byte(string.byte(ch)) or ch == "_" then ident_start = ident_start - 1 @@ -204,6 +235,7 @@ local function extract_arg_names(args_str) end end ident_start = ident_start + 1 + --- @type string local name = trimmed:sub(ident_start, ident_end) if name ~= "" then names[#names + 1] = name end ::continue:: @@ -213,7 +245,10 @@ local function extract_arg_names(args_str) return names end +--- @param args_str string|nil +--- @return string[]|nil local function formal_arg_names(args_str) + --- @type string[]|nil local names = extract_arg_names(args_str) if not names then return nil end if names[1] == "ab" then table.remove(names, 1) end @@ -233,10 +268,12 @@ end --- Carries the scanner-owned `debug_skip` flag forward so the generated projection can emit `/* atom_dbg_skip */` --- before the authored comment and so `update_canonical_components` can mirror the same field onto `corpus.components[name]`. --- @param source string -- the full source text (needed for backward lookups) ---- @param scan table -- SourceScan from duffle.scan_source +--- @param scan SourceScan --- @return Component[] local function project_components(source, scan) + --- @type Component[] local out = {} + --- @type integer, AtomEntry for _, a in ipairs(scan.atoms) do -- Only `MipsAtomComp_(ac_X)` (kind="comp_bare") and `MipsAtomComp_Proc_(ac_X, ...)` (kind="comp_proc") -- are COMPONENTS — they get inlined via `mac_` aliases inside atom bodies. @@ -248,9 +285,11 @@ local function project_components(source, scan) -- Function-args lookup is meaningful for `MipsAtomComp_Proc_` components -- (the macro sits inside `FI_ Slice_MipsCode ac_X(...)`); the alias expansion -- discards the `ab` (atom-builder) arg the same way both forms do. + --- @type string|nil local args = find_function_args_for(source, a.raw_name, a.ident_pos) -- Comment ownership: scan_source.lua stamps `declaration_comment` on the record by walking backward past any associated bare marker. -- The pass reads `declaration_comment` directly. + --- @type string local comment = a.declaration_comment or "" out[#out + 1] = { line = a.line, @@ -282,22 +321,30 @@ end --- @param s string --- @return string local function convert_line_comments_to_block(s) + --- @type string local result = s + --- @type integer local pos = 1 + --- @type integer local len = #result while pos <= len do + --- @type boolean local is_double_slash = result:byte(pos) == BYTE_SLASH and pos + 1 <= len and result:byte(pos + 1) == BYTE_SLASH if not is_double_slash then pos = pos + 1 else -- Find end of line. + --- @type integer local eol = pos while eol <= len and result:byte(eol) ~= BYTE_NEWLINE do eol = eol + 1 end + --- @type string local before = result:sub(1, pos - 1) + --- @type string local comment = result:sub(pos + 2, eol - 1) -- skip the `//` + --- @type string local after if eol <= len and result:byte(eol) == BYTE_NEWLINE then after = " */" .. result:sub(eol) -- keep the newline @@ -335,10 +382,13 @@ end --- @param tok string --- @return string local function strip_leading_delay_marker(tok) + --- @type string|nil local ident = duffle.read_ident(tok, 1) if not ident or not duffle.DELAY_MARKERS[ident] then return tok end + --- @type string local rest = tok:sub(#ident + 1):match("^%s*(.*)$") or "" while rest:sub(1, 2) == "/*" do + --- @type integer|nil local close = rest:find("*/", 3, true) if not close then return "" end rest = rest:sub(close + 2):match("^%s*(.*)$") or "" @@ -350,22 +400,29 @@ end --- in a single source's `count_all_components` pass; the in-progress -1 sentinel detects cycles (A -> B -> A). --- @param name string -- the component name (without `mac_`) --- @param comp_by_name table ---- @param wc table ---- @param cache table +--- @param wc WordCounts +--- @param cache table -- bag: name -> count; -1 in-progress sentinel --- @return integer local function word_count_rec(name, comp_by_name, wc, cache) if cache[name] ~= nil then return cache[name] end cache[name] = -1 -- mark in-progress (cycle detection) + --- @type Component|nil local cc = comp_by_name[name] + --- @type integer local n if cc then n = 0 + --- @type BodyToken[] local tokens = cc.body_tokens + --- @type integer, BodyToken for _, t in ipairs(tokens) do + --- @type string local trimmed = t.tok if trimmed ~= "" then + --- @type string local work = trimmed while true do + --- @type string|nil local marker = duffle.read_ident(work, 1) if marker and duffle.DELAY_MARKERS[marker] then work = strip_leading_delay_marker(work) @@ -375,6 +432,7 @@ local function word_count_rec(name, comp_by_name, wc, cache) end end if work ~= "" then + --- @type string|nil local lookup = strip_mac_prefix(duffle.read_ident(work, 1)) if lookup == "atom_label" or lookup == "atom_offset" then -- Pure metaprogram anchors; emit zero words. @@ -405,13 +463,18 @@ end --- references hit memoized values instead of re-walking the body. --- Cycle detection (A -> B -> A) is preserved via the in-progress `-1` sentinel in `cache`. --- @param components Component[] ---- @param wc table ---- @return table -- map of component name (without `mac_`) -> word count +--- @param wc WordCounts +--- @return table -- bag: bare component name -> word count local function count_all_components(components, wc) + --- @type table local comp_by_name = {} + --- @type integer, Component for _, cc in ipairs(components) do comp_by_name[cc.name] = cc end + --- @type table -- bag: memo; -1 in-progress sentinel local cache = {} + --- @type table -- bag: bare name -> word count local counts = {} + --- @type integer, Component for _, c in ipairs(components) do counts[c.name] = word_count_rec(c.name, comp_by_name, wc, cache) end @@ -435,28 +498,39 @@ end --- Missing component: cycle 1, gp0 0. --- @param name string -- component bare name (e.g. "yield", "pack_color_word") --- @param comp_by_name table ---- @param latency table ---- @param cache table ---- @return {cycle_cost=integer, gp0_contrib=integer} +--- @param latency table -- bag: ident -> cycle cost +--- @param cache ComponentMetaMap +--- @return ComponentMeta local function component_meta_rec(name, comp_by_name, latency, cache) if cache[name] ~= nil then return cache[name] end cache[name] = { cycle_cost = -1, gp0_contrib = -1 } + --- @type Component|nil local cc = comp_by_name[name] + --- @type integer local cycle_cost + --- @type integer local gp0_contrib if cc then + --- @type boolean local skip_cycle = (name == "yield") + --- @type boolean local skip_gp0 = name:match("^insert_ot_tag") ~= nil cycle_cost = 0 gp0_contrib = 0 if not skip_cycle or not skip_gp0 then + --- @type BodyToken[] local tokens = cc.body_tokens + --- @type integer, BodyToken for _, t in ipairs(tokens) do + --- @type string local trimmed = t.tok if trimmed ~= "" then + --- @type string|nil local ident = duffle.read_ident(trimmed, 1) if ident and ident:sub(1, MAC_PREFIX_LEN) == MAC_PREFIX then + --- @type string local nested = ident:sub(MAC_PREFIX_LEN + 1) + --- @type ComponentMeta local nested_meta = component_meta_rec(nested, comp_by_name, latency, cache) if not skip_cycle then cycle_cost = cycle_cost + nested_meta.cycle_cost @@ -466,7 +540,9 @@ local function component_meta_rec(name, comp_by_name, latency, cache) end else if not skip_cycle then + --- @type InstructionRow|nil local isa = duffle.instr(ident) + --- @type GteCommandRow|nil local gte = duffle.gte(ident) cycle_cost = cycle_cost + ((isa and isa.cycles) or (gte and gte.cycles) or latency[ident] or 1) end @@ -499,13 +575,18 @@ end --- Compute `cycle_cost` + `gp0_contrib` for every component in `components` in a single pass. --- One memoization cache; a nested `mac_Y` inside a `mac_X` body computes both fields once. --- @param components Component[] ---- @param latency table ---- @return table +--- @param latency table -- bag: ident -> cycle cost +--- @return ComponentMetaMap local function compute_components_metadata(components, latency) + --- @type table local comp_by_name = {} + --- @type integer, Component for _, cc in ipairs(components) do comp_by_name[cc.name] = cc end + --- @type ComponentMetaMap local cache = {} + --- @type ComponentMetaMap local out = {} + --- @type integer, Component for _, c in ipairs(components) do out[c.name] = component_meta_rec(c.name, comp_by_name, latency, cache) end @@ -521,10 +602,14 @@ end --- @param s string --- @return string[] local function split_comment_lines(s) + --- @type string[] local out = {} + --- @type integer local pos = 1 + --- @type integer local s_len = #s while pos <= s_len do + --- @type integer|nil local nl = s:find("\n", pos, true) if not nl then out[#out + 1] = s:sub(pos) @@ -544,6 +629,7 @@ end --- @param args_str string|nil --- @return string local function signature_from_args(args_str) + --- @type string[]|nil local names = formal_arg_names(args_str) if names then return table.concat(names, ", ") @@ -553,7 +639,10 @@ end --- Strip the trailing `" \"` (space + backslash) line continuation from the last body line. --- The last 2 chars are always that pair. +--- @param lines string[] +--- @return nil local function strip_trailing_continuation(lines) + --- @type string local last = lines[#lines] if last:sub(-2) == " \\" then lines[#lines] = last:sub(1, -3) @@ -580,12 +669,15 @@ end --- @param tok string -- a single token from split_top_level_commas (already trimmed at the start, may contain trailing whitespace + block comment) --- @return boolean local function is_pure_delay_marker_token(tok) + --- @type table -- bag: delay-marker ident -> true local markers = duffle.DELAY_MARKERS if type(markers) ~= "table" then return false end -- Identify a leading delay-marker identifier (e.g. `GteDelay_`). + --- @type integer local ident_end = 1 while ident_end <= #tok do + --- @type string local ch = tok:sub(ident_end, ident_end) if ch:match("[%w_]") then ident_end = ident_end + 1 @@ -593,16 +685,20 @@ local function is_pure_delay_marker_token(tok) break end end + --- @type string local ident = tok:sub(1, ident_end - 1) if not markers[ident] then return false end -- Walk the remainder: only whitespace and block comments are allowed. + --- @type integer local scan = ident_end while scan <= #tok do + --- @type string local ch = tok:sub(scan, scan) if ch:match("%s") then scan = scan + 1 elseif ch == "/" and tok:sub(scan + 1, scan + 1) == "*" then + --- @type integer|nil local close = tok:find("*/", scan + 2, true) if not close then return false end scan = close + 2 @@ -641,14 +737,22 @@ end --- the annotation IS preserved in the generated header --- (so the comment + marker remain visible to anyone reading `gen/macs.h`), but the C preprocessor expands the marker to empty, so leaving the `,` --- separator out is what stops the `,,` syntax error. See `token_skips_leading_comma` for the contract. +--- @param lines string[] +--- @param c Component +--- @param sig string +--- @param tokens string[] +--- @return nil local function emit_macro_body(lines, c, sig, tokens) + --- @type integer for tok_idx = 1, #tokens do tokens[tok_idx] = convert_line_comments_to_block(tokens[tok_idx]) end if #tokens == 0 then return end lines[#lines + 1] = "#define mac_" .. c.name .. "(" .. sig .. ") \\" lines[#lines + 1] = "\t" .. tokens[1] .. " \\" + --- @type integer for tok_idx = 2, #tokens do + --- @type string local sep = token_skips_leading_comma(tokens[tok_idx]) and "\t" or ",\t" lines[#lines + 1] = sep .. tokens[tok_idx] .. " \\" end @@ -660,11 +764,11 @@ end --- For skipped components, a `/* atom_dbg_skip */` marker comment is emitted immediately before the authored comment block. --- The marker is a single line, the comment comes next, and the `#define` line follows. The `debug_skip` stamp is scanner-owned --- (`a.debug_skip == true` on the declaration record); the components pass projects it directly. ---- @param c Component ---- @param components Component[] ---- @param wc table +--- @param c Component +--- @param counts table -- bag: bare component name -> word count --- @return string[] -- list of lines for this component local function build_component_lines(c, counts) + --- @type string[] local lines = {} -- Marker comment: emitted once for every skipped component. @@ -675,15 +779,20 @@ local function build_component_lines(c, counts) end if c.comment and c.comment ~= "" then + --- @type integer, string for _, line in ipairs(split_comment_lines(c.comment)) do lines[#lines + 1] = line end end + --- @type string[] local tokens = duffle.split_top_level_commas(c.body) + --- @type integer for i = 1, #tokens do tokens[i] = duffle.trim(tokens[i]) end + --- @type string local sig = signature_from_args(c.args) -- Direct lookup against the per-source precomputed `counts` table (built once by count_all_components). + --- @type integer local n = counts[c.name] if n > 0 then @@ -707,10 +816,13 @@ end --- @param sources SourceFile[] -- Sources contributing to this directory (for the header comment) --- @return string[] local function header_boilerplate(dir, sources) + --- @type string[] local source_lines = { "// Directory: " .. duffle.to_absolute_path(dir) .. "/" } + --- @type integer, SourceFile for _, src in ipairs(sources) do source_lines[#source_lines + 1] = "// source: " .. duffle.to_absolute_path(src.path) end + --- @type string local source_blob = table.concat(source_lines, "\n") return { -- #pragma once wrapped in #ifdef INTELLISENSE_DIRECTIVES, matching the convention in lottes_tape.h. @@ -739,7 +851,9 @@ end --- @return string -- Output directory --- @return string -- Full output path local function compute_macs_h_path(dir) + --- @type string local out_dir = dir .. "/" .. GEN_SUBDIR + --- @type string local out_path = out_dir .. "/" .. MACS_FILENAME return out_dir, out_path end @@ -750,19 +864,24 @@ end --- @param dir string -- Absolute source directory --- @param sources SourceFile[] -- Sources contributing to this directory (for the header comment) --- @param components Component[] -- Aggregated components from all sources in this directory ---- @param counts table -- Precomputed word counts (from count_all_components) +--- @param counts table -- bag: bare component name -> word count --- @return string|nil -- Path to the written file (nil if no components) local function emit_component_macros_h(ctx, dir, sources, components, counts) if #components == 0 then return nil end + --- @type string, string local out_dir, out_path = compute_macs_h_path(dir) + --- @type string[] local lines = header_boilerplate(dir, sources) + --- @type integer, Component for _, c in ipairs(components) do + --- @type integer, string for _, l in ipairs(build_component_lines(c, counts)) do lines[#lines + 1] = l end end + --- @type string local content = table.concat(lines, "\n") .. "\n" duffle.ensure_dir(out_dir) duffle.write_file_lf(out_path, content) @@ -776,12 +895,16 @@ end --- (internal) Extend `corpus.word_counts` with this source's component macros so offsets sees them without re-reading the file. --- First declaration wins: a later caller's count is dropped (the existing entry from the first source is preserved). ---- @param corpus table -- the corpus +--- @param corpus Corpus --- @param components Component[] ---- @param counts table -- precomputed word counts (from count_all_components) +--- @param counts table -- bag: bare component name -> word count +--- @return nil local function update_canonical_word_counts(corpus, components, counts) + --- @type WordCounts local wc = corpus.word_counts + --- @type integer, Component for _, c in ipairs(components) do + --- @type string local key = "mac_" .. c.name if wc[key] == nil then wc[key] = counts[c.name] @@ -790,27 +913,33 @@ local function update_canonical_word_counts(corpus, components, counts) end --- @class ComponentDef ---- @field name string -- Bare name (without ac_/mac_ prefix) ---- @field line integer -- Definition source line (line of `MipsAtomComp_(ac_X)` / `MipsAtomComp_Proc_(ac_X, ...)`) ---- @field path string -- Absolute source path of the definition ---- @field kind string -- "comp_bare" | "comp_proc" (atom_proc is NOT a component) ---- @field debug_skip boolean -- Mirror of the scanner-owned `a.debug_skip`; consumers read this directly +--- @field name string -- Bare name (without ac_/mac_ prefix) +--- @field line integer -- Definition source line (line of `MipsAtomComp_(ac_X)` / `MipsAtomComp_Proc_(ac_X, ...)`) +--- @field path string -- Absolute source path of the definition +--- @field kind string -- "comp_bare" | "comp_proc" (atom_proc is NOT a component) +--- @field debug_skip boolean -- Mirror of the scanner-owned `a.debug_skip`; consumers read this directly +--- @field cycle_cost integer|nil -- From metadata[c.name]; nil when the body was not costed +--- @field gp0_contrib integer|nil -- From metadata[c.name]; nil when the body was not costed --- (internal) Populate `corpus.components` with this source's components-by-name map. --- First declaration wins; later declarations of the same bare name are dropped and recorded as a collision via `corpus.collisions` (kind = "component"). --- The pass does NOT write to `ctx.shared.components`. --- No parallel skip map is built here; consumers that need the per-component skip state read `corpus.components[name].debug_skip` directly. --- The `cycle_cost` + `gp0_contrib` fields are populated from `metadata[c.name]` (computed by `compute_components_metadata` against the original `MipsAtomComp_` body). ---- @param corpus table -- the corpus +--- @param corpus Corpus --- @param src SourceFile --- @param components Component[] ---- @param metadata table +--- @param metadata ComponentMetaMap +--- @return nil local function update_canonical_components(corpus, src, components, metadata) + --- @type string local rel_path = src.path:gsub("\\", "/") + --- @type integer, Component for _, c in ipairs(components) do -- Keyed by bare name (e.g. `yield`, `load_tri_indices`). -- The atoms_source_map pass looks up components by bare name from the corpus; -- `mac_` prefix lives at the call-site identifier and is stripped before lookup. + --- @type ComponentMeta|nil local m = metadata and metadata[c.name] or nil if corpus.components[c.name] == nil then corpus.components[c.name] = { @@ -825,9 +954,12 @@ local function update_canonical_components(corpus, src, components, metadata) else -- A second declaration of the same bare name: record a typed collision so static-analysis + the report can surface it. -- Identical-shape declarations (same path + line) reuse the first-wins entry without a collision record. + --- @type ComponentDef local existing = corpus.components[c.name] if existing.path ~= rel_path or existing.line ~= c.line then + --- @type string local kind = c.kind or "comp_bare" + --- @type string local first_kind = existing.kind or "comp_bare" corpus.collisions[#corpus.collisions + 1] = { kind = "component", @@ -845,12 +977,15 @@ end --- (internal) Populate `corpus.component_body_index` with this source's body index entries. --- First declaration wins; later declarations are dropped (no separate collision record: the components collision is already surfaced by `update_canonical_components`). --- The pass writes to `corpus.component_body_index` only (the corpus owns this projection). ---- @param corpus table -- the corpus +--- @param corpus Corpus --- @param src SourceFile --- @param components Component[] ---- @param scan table -- the SourceScan payload (for line_of) +--- @param scan SourceScan +--- @return nil local function update_canonical_component_body_index(corpus, src, components, scan) + --- @type (fun(pos: integer): integer)|nil local line_of = scan and scan.line_of + --- @type integer, Component for _, c in ipairs(components) do if corpus.component_body_index[c.name] == nil then corpus.component_body_index[c.name] = { @@ -869,11 +1004,15 @@ end --- @param ctx PassCtx --- @return PassResult function M.run(ctx) + --- @type MacsOutput[] local outputs = {} + --- @type PassFinding[] local errors = {} + --- @type PassFinding[] local warnings = {} -- Corpus ownership gate. + --- @type Corpus|nil local corpus = ctx.shared and ctx.shared.corpus if type(corpus) ~= "table" then error("components.run requires ctx.shared.corpus.", 0) @@ -895,14 +1034,21 @@ function M.run(ctx) -- Per-directory aggregation: every source in the same directory contributes to one `gen/macs.h`. -- The directory itself is the namespace. `corpus.sources_by_dir` preserves source-order within each bucket (matches `corpus.source_order`). + --- @type table local sources_by_dir = corpus.sources_by_dir or duffle.group_sources_by_dir(corpus.source_order) + --- @type string, SourceFile[] for dir, sources in pairs(sources_by_dir) do -- Aggregate components from every source in this directory. -- `project_components` returns nil for sources with no `MipsAtomComp_` declarations; we skip those. + --- @type Component[] local aggregated_components = {} + --- @type table local metadata_per_source = {} + --- @type integer, SourceFile for _, src in ipairs(sources) do + --- @type Component[] local per_source = project_components(src.text, src.scan) or {} + --- @type integer, Component for _, c in ipairs(per_source) do aggregated_components[#aggregated_components + 1] = c end @@ -913,13 +1059,17 @@ function M.run(ctx) if #aggregated_components > 0 then -- Compute word counts across the aggregated set. `corpus.word_counts` carries the -- same-source + prior-directory entries so the recursive lookup sees both. + --- @type table -- bag: bare name -> word count local counts = count_all_components(aggregated_components, corpus.word_counts) + --- @type string|nil local macs_path = emit_component_macros_h(ctx, dir, sources, aggregated_components, counts) if macs_path then outputs[#outputs + 1] = { macs_h = macs_path } -- Populate the projections AFTER disk emission (byte-identical `.macs.h` contract). update_canonical_word_counts(corpus, aggregated_components, counts) + --- @type integer, SourceFile for _, src in ipairs(sources) do + --- @type Component[] local per_source = project_components(src.text, src.scan) or {} if #per_source > 0 then update_canonical_components(corpus, src, per_source, metadata_per_source[src]) diff --git a/scripts/passes/dwarf_injection.lua b/scripts/passes/dwarf_injection.lua index c68bde6..4a6d761 100644 --- a/scripts/passes/dwarf_injection.lua +++ b/scripts/passes/dwarf_injection.lua @@ -33,18 +33,24 @@ -- Load `duffle_paths.lua` via `debug.getinfo(1, "S").source` (works both standalone + when require'd). -- Sets package.path + package.cpath then returns duffle. +--- @type string local _bootstrap_dir = debug.getinfo(1, "S").source:match("^@?(.*[/\\])") or "./" +--- @type DuffleExport local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua") -- ELF32 / DWARF / atoms-source-map utilities (post-link debug-info injection). -- Sister module to duffle.lua — contains the format-constant tables (ELF32 byte offsets, DWARF opcodes, etc.) and the I/O helpers -- (read_elf_sections, nm, source-map parser, LE byte r/w). `list_dir` is the general directory primitive in duffle.lua. +--- @type ElfDwarf local elf_dwarf = require("elf_dwarf") -- File-scope aliases to elf_dwarf helpers; the canonical implementations live in scripts/elf_dwarf.lua. +--- @type fun(table_bytes: string, table_start: integer): integer|nil local find_abbrev_table_end = elf_dwarf.find_abbrev_table_end -- Local DWARF opcode constants + length-prefixed integers (uleb128 + sleb128 encoders are in elf_dwarf.lua). +--- @type fun(n: integer): string local uleb128 = elf_dwarf.uleb128 +--- @type fun(n: integer): string local sleb128 = elf_dwarf.sleb128 -- ════════════════════════════════════════════════════════════════════════════ @@ -55,26 +61,42 @@ local sleb128 = elf_dwarf.sleb128 -- All values lifted from `elf_dwarf.DWARF_LINE_OPS` + `elf_dwarf.DWARF5_RNGLISTS`. -- Local aliases preserve the code's readability -- (e.g. `DW_LNS_copy` reads better than `elf_dwarf.DWARF_LINE_OPS.DW_LNS_copy` in an emitter body). +--- @type DwarfLineOps local DWARF_LINE_OPS = elf_dwarf.DWARF_LINE_OPS +--- @type Dwarf5Rnglists local DWARF5_RNGLISTS = elf_dwarf.DWARF5_RNGLISTS +--- @type integer local MIPS_BYTES_PER_WORD = elf_dwarf.MIPS_BYTES_PER_WORD +--- @type integer local DW_LNS_copy = DWARF_LINE_OPS.DW_LNS_copy +--- @type integer local DW_LNS_advance_pc = DWARF_LINE_OPS.DW_LNS_advance_pc +--- @type integer local DW_LNS_advance_line = DWARF_LINE_OPS.DW_LNS_advance_line +--- @type integer local DW_LNS_set_file = DWARF_LINE_OPS.DW_LNS_set_file +--- @type integer local DW_LNS_negate_stmt = DWARF_LINE_OPS.DW_LNS_negate_stmt +--- @type integer local DW_LNS_extended = DWARF_LINE_OPS.DW_LNS_extended +--- @type integer local DW_LNE_end_sequence = DWARF_LINE_OPS.DW_LNE_end_sequence +--- @type integer local DW_LNE_set_address = DWARF_LINE_OPS.DW_LNE_set_address +--- @type integer local DW_RLE_end_of_list = DWARF5_RNGLISTS.end_of_list +--- @type integer local DW_RLE_start_length = DWARF5_RNGLISTS.start_length -- File-index lookup for the existing main line unit (Unit 2). -- Populated at pass start by `init_file_index_lookup(elf_path)` from the runtime ELF (see `elf_dwarf.read_line_unit_file_table`). +--- @type table|nil -- bag local _file_index_by_basename = nil -- [basename] = 1-based line-table file index +--- @type table|nil -- bag local _file_path_by_index = nil -- [1-based index] = full source path (diagnostics / future consumers) +--- @type integer local _default_atom_source_index = nil -- any valid index used in opaque-row fallbacks -- RR_ debug-visible variables come from the merged register_alias_registry filtered to aliases whose code is a valid MIPS GPR 0..31 @@ -86,26 +108,38 @@ local _default_atom_source_index = nil -- any valid index used in opaque-row fa -- DW_OP_bregN would describe a memory location addressed from a register; the breg form would make gdb dereference the atom register value rather than display it. -- New abbreviation codes (100+ to avoid collision with gcc's existing 1-60+ codes). +--- @type integer local ABBREV_CU = 0x64 -- 100: DW_TAG_compile_unit +--- @type integer local ABBREV_SUBPROGRAM = 0x65 -- 101: DW_TAG_subprogram +--- @type integer local ABBREV_VARIABLE = 0x66 -- 102: DW_TAG_variable with DW_AT_type = ref4 to U4 +--- @type integer local ABBREV_STRUCT_TYPE = 0x67 -- 103: DW_TAG_structure_type with children (Binds_X mirror) +--- @type integer local ABBREV_MEMBER = 0x68 -- 104: DW_TAG_member no children (DW_AT_type = ref4 to U4 base) +--- @type integer local ABBREV_BIND_VAR = 0x69 -- 105: DW_TAG_variable no children + DW_AT_type = ref4 (the bind_args variable) +--- @type integer local ABBREV_BASE_TYPE = 0x6A -- 106: DW_TAG_base_type no children (U4) -- Component step-into (DW_TAG_inlined_subroutine + abstract DW_TAG_subprogram). +--- @type integer local ABBREV_ABSTRACT_SUBPROGRAM = 0x6B -- 107: DW_TAG_subprogram (abstract — no low_pc/high_pc); for each unique mac_X component +--- @type integer local ABBREV_INLINED_SUBROUTINE = 0x6C -- 108: DW_TAG_inlined_subroutine with children (per-component invocation range) -- Bind_args uses DW_FORM_sec_offset → .debug_loclists for PC-ranged liveness -- (each field transitions from tape memory to GPR at load_pc + 8 = MIPS I load-delay slot boundary). +--- @type integer local ABBREV_BIND_VAR_LOCLIST = 0x6D -- 109: DW_TAG_variable no children + DW_AT_type = ref4 + DW_AT_location = sec_offset -- Typed-view pointer_type (for the synthetic V4_S2* / V3_S2* / U4* / void* chains). -- MUST be a fresh abbrev code in the appended table — emitting uleb128(9) collides with GCC's existing abbrev 9 -- (a pointer_type that carries DW_AT_byte_size + DW_AT_type), so gdb misparses our 4-byte ref4 as (byte_size, type[0..2]) and lands the cursor mid-attribute. +--- @type integer local ABBREV_TYPED_VIEW_POINTER = 0x6E -- 110: DW_TAG_pointer_type no children + DW_AT_type = ref4 (typed-view / U4 / void chain) -- One row per DIE kind build_inserted_children emits. -- attrs list form + the value key filled from the atom / registry / local table. +--- @type table local DIE_SCHEMA = { base_type = { abbrev = ABBREV_BASE_TYPE, @@ -185,53 +219,87 @@ local DIE_SCHEMA = { } -- DWARF5 §7.7.3 loclist opcodes. +--- @type integer local DW_LLE_end_of_list = 0x00 +--- @type integer local DW_LLE_start_length = 0x08 +--- @type integer local DW_OP_reg0 = 0x50 -- base reg op; regN = 0x50 + N +--- @type integer local DW_OP_breg0 = 0x70 -- base breg op; bregN = 0x70 + N (SLEB offset) +--- @type integer local DW_OP_piece = 0x93 +--- @type integer local MIPS_LOAD_DELAY_BYTES = 0x08 -- 1 load word + 1 BD-slot word -- DIE children-list terminator: each DIE that has children ends with a single 0 byte (DWARF5 §7.5.3). -- This is NOT the same as DW_LLE_end_of_list despite sharing the value 0x00 — different spec sections. +--- @type integer local DIE_CHILDREN_TERMINATOR = 0x00 -- Field/piece byte sizes for the typed-view piece chains. -- All Binds_* struct fields are U4 (sizeof(uint32_t) on MIPS32 = 4 bytes). +--- @type integer local U4_BYTE_SIZE = 4 -- DWARF3/4/5 opcodes / attributes / forms (for the .debug_info synth). -- DW_TAG values are stable across DWARF3-5 per the standard's Table 7.1; -- gcc emits DW_TAG_structure_type=0x13 + DW_TAG_base_type=0x24 even in DWARF5-versioned CUs, so we match those exact byte values. +--- @type integer local DW_TAG_compile_unit = 0x11 +--- @type integer local DW_TAG_subprogram = 0x2E +--- @type integer local DW_TAG_variable = 0x34 +--- @type integer local DW_TAG_structure_type = 0x13 +--- @type integer local DW_TAG_member = 0x0D +--- @type integer local DW_TAG_base_type = 0x24 +--- @type integer local DW_TAG_pointer_type = 0x0F -- Component step-into. +--- @type integer local DW_TAG_inlined_subroutine = 0x1D +--- @type integer local DW_AT_name = 0x03 +--- @type integer local DW_AT_low_pc = 0x11 +--- @type integer local DW_AT_high_pc = 0x12 +--- @type integer local DW_AT_language = 0x13 +--- @type integer local DW_AT_location = 0x02 +--- @type integer local DW_AT_comp_dir = 0x1B +--- @type integer local DW_AT_byte_size = 0x0B +--- @type integer local DW_AT_encoding = 0x3E -- DWARF5 §7.7.1: DW_AT_encoding for the DW_ATE_unsigned base type +--- @type integer local DW_AT_data_member_location = 0x38 +--- @type integer local DW_AT_type = 0x49 +--- @type integer local DW_AT_linkage_name = 0x6E -- DWARF5 §7.7.1: DW_AT_linkage_name with DW_FORM_string +--- @type integer local DW_AT_external = 0x3F -- marks a variable/function as externally visible -- Inlined_subroutine + abstract_origin attributes. +--- @type integer local DW_AT_abstract_origin = 0x31 +--- @type integer local DW_AT_call_file = 0x58 +--- @type integer local DW_AT_call_line = 0x59 +--- @type integer local DW_AT_inline = 0x20 -- DWARF5 §7.7.1: DW_AT_inline (used by abstract subprogram for the mac_X() components) -- decl_file + decl_line on the abstract subprogram so consumers can resolve an abstract origin back to its definition site even when no inlined_subroutine instance currently maps to it. +--- @type integer local DW_AT_decl_file = 0x3A -- DWARF5 §7.7.1: DW_AT_decl_file (1-based file index into the CU's file table) +--- @type integer local DW_AT_decl_line = 0x3B -- DWARF5 §7.7.1: DW_AT_decl_line -- Replaced the hardcoded `ATOM_SOURCE_FILE_INDEX = 11` and the `PROVENANCE_BASENAME_TO_FILE_INDEX` table below with a runtime lookup @@ -244,8 +312,10 @@ local DW_AT_decl_line = 0x3B -- DWARF5 §7.7.1: DW_AT_decl_line --- The lookup uses `elf_dwarf.read_line_unit_file_table` (which parses both DWARF3 and DWARF5 line-program units — --- the crt0.s assembler-side DWARF5 unit may emit non-standard form codes for paths and is intentionally skipped). --- @param elf_path string|nil +--- @return nil local function init_file_index_lookup(elf_path) if not elf_path or elf_path == "" then return end + --- @type table|nil, table|nil, table|nil local b2i, _basenames, paths = elf_dwarf.read_line_unit_file_table(elf_path) if type(b2i) ~= "table" or type(paths) ~= "table" then io.stderr:write("[dwarf_injection] read_line_unit_file_table returned no file table for: " .. tostring(elf_path) .. "\n") @@ -255,6 +325,7 @@ local function init_file_index_lookup(elf_path) _file_path_by_index = paths -- Pick any valid index for the opaque-row fallbacks at lines 466 + 570 -- (both sites legitimately want "any file index"; gdb resolves whatever index we emit to whatever that file's line happens to be). + --- @type integer|nil for idx in pairs(paths) do _default_atom_source_index = idx break @@ -280,12 +351,16 @@ local function resolve_provenance_file_index(path) error("[dwarf_injection] resolve_provenance_file_index: empty path") end -- Normalize backslashes → forward slashes (paths arrive with mixed separators from the provenance file). + --- @type string local normalized = path:gsub("\\", "/") -- Take the last path component (the basename). + --- @type string local basename = normalized:match("([^/]+)$") or normalized + --- @type integer|nil local idx = _file_index_by_basename[basename] if idx ~= nil then return idx end -- Last-resort exact-path match (handles paths that don't reduce to a known basename). + --- @type integer, string for i, p in pairs(_file_path_by_index) do if p and p:gsub("\\", "/") == normalized then return i end end @@ -296,18 +371,28 @@ local function resolve_provenance_file_index(path) return 0 end +--- @type integer local DW_FORM_addr = 0x01 +--- @type integer local DW_FORM_data1 = 0x0B +--- @type integer local DW_FORM_string = 0x08 -- inline null-terminated +--- @type integer local DW_FORM_strp = 0x0E -- 4-byte offset into .debug_str +--- @type integer local DW_FORM_exprloc = 0x18 -- length-prefixed (ULEB128) DW_OP bytes +--- @type integer local DW_FORM_ref4 = 0x13 -- 4-byte offset within the same .debug_info CU +--- @type integer local DW_FORM_udata = 0x0F -- ULEB128 (DW_AT_byte_size for struct_type, DW_AT_data_member_location for member) +--- @type integer local DW_FORM_implicit_const = 0x21 -- DWARF5 §7.5.6: abbrev declaration carries a SLEB constant (used by the abbrev-table walker) +--- @type integer local DW_FORM_sec_offset = 0x17 -- 4-byte section-relative offset (into .debug_loclists / .debug_rnglists) -- DW_OP_reg0 + DW_OP_piece are declared above (lines 114-116) alongside the other DWARF5 §7.7.3 loclist opcodes. +--- @type integer local DW_ATE_unsigned = 0x07 -- DWARF5 §7.8.1: DW_ATE_unsigned (used for U4 base type) -- (DW_LANG_Mips_Assembler = 0x8001 was used in the, but we want this CU to look like a C TU so VSCode's Variables pane treats it as code.) @@ -327,39 +412,56 @@ local DW_ATE_unsigned = 0x07 -- DWARF5 §7.8.1: DW_ATE_unsigned (used f -- -- `tape_alias` (default: "R_TapePtr") names the wave-runtime pointer register whose value is the tape address. -- The GPR integer comes from the merged registry; absent alias fails loud — rbind atoms depend on R_TapePtr being in the registry for their piece-chain DW_OP_breg location. --- @param atom_table table[] -- list of atoms with .rbind set --- @param registries table -- merged registries from collect_per_source_registries +-- @param atom_table DwarfAtom[] -- list of atoms with .rbind set +-- @param registries DwarfRegistries -- merged registries from collect_per_source_registries -- @return string -- section bytes local function build_debug_loclists_section(atom_table, registries) registries = registries or {} -- R_TapePtr comes from the merged register_alias_registry (only present if the user opted it in via `#define atom_reg` in lottes_tape.h). -- When absent we emit just the section terminator (a single DW_LLE_end_of_list byte); the .debug_loclists section stays non-empty so the linker accepts it, -- and `bind_args` will be emitted with no loclist PC range (readelf will display it as having no .debug_loclists entries). + --- @type AliasEntry|nil local tape_alias_entry = registries.register_alias_registry and registries.register_alias_registry["R_TapePtr"] + --- @type integer local tape_reg = tape_alias_entry and tape_alias_entry.code + --- @type string[] local parts = {} + --- @type integer, DwarfAtom for _, atom in ipairs(atom_table) do if atom.rbind and tape_reg then + --- @type TypeField[] local fields = atom.rbind.fields or {} + --- @type DwarfLoadPair[] local regs = atom.rbind.regs or {} + --- @type integer local n_fields = #fields + --- @type integer local last_load_pc = atom.addr + (n_fields - 1) * MIPS_BYTES_PER_WORD + --- @type integer local transition_pc = last_load_pc + MIPS_LOAD_DELAY_BYTES + --- @type string[] local tape_pieces = {} + --- @type integer, TypeField for _, f in ipairs(fields) do + --- @type integer local offset = f.offset or 0 + --- @type string local offset_sleb = elf_dwarf.sleb128(offset) -- (DW_OP_bregN, SLEB128(offset), DW_OP_piece, ULEB128(U4_BYTE_SIZE)) -- 4 = U4_BYTE_SIZE: each piece is sizeof(uint32_t) on MIPS32. table.insert(tape_pieces, string.char(DW_OP_breg0 + tape_reg) .. offset_sleb .. string.char(DW_OP_piece) .. uleb128(U4_BYTE_SIZE)) end + --- @type string local tape_expr = table.concat(tape_pieces) + --- @type string[] local gpr_pieces = {} + --- @type integer, DwarfLoadPair for _, pair in ipairs(regs) do -- (DW_OP_regN, DW_OP_piece, ULEB128(4)) — one piece per GPR-resident field. -- The 4 = U4_BYTE_SIZE: each piece is sizeof(uint32_t) on MIPS32. table.insert(gpr_pieces, string.char(DW_OP_reg0 + pair.reg) .. string.char(DW_OP_piece) .. uleb128(U4_BYTE_SIZE)) end + --- @type string local gpr_expr = table.concat(gpr_pieces) parts[#parts + 1] = string.char(DW_LLE_start_length) .. elf_dwarf.write_u32_le(atom.addr) @@ -375,9 +477,13 @@ local function build_debug_loclists_section(atom_table, registries) -- Loclist unit header (DWARF5 §7.7.2): -- unit_length(4) + version(2) + address_size(1) + segment_size(1) + offset_entry_count(4) = 12 bytes header. -- version = 5 (DWARF5); address_size = 4 (MIPS32); segment_size = 0; offset_entry_count = 0 (we use DW_LLE_start_length, not offsets). + --- @type integer local LOCLIST_HEADER_SIZE = 12 + --- @type string local body = table.concat(parts) + --- @type integer local unit_length = LOCLIST_HEADER_SIZE - 4 + #body -- -4 because unit_length excludes itself + --- @type string local header = elf_dwarf.write_u32_le(unit_length) .. elf_dwarf.write_u16_le(5) -- DWARF5 .. string.char(U4_BYTE_SIZE) -- address_size @@ -397,29 +503,40 @@ local function tape_piece_size(offset) end -- Compute the per-atom loclist offset within a .debug_loclists section. --- @param atom_table table[] -- list of atoms with .rbind set --- @return table -- {[atom_name] = offset_in_section} +-- @param atom_table DwarfAtom[] -- list of atoms with .rbind set +-- @return table -- bag: atom name -> offset_in_section local function compute_loclists_offsets(atom_table) + --- @type integer local LOCLIST_ENTRY_HEADER_SIZE = 1 + 4 + 1 -- DW_LLE_start_length(1) + addr(4) + uleb_length(1) + --- @type table -- bag local offsets = {} -- Loclist unit header (DWARF5 §7.7.2): unit_length(4) + version(2) + address_size(1) + segment_size(1) + offset_entry_count(4) = 12 bytes. -- The unit_length itself is not counted in the unit_length value, so the body starts at byte 12. + --- @type integer local cursor = 4 + 2 + 1 + 1 + 4 -- = 12 + --- @type integer, DwarfAtom for _, atom in ipairs(atom_table) do if atom.rbind then offsets[atom.name] = cursor + --- @type integer local n_fields = #atom.rbind.fields -- Sum the actual size of each tape piece based on the field's offset (not an assumed constant); this expression mirrors what build_debug_loclists_section produces: -- 1 (DW_LLE_start_length) + 4 (PC) + 1 (uleb length prefix) + sum(tape_piece_size(field.offset)) -- + 1 (DW_LLE_start_length) + 4 (transition_pc) + 1 (uleb length prefix) + n_fields * 3 (gpr pieces) -- + 1 (DW_LLE_end_of_list) + --- @type integer local tape_pieces_size = 0 + --- @type integer, TypeField for _, f in ipairs(atom.rbind.fields or {}) do tape_pieces_size = tape_pieces_size + tape_piece_size(f.offset or 0) end + --- @type integer local gpr_pieces_size = n_fields * 3 -- each gpr piece: DW_OP_regN(1) + DW_OP_piece(1) + uleb(4)(1) = 3 bytes + --- @type integer local tape_entry = LOCLIST_ENTRY_HEADER_SIZE + tape_pieces_size + --- @type integer local gpr_entry = LOCLIST_ENTRY_HEADER_SIZE + gpr_pieces_size + --- @type integer local body_len = tape_entry + gpr_entry + 1 -- +1 for DW_LLE_end_of_list cursor = cursor + body_len end @@ -428,12 +545,15 @@ local function compute_loclists_offsets(atom_table) end -- Default name for the synthetic CU (so VSCode lists it as a known source). +--- @type string local DEFAULT_CU_NAME = "tape_atom_locals" +--- @type string local DEFAULT_CU_COMP_DIR = "." -- SECTION_WRITERS owns the .bin output path templates. -- Default basename if not provided via ctx. +--- @type string local DEFAULT_BASENAME = "hello_gte" -- ════════════════════════════════════════════════════════════════════════════ @@ -441,10 +561,128 @@ local DEFAULT_BASENAME = "hello_gte" -- ════════════════════════════════════════════════════════════════════════════ --- @class DwarfInjectionCtx ---- @field flags table -- ctx.flags; reads flags.elf_path + flags.dwarf_injection ---- @field sources table[] -- source files (reserved for future per-source state) ---- @field out_root string -- output root (e.g. "build/gen") ---- @field basename string -- input ELF basename (default "hello_gte") +--- @field flags PassFlags +--- @field sources SourceFile[] +--- @field out_root string +--- @field basename string +--- @field shared PassShared|nil + +--- DieSchema / FORM_WRITERS / atom-table records live on this file. +--- SourceFile, Path, AtomName: see duffle.lua. +--- Corpus, PassCtx, PassResult, PassFlags, PassShared, PassFinding: see ps1_meta.lua. +--- AtomEntry, AliasEntry, TypeNameEntry, TypeField, AtomInfoEntry, BindsEntry, +--- AtomViewEntry, AtomCtxEntry, AtomPhaseGroup, SourceScan, RegTypeOverride: see scan_source.lua. +--- AtomPaths, WordEvent, BodyToken: see emission_model.lua. +--- InvocationRecord: see duffle_emit.lua. +--- NmAddr: see atoms_source_map.lua. +--- ElfDwarf, DwarfLineOps, Dwarf5Rnglists: see elf_dwarf.lua. + +--- @class DieSchemaAttr +--- @field form string +--- @field key string + +--- @class DieSchema +--- @field abbrev integer +--- @field attrs DieSchemaAttr[] + +--- @class DieValues +--- @field name string|nil +--- @field byte_size integer|nil +--- @field encoding integer|nil +--- @field inline integer|nil +--- @field external integer|nil +--- @field decl_file integer|nil +--- @field decl_line integer|nil +--- @field low_pc integer|nil +--- @field high_pc integer|nil +--- @field linkage_name string|nil +--- @field location string|integer|nil +--- @field type integer|nil +--- @field data_member_location integer|nil +--- @field abstract_origin integer|nil +--- @field call_file integer|nil +--- @field call_line integer|nil + +--- @alias DieFormWriter fun(emit: fun(s: string), v: string|integer) +--- @alias DwarfSectionPathWriter fun(out_root: string, basename: string): string +--- @alias DwarfPrecedenceStep fun(r_name: string, alias_code: integer): integer|nil + +--- @class DwarfAtomWord +--- @field pos integer +--- @field line integer +--- @field text string + +--- @class DwarfLoadPair +--- @field reg integer +--- @field field string + +--- @class DwarfRbind +--- @field binds string +--- @field fields TypeField[] +--- @field bytes integer +--- @field regs DwarfLoadPair[] +--- @field info_line integer + +--- @class DwarfAtom +--- @field name string +--- @field addr integer +--- @field size_bytes integer +--- @field words integer +--- @field entries DwarfAtomWord[] +--- @field invocations InvocationRecord[] +--- @field debug_skip boolean +--- @field src_path string +--- @field rbind DwarfRbind|nil + +--- @class DwarfRegistries +--- @field register_alias_registry table +--- @field type_name_registry table +--- @field atom_views table +--- @field atom_ctxs table +--- @field atom_phases table +--- @field atom_infos AtomInfoEntry[] + +--- @class DwarfRbindStruct +--- @field bytes integer +--- @field fields TypeField[] +--- @field atom_names string[] + +--- @class DwarfComponentSite +--- @field name string +--- @field def_file string +--- @field def_line integer + +--- @class DwarfTypeLayoutMember +--- @field name string +--- @field offset integer +--- @field byte_size integer +--- @field type_name string|nil +--- @field pointer_depth integer + +--- @class DwarfTypeLayout +--- @field byte_size integer +--- @field members DwarfTypeLayoutMember[] + +--- @class DwarfEmitState +--- @field bytes string[] +--- @field next_offset integer +--- @field type_chain_offsets table -- bag: "T|depth" -> section offset +--- @field struct_section_offsets table -- bag: binds name -> section offset +--- @field abstract_offsets table -- bag: component name -> section offset +--- @field member_base_type_offsets table -- bag: "tn|size|enc" -> section offset +--- @field base_type_section_offset integer|nil + +--- @class DwarfSectionBlob +--- @field name string +--- @field data string + +--- @class DwarfInjectionPass +--- @field run fun(ctx: PassCtx): PassResult +--- @field compute_loclists_offsets_for_test fun(atom_table: DwarfAtom[]): table +--- @field build_debug_loclists_section_for_test fun(atom_table: DwarfAtom[], registries: DwarfRegistries): string +--- @field tape_piece_size_for_test fun(offset: integer): integer +--- @field build_atom_table_for_test fun(corpus: Corpus, addrs: table): DwarfAtom[] + --- Project the corpus registries into the shape the section builders expect. --- The corpus already owns the merged `register_alias_registry`, `type_name_registry`, `atom_views`, `atom_ctxs`, `atom_phases`, and `atom_infos` projections (populated by `passes.scan_source.lua`). @@ -455,8 +693,8 @@ local DEFAULT_BASENAME = "hello_gte" --- --- When two sources register the same key, the last-writer wins (later sources override earlier). --- Today only one source declares wave-context enums, so collisions are absent. ---- @param corpus table -- the corpus from `ctx.shared.corpus` ---- @return table -- { +--- @param corpus Corpus -- the corpus from `ctx.shared.corpus` +--- @return DwarfRegistries -- { --- register_alias_registry = {[R_Name] = AliasEntry}, --- type_name_registry = {[T] = TypeEntry}, --- atom_views = {[atom_name] = AtomViewEntry}, @@ -466,7 +704,9 @@ local function collect_per_source_registries(corpus) -- `passes.scan_source.lua` has already folded every per-source scan into the corpus tables, so no per-source iteration is needed here. -- `atom_infos` is preserved byte-for-byte with no filtering; consumers consult `corpus.atoms_by_name` -- themselves when they need to know whether a particular atom_info corresponds to an actual atom record. + --- @type AtomInfoEntry[] local atom_infos_list = {} + --- @type integer, AtomInfoEntry for _, ai in ipairs((corpus and corpus.atom_infos) or {}) do atom_infos_list[#atom_infos_list + 1] = ai end @@ -533,21 +773,34 @@ end --- * The previous per-word `marked_idx` ancestor walk and the GDB 12 zero-instruction-prologue duplicate row at atom entry are DELETED; the new --- first-word emission IS the entry statement. --- * Whole-atom suppression wins over component markers; no nested inversion. ---- @param atom table -- {name, addr, size_bytes, words, entries, invocations, debug_skip, word_events} +--- @param atom DwarfAtom -- {name, addr, size_bytes, words, entries, invocations, debug_skip, word_events} --- @return string local function build_atom_sequence(atom) + --- @param addr integer + --- @return string local function set_address(addr) -- Per DWARF5 §6.2.5.3: marker(0) + size(ULEB128, includes sub_opcode byte) + sub_opcode + payload -- For set_address: size = 1 (sub_opcode) + 4 (addr) = 5 + --- @type string local addr_bytes = elf_dwarf.write_u32_le(addr) + --- @type string local sub_size = string.char(DW_LNE_set_address) .. addr_bytes return string.char(DW_LNS_extended) .. uleb128(#sub_size) .. sub_size end + --- @return string local function copy_op() return string.char(DW_LNS_copy) end + --- @param file_index integer + --- @return string local function set_file(file_index) return string.char(DW_LNS_set_file) .. uleb128(file_index) end + --- @param bytes_delta integer + --- @return string local function advance_pc(bytes_delta) return string.char(DW_LNS_advance_pc) .. uleb128(bytes_delta) end + --- @param line_delta integer + --- @return string local function advance_line(line_delta) return string.char(DW_LNS_advance_line) .. sleb128(line_delta) end + --- @return string local function negate_stmt() return string.char(DW_LNS_negate_stmt) end + --- @return string local function end_sequence() -- size = 1 (just the sub_opcode byte, no payload) return string.char(DW_LNS_extended) @@ -583,19 +836,28 @@ local function build_atom_sequence(atom) -- -- `start_pos` / `end_pos` are 0-based emitted-word positions stamped at construction/close time by `duffle.emit_invoke_begin` / `duffle.emit_invoke_end`; -- Missing values are a corpus-plumbing bug, so we let the index expression fail loud with arithmetic-on-nil rather than silently producing `0+1=1` for a missing start_pos. + --- @type InvocationRecord[] local invs = atom.invocations or {} + --- @type table -- bag local innermost_idx = {} + --- @type table -- bag local ancestry_idx = {} + --- @type integer for idx = 1, #atom.entries do innermost_idx[idx] = nil ancestry_idx[idx] = {} + --- @type InvocationRecord[] local active = {} + --- @type integer, InvocationRecord for _, inv in ipairs(invs) do if idx >= inv.start_pos + 1 and idx <= inv.end_pos + 1 then active[#active + 1] = inv end end -- Sort outermost-first (widest range first); innermost is the LAST entry (narrowest range). + --- @param a InvocationRecord + --- @param b InvocationRecord + --- @return boolean table.sort(active, function(a, b) return (a.end_pos - a.start_pos) > (b.end_pos - b.start_pos) end) @@ -613,10 +875,15 @@ local function build_atom_sequence(atom) -- This is the value the multi-row PC's body_lines[1] row must reference for source-order display: `anc.body_lines[1]` is the line of the FIRST WORD -- (which for an outer whose body starts with a nested expansion is inside the inner's expansion = wrong for display purposes); -- `anc.body_first_line` is the body's first content line in the parent's source (= correct for display). + --- @type table -- bag local body_first_line_of = {} + --- @type integer, InvocationRecord for _, top_inv in ipairs(invs) do + --- @type integer local earliest_nested_call_line = nil + --- @type integer local earliest_nested_start_pos = nil + --- @type integer, InvocationRecord for _, cand in ipairs(invs) do if cand.parent_id == top_inv.id and cand.call_line ~= nil then if earliest_nested_start_pos == nil or cand.start_pos < earliest_nested_start_pos then @@ -634,16 +901,22 @@ local function build_atom_sequence(atom) end end + --- @type string[] local parts = { set_address(atom.addr), -- 7 bytes: marker + size + sub + addr; PC := atom.addr } -- Source state tracker: emits set_file + advance_line only on transitions, keeps bytes minimal. -- Both fields stay in sync with what we emit, so we duplicate no set_file and skip no state-change emit. + --- @type integer|nil local cur_file = nil -- line-state.file_idx (nil = uninitialized) + --- @type integer local cur_line = 1 -- line-state.line starts at 1 (per DWARF spec) + --- @type boolean local is_stmt = true -- main line unit default_is_stmt; every sequence ends restored + --- @param want_stmt boolean + --- @return nil local function set_statement_state(want_stmt) if is_stmt ~= want_stmt then parts[#parts + 1] = negate_stmt() @@ -653,6 +926,10 @@ local function build_atom_sequence(atom) -- Emit a state transition (set_file + advance_line + is_stmt as needed) before a row, then the row itself. -- Used for the row(s) at one entry PC. + --- @param file_idx integer + --- @param line integer + --- @param want_stmt boolean + --- @return nil local function emit_row(file_idx, line, want_stmt) if cur_file ~= file_idx then parts[#parts + 1] = set_file(file_idx) @@ -666,10 +943,13 @@ local function build_atom_sequence(atom) parts[#parts + 1] = copy_op() end + --- @type integer local call_file_idx = resolve_provenance_file_index(atom.src_path) -- --- Atom entry (idx 1) ------------------------------------------------- + --- @type DwarfAtomWord local entry_1 = atom.entries[1] + --- @type InvocationRecord[] local entry_1_ancestry = ancestry_idx[1] -- If atom entry 1 starts inside an invocation, walk the ancestry and emit a call-site row + (when applicable) @@ -689,11 +969,13 @@ local function build_atom_sequence(atom) -- (statement iff unmarked; suppressed for marked outermost). -- The body_lines[1] row references body_first_line_of[anc.id] (= the body's first content line in the parent's source), -- NOT anc.body_lines[1] (= the line of the first WORD, which is wrong when the outer's body starts with a nested call). + --- @type integer, InvocationRecord for ai, anc in ipairs(entry_1_ancestry) do assert(anc.body_lines, "missing body_lines: emitter did not run emission-model") assert(anc.body_lines[1] ~= nil, "dwarf_injection: body_lines[1] missing on first-word entry for inv=" .. tostring(anc.component_name)) assert(anc.call_path and anc.call_path ~= "", "dwarf_injection: inv.call_path is missing on invocation " .. tostring(anc.component_name) .. "; emitter did not run emission-model.") emit_row(resolve_provenance_file_index(anc.call_path), anc.call_line, true) + --- @type boolean local is_outermost = (ai == 1) if not (is_outermost and anc.debug_skip) then emit_row(resolve_provenance_file_index(anc.def_path), body_first_line_of[anc.id] or anc.body_lines[1], not anc.debug_skip) @@ -702,8 +984,11 @@ local function build_atom_sequence(atom) end -- --- Subsequent entries (idx 2..N) -------------------------------------- + --- @type integer|nil for idx = 2, #atom.entries do + --- @type DwarfAtomWord local entry = atom.entries[idx] + --- @type InvocationRecord local inv = innermost_idx[idx] -- Advance PC by 1 .word (4 bytes on MIPS). @@ -722,12 +1007,15 @@ local function build_atom_sequence(atom) -- NOT anc.body_lines[1] (= the line of the first WORD, which is wrong when the outer's body starts with a nested call: -- gdb 12.1 picks the displayed line as the LAST row at the same PC in byte-stream order, -- so the disc=1 row's value matters for what's shown when stepping into the nested case). + --- @type InvocationRecord[] local ancestry = ancestry_idx[idx] + --- @type integer, InvocationRecord for ai, anc in ipairs(ancestry) do assert(anc.body_lines, "missing body_lines: emitter did not run emission-model") assert(anc.body_lines[1] ~= nil, string.format("missing body_lines[1] for inv=%s start_pos=%d len=%d", anc.component_name, anc.start_pos, #(anc.body_lines or {}))) assert(anc.call_path and anc.call_path ~= "", "dwarf_injection: inv.call_path is missing on invocation " .. tostring(anc.component_name) .. "; emitter did not run emission-model.") emit_row(resolve_provenance_file_index(anc.call_path), anc.call_line, true) + --- @type boolean local is_outermost = (ai == 1) if not (is_outermost and anc.debug_skip) then emit_row(resolve_provenance_file_index(anc.def_path), body_first_line_of[anc.id] or anc.body_lines[1], not anc.debug_skip) @@ -741,6 +1029,7 @@ local function build_atom_sequence(atom) -- The previous `want = not marked_idx[idx]` (which suppressed ALL body rows when any ancestor was marked) is replaced by the per-invocation predicate. -- Marked invocations emit non-statement body rows at every body word; unmarked invocations emit statement body rows. assert(inv.body_lines, "missing body_lines: emitter did not run emission-model") + --- @type integer local words_into = idx - inv.start_pos assert(inv.body_lines[words_into] ~= nil, string.format("missing body_lines[%d] for inv=%s start_pos=%d len=%d idx=%d", words_into, inv.component_name, inv.start_pos, #(inv.body_lines or {}), idx)) emit_row(resolve_provenance_file_index(inv.def_path), inv.body_lines[words_into], not inv.debug_skip) @@ -783,27 +1072,38 @@ end --- `{comp_name, call_file, call_line, comp_file, comp_line, start_pos, end_pos, body_lines, debug_skip}`. `body_lines[k]` --- is the k-th word's source line within the component body. --- ---- @param corpus table -- From `ctx.shared.corpus` ---- @param addrs table -- ELF symbols keyed by atom name from `elf_dwarf.read_nm` ---- @return table[] -- List of {name, addr, size_bytes, words, entries, invocations, debug_skip?} +--- @param corpus Corpus -- From `ctx.shared.corpus` +--- @param addrs table -- ELF symbols keyed by atom name from `elf_dwarf.read_nm` +--- @return DwarfAtom[] -- List of {name, addr, size_bytes, words, entries, invocations, debug_skip?} local function build_atom_table(corpus, addrs) -- Cross-ref: keep only atoms present in BOTH the nm symbol table AND `corpus.atoms_by_name`. Output is sorted by ascending addr. + --- @type table -- bag local atoms_by_name = corpus.atoms_by_name or {} -- Per-atom ingest. Returns nil if the atom is absent from the corpus; the caller skips it via the `if atom then ...` guard. -- `src_path` is the absolute source path that declared this atom; the build_atom_table iteration below threads `src.path` through. -- This is consumed by `build_atom_sequence::set_file(...)` for opaque-row fallbacks + raw-word rows (atoms where no invocation ancestry exists). + --- @param name string + --- @param info NmAddr + --- @param src_path string + --- @return DwarfAtom|nil local function ingest_atom(name, info, src_path) + --- @type AtomEntry|nil local atom_record = atoms_by_name[name] if not atom_record then return nil end + --- @type AtomPaths local paths = atom_record.paths or {} + --- @type WordEvent[] local word_events = paths.word_events or {} + --- @type InvocationRecord[] local invocations_proj = paths.invocations or {} -- Build the dense entries list from `word_events`. -- `word_events[i].i` = the 0-based `.word` position -- `call_line` = the root atom's physical source line for that word (stamped by emission_model) + --- @type DwarfAtomWord[] local entries = {} + --- @type integer, WordEvent for idx, ev in ipairs(word_events) do entries[#entries + 1] = { pos = ev.i or (idx - 1), @@ -812,6 +1112,7 @@ local function build_atom_table(corpus, addrs) } end -- Whole-atom skip is read from the atom declaration record; the scanner owns it, no parallel lookup table. + --- @type DwarfAtom local atom = { name = name, addr = info[1], @@ -830,6 +1131,7 @@ local function build_atom_table(corpus, addrs) -- (which differs by the count of `invoke_begin`/`invoke_end`/marker items between this call and the previous one). -- Using `start_word` would shift every `words_into` lookup by the marker count and break the call-site + body row pairing at the first word of every invocation. atom.invocations = invocations_proj + --- @type integer, InvocationRecord for _, inv in ipairs(atom.invocations) do -- `debug_skip` flag is already stamped by `duffle.emit_invoke_begin` from `corpus.components[name].debug_skip`. -- Normalize to boolean for downstream dispatch. A missing value is a corpus-plumbing bug; the fail-loud error was raised at the construction site. @@ -842,38 +1144,53 @@ local function build_atom_table(corpus, addrs) return atom end + --- @type DwarfAtom[] local out = {} -- Walk every source's atom list (which preserves source order + per-source src_path). -- Cross-ref with the nm symbol table; atoms absent from `addrs` are skipped -- (an atom declared in source but not emitted as a symbol is a metaprogram or atom-info bug, not a source-correlation bug — emit_no_emit would catch it upstream). + --- @type integer, SourceFile for _, src in ipairs((corpus and corpus.source_order) or {}) do + --- @type string local src_path = src.path or "" + --- @type integer, AtomEntry for _, atom_rec in ipairs(((src.scan or {}).atoms) or {}) do + --- @type NmAddr|nil local info = addrs[atom_rec.name or atom_rec.raw_name] if info then + --- @type DwarfAtom|nil local atom = ingest_atom(atom_rec.name or atom_rec.raw_name, info, src_path) if atom then out[#out + 1] = atom end end end + --- @type integer, AtomEntry for _, atom_rec in ipairs(((src.scan or {}).raw_atoms) or {}) do + --- @type NmAddr|nil local info = addrs[atom_rec.name or atom_rec.raw_name] if info then + --- @type DwarfAtom|nil local atom = ingest_atom(atom_rec.name or atom_rec.raw_name, info, src_path) if atom then out[#out + 1] = atom end end end end + --- @param a DwarfAtom + --- @param b DwarfAtom + --- @return boolean table.sort(out, function(a, b) return a.addr < b.addr end) return out end --- Compute the set of distinct components invoked across all atoms. --- Returns `{name -> {kind, def_file, def_line}}` keyed by component name (e.g. `yield`, `gte_load_tri_verts`). ---- @param atom_table table[] ---- @return table +--- @param atom_table DwarfAtom[] +--- @return table local function collect_component_defs(atom_table) + --- @type table -- bag local out = {} + --- @type integer, DwarfAtom for _, atom in ipairs(atom_table) do + --- @type integer, InvocationRecord for _, inv in ipairs(atom.invocations or {}) do if not out[inv.component_name] then out[inv.component_name] = { @@ -914,29 +1231,40 @@ end --- This is intentional: silently falling back to a hardcoded GPR would mask the missing opt-in. --- --- Pre-tokenized: `body_tokens` is the scan-source pass's pre-split list of top-level statements (each entry is a single `load_*` call or other statement). ---- @param body_tokens table[] -- The atom's pre-tokenized body statements (from atom.body_tokens) +--- @param body_tokens BodyToken[] -- The atom's pre-tokenized body statements (from atom.body_tokens) --- @param binds_name string -- Expected Binds_X name (skip pairs with mismatching binds) ---- @param registries table -- Merged registries from collect_per_source_registries ---- @return table[] -- List of {reg = , field = } +--- @param registries DwarfRegistries -- Merged registries from collect_per_source_registries +--- @return DwarfLoadPair[] -- List of {reg = , field = } local function parse_body_load_pairs(body_tokens, binds_name, registries) + --- @type DwarfLoadPair[] local pairs = {} + --- @type table -- bag local reg_index_by_name = (registries and registries.register_alias_registry) or {} -- One regex that matches any of: load_word, load_half, load_half_u, load_byte, load_byte_u, gte_lw, gte_lwc2. -- The captured ident is `kind`; `inner` holds the parens body for arg parsing. + --- @type string local load_pattern = "^(load_word|load_half|load_half_u|load_byte|load_byte_u|gte_lw|gte_lwc2)%s*%((.*)%)$" + --- @type integer, BodyToken for _, t in ipairs(body_tokens or {}) do + --- @type string local tok = duffle.trim(t.tok or "") + --- @type string|nil, string|nil local kind, inner = tok:match(load_pattern) if kind then + --- @type string[] local args = duffle.split_top_level_commas(inner) -- Expected shape for an rbind piece-chain load: (R_, R_TapePtr, O_(Binds_, FieldName)) -- The second arg MUST be R_TapePtr — loads from other bases (e.g. `load_byte_u(R_RawStatus, R_PadRaw, 0)`) -- are field-derivative loads that read already-bound tape values; they're NOT a new piece-chain. if #args >= 3 and duffle.trim(args[2]) == "R_TapePtr" then + --- @type string local reg_name = duffle.trim(args[1]) + --- @type string local third_arg = duffle.trim(args[3]) -- Match O_(Binds_, FieldName) + --- @type string|nil, string|nil local b, f = third_arg:match("^O_%((Binds_[%w_]+)%s*,%s*(.-)%s*%)$") + --- @type AliasEntry|nil local alias_entry = reg_index_by_name[reg_name] if b and b == binds_name and alias_entry and alias_entry.code then pairs[#pairs + 1] = { @@ -960,28 +1288,35 @@ end --- The piece chain uses (DW_OP_regN, DW_OP_piece, ULEB128(field_size)). --- --- Binds fields come from `scan.binds`; the per-source `scan.binds[i].fields` already carries the typed-field record after the scan-source generalization. ---- @param corpus table -- From `ctx.shared.corpus` ---- @param atom_table table[] -- Cross-ref'd atom table from build_atom_table ---- @param registries table -- Merged registries from collect_per_source_registries ---- @return table, table -- (rbind_atoms, rbind_structs) +--- @param corpus Corpus -- From `ctx.shared.corpus` +--- @param atom_table DwarfAtom[] -- Cross-ref'd atom table from build_atom_table +--- @param registries DwarfRegistries -- Merged registries from collect_per_source_registries +--- @return table, table local function parse_rbind_atoms(corpus, atom_table, registries) registries = registries or {} + --- @type table -- bag local rbind_atoms = {} + --- @type table -- bag local rbind_structs = {} -- Index binds by struct name; consume `scan.binds[i].fields` directly. -- The scan-source pass emits each Binds_X's fields as {[type_name, pointer_depth, offset, byte_size, ...]}, -- so this pass builds the rbind_structs entry without re-parsing. + --- @type table -- bag local binds_by_name = {} + --- @type integer, SourceFile for _, src in ipairs((corpus and corpus.source_order) or {}) do + --- @type SourceScan|nil local scan = src.scan if scan then + --- @type integer, BindsEntry for _, b in ipairs(scan.binds or {}) do binds_by_name[b.name] = b end end end + --- @type string, BindsEntry for binds_name, b in pairs(binds_by_name) do if b.fields and b.bytes then rbind_structs[binds_name] = { @@ -993,31 +1328,43 @@ local function parse_rbind_atoms(corpus, atom_table, registries) end -- Walk every atom_info; if `binds` is set, find the atom body_tokens + parse load_word pairs. + --- @type table -- bag local body_tokens_by_atom = {} + --- @type integer, SourceFile for _, src in ipairs((corpus and corpus.source_order) or {}) do + --- @type SourceScan|nil local scan = src.scan if scan then + --- @type integer, AtomEntry for _, atom in ipairs(scan.atoms or {}) do body_tokens_by_atom[atom.name] = atom.body_tokens end end end + --- @type table -- bag local ai_by_atom = {} + --- @type integer, SourceFile for _, src in ipairs((corpus and corpus.source_order) or {}) do + --- @type SourceScan|nil local scan = src.scan if scan then + --- @type integer, AtomInfoEntry for _, ai in ipairs(scan.atom_infos or {}) do ai_by_atom[ai.atom_name] = ai end end end + --- @type string, AtomInfoEntry for atom_name, ai in pairs(ai_by_atom) do if ai.binds then + --- @type DwarfRbindStruct|nil local struct = rbind_structs[ai.binds] + --- @type BodyToken[]|nil local body_toks = body_tokens_by_atom[atom_name] if struct and body_toks then + --- @type DwarfLoadPair[] local pairs = parse_body_load_pairs(body_toks, ai.binds, registries) if #pairs > 0 then rbind_atoms[atom_name] = { @@ -1034,6 +1381,7 @@ local function parse_rbind_atoms(corpus, atom_table, registries) end -- Mark rbind atoms in the main atom_table. + --- @type integer, DwarfAtom for _, atom in ipairs(atom_table) do if rbind_atoms[atom.name] then atom.rbind = rbind_atoms[atom.name] @@ -1056,23 +1404,29 @@ end --- The existing final unit already contains hello_gte_tape.c as file index 11 and ends with a valid end_sequence. --- We preserve its bytes, append independent atom sequences, and increase only that unit's DWARF32 unit_length. --- @param existing string -- existing section bytes, byte-for-byte ---- @param atom_table table -- list of {name, addr, size_bytes, words, entries} +--- @param atom_table DwarfAtom[] -- list of {name, addr, size_bytes, words, entries} --- @return string local function build_dwarf_line_section(existing, atom_table) if #atom_table == 0 then return existing end -- Build the sequences. + --- @type string[] local sequences = {} + --- @type integer, DwarfAtom for _, atom in ipairs(atom_table) do sequences[#sequences + 1] = build_atom_sequence(atom) end + --- @type string local appended = table.concat(sequences) -- Walk DWARF32 line units and retain the final unit's bounds. -- The main C CU points at this final unit (DW_AT_stmt_list = 0x5b in today's ELF). + --- @type integer, integer|nil, integer|nil, integer|nil local unit_pos, last_pos, last_length, last_end = 0, nil, nil, nil while unit_pos < #existing do if unit_pos + 4 > #existing then return existing end + --- @type integer local unit_length = elf_dwarf.read_u32_le(existing, unit_pos) if unit_length == elf_dwarf.dw_dwarf32_terminator then return existing end + --- @type integer local unit_end_excl = unit_pos + 4 + unit_length if unit_end_excl > #existing then return existing end last_pos, last_length, last_end = unit_pos, unit_length, unit_end_excl @@ -1080,7 +1434,9 @@ local function build_dwarf_line_section(existing, atom_table) end if unit_pos ~= #existing or not last_pos then return existing end + --- @type integer local new_length = last_length + #appended + --- @type string local new_length_bytes = elf_dwarf.write_u32_le(new_length) return existing:sub(1, last_pos) @@ -1102,7 +1458,7 @@ end --- [entries...] -- address(4) + length(4) per entry --- terminator -- address=0 + length=0 (8 zero bytes) --- @param existing string ---- @param atom_table table +--- @param atom_table DwarfAtom[] --- @return string local function build_dwarf_aranges_section(existing, atom_table) if #existing < 12 then @@ -1126,12 +1482,16 @@ local function build_dwarf_aranges_section(existing, atom_table) -- Walk all units and emit each one (preserving existing structure). -- For the LAST unit, replace the terminator with my entries + new term. + --- @type string[] local result = {} + --- @type integer local i = 0 -- zero-based wire offset + --- @type boolean local is_last_unit = false while i < #existing do -- Read this unit's length. + --- @type integer local ul = elf_dwarf.read_u32_le(existing, i) if ul == elf_dwarf.dw_dwarf32_terminator then -- DWARF64 marker - not supported. @@ -1139,21 +1499,29 @@ local function build_dwarf_aranges_section(existing, atom_table) return existing end + --- @type integer local unit_start = i + --- @type integer local unit_end_excl = i + 4 + ul is_last_unit = (unit_end_excl == #existing) if is_last_unit then -- The old terminator is replaced by entries + a new terminator, so net section growth (and unit_length growth) is entries only. + --- @type integer local added_bytes = #atom_table * elf_dwarf.DWARF4_ARANGES.entry_size + --- @type integer local new_ul = ul + added_bytes + --- @type string local new_ul_bytes = elf_dwarf.write_u32_le(new_ul) -- Emit everything EXCEPT the last 8 bytes (terminator). result[#result + 1] = new_ul_bytes .. existing:sub(i + 5, unit_end_excl - elf_dwarf.DWARF4_ARANGES.terminator_size) -- Append my atom entries. + --- @type integer, DwarfAtom for _, atom in ipairs(atom_table) do + --- @type integer local a = atom.addr + --- @type integer local size = atom.size_bytes result[#result + 1] = elf_dwarf.write_u32_le(a) .. elf_dwarf.write_u32_le(size) end @@ -1186,15 +1554,20 @@ end --- unit_length(4), version=5(2), address_size=4(1), segment_size=0(1), --- offset_entry_count=0(4), start_length entries..., end_of_list(1). --- @param existing string ---- @param atom_table table +--- @param atom_table DwarfAtom[] --- @return string local function build_dwarf_rnglists_section(existing, atom_table) if #existing <= elf_dwarf.DWARF5_RNGLISTS.first_entry_offset or #atom_table == 0 then return existing end + --- @type integer local unit_length = elf_dwarf.read_u32_le(existing, elf_dwarf.DWARF5_RNGLISTS.unit_length_offset) + --- @type integer local version = elf_dwarf.read_u16_le(existing, elf_dwarf.DWARF5_RNGLISTS.version_offset) + --- @type integer local address_size = existing:byte(elf_dwarf.DWARF5_RNGLISTS.addr_size_offset + 1) + --- @type integer local segment_size = existing:byte(elf_dwarf.DWARF5_RNGLISTS.seg_size_offset + 1) + --- @type integer local offset_entry_count = elf_dwarf.read_u32_le(existing, elf_dwarf.DWARF5_RNGLISTS.offset_count_offset) if unit_length + 4 ~= #existing @@ -1206,14 +1579,19 @@ local function build_dwarf_rnglists_section(existing, atom_table) return existing end + --- @type string[] local entries = {} + --- @type integer, DwarfAtom for _, atom in ipairs(atom_table) do entries[#entries + 1] = string.char(DW_RLE_start_length) .. elf_dwarf.write_u32_le(atom.addr) .. uleb128(atom.size_bytes) end + --- @type string local appended = table.concat(entries) + --- @type integer local new_length = unit_length + #appended + --- @type string local new_length_bytes = elf_dwarf.write_u32_le(new_length) return new_length_bytes @@ -1241,22 +1619,31 @@ end --- --- Each piece's size = byte offset of next field - byte offset of this field (or struct.byte_size for the last piece). --- Since all fields are U4/pointer (4 bytes each) in the current source, each piece is 4 bytes. ---- @param rbind table -- {regs = {{reg, field}, ...}, fields = {{name, offset}, ...}, bytes = N} +--- @param rbind DwarfRbind -- {regs = {{reg, field}, ...}, fields = {{name, offset}, ...}, bytes = N} --- @return string -- the exprloc byte sequence (length-prefixed) local function piece_chain_exprloc(rbind) + --- @type string[] local op_bytes = {} + --- @type table -- bag local field_offset_by_name = {} + --- @type integer, TypeField for _, f in ipairs(rbind.fields) do field_offset_by_name[f.name] = f.offset end + --- @type integer local next_offset = rbind.bytes + --- @type integer for i = #rbind.regs, 1, -1 do -- walk backwards to know each piece's size + --- @type DwarfLoadPair local pair = rbind.regs[i] + --- @type integer local off = field_offset_by_name[pair.field] or 0 + --- @type integer local size if i == #rbind.regs then size = next_offset - off else + --- @type integer local next_off = field_offset_by_name[rbind.regs[i + 1].field] if not next_off then -- Defensive: a load_word references a field not in the Binds_X struct. @@ -1270,8 +1657,11 @@ local function piece_chain_exprloc(rbind) next_offset = off end -- We built it back-to-front; reverse it. + --- @type string[] local rev = {} + --- @type integer for i = #op_bytes, 1, -1 do rev[#rev + 1] = op_bytes[i] end + --- @type string local op = table.concat(rev) return uleb128(#op) .. op end @@ -1294,9 +1684,13 @@ end -- "math.floor(/16) instead of bit.rshift for LuaJIT 2.1 compat" comment at line 352). -- DWARF5 compile-unit header constants. +--- @type integer local DW_VERSION_5 = 5 +--- @type integer local DW_UT_compile = 0x01 +--- @type integer local DWARF32_TERMINATOR = 0xFFFFFFFF -- sentinel for DWARF64 marker +--- @type integer local CU_HEADER_SIZE = 12 -- 4 + 2 + 1 + 1 + 4 --- Walk .debug_info to find the FINAL compilation unit, validate it as a DWARF5 32-bit compile-unit, and extract its bounds + abbrev-table offset. @@ -1318,15 +1712,21 @@ local CU_HEADER_SIZE = 12 -- 4 + 2 + 1 + 1 + 4 --- @param existing string -- the .debug_info section bytes --- @return integer|nil, integer|nil, integer|nil local function find_main_cu_layout(existing) + --- @type integer local buf_len = #existing if buf_len < CU_HEADER_SIZE then return nil end + --- @type integer local pos = 0 + --- @type integer local main_cu_start = nil + --- @type integer local main_cu_end_excl = nil while pos + 4 <= buf_len do + --- @type integer local unit_length = elf_dwarf.read_u32_le(existing, pos) if unit_length == DWARF32_TERMINATOR then return nil end + --- @type integer local unit_end_excl = pos + 4 + unit_length if unit_end_excl > buf_len then return nil end main_cu_start = pos @@ -1343,16 +1743,22 @@ local function find_main_cu_layout(existing) -- [6] unit_type -- [7] address_size -- [8..11] debug_abbrev_offset + --- @type integer local hdr = main_cu_start + 4 + --- @type integer local version = elf_dwarf.read_u16_le(existing, hdr) + --- @type integer local unit_type = existing:byte(hdr + 2 + 1) + --- @type integer local address_size = existing:byte(hdr + 3 + 1) + --- @type integer local abbrev_off = elf_dwarf.read_u32_le(existing, hdr + 4) if version ~= DW_VERSION_5 or unit_type ~= DW_UT_compile or address_size ~= 4 then return nil end -- Final byte of the main CU must be the root children-terminator (0). + --- @type integer local final_pos = main_cu_end_excl - 1 if final_pos >= buf_len or existing:byte(final_pos + 1) ~= 0 then return nil end @@ -1405,8 +1811,17 @@ end --- which is a valid (extended) abbrev table. --- @return string local function build_new_abbrev() + --- @param name string + --- @param form integer + --- @return string local function attr(name, form) return uleb128(name) .. uleb128(form) end + --- @param code integer + --- @param tag integer + --- @param has_children boolean + --- @param attrs string + --- @return string local function abbrev(code, tag, has_children, attrs) + --- @type integer local children = has_children and 0x01 or 0x00 -- DW_CHILDREN_yes / no return uleb128(code) .. uleb128(tag) @@ -1415,17 +1830,20 @@ local function build_new_abbrev() .. string.char(0x00, 0x00) -- end of attr list (2 zeros) end + --- @type integer local abbrev_cu = abbrev(ABBREV_CU, DW_TAG_compile_unit, true, -- DW_CHILDREN_yes attr( DW_AT_name, DW_FORM_strp) .. attr(DW_AT_comp_dir, DW_FORM_strp) .. attr(DW_AT_language, DW_FORM_data1)) + --- @type integer local abbrev_subprogram = abbrev(ABBREV_SUBPROGRAM, DW_TAG_subprogram, true, -- DW_CHILDREN_yes attr( DW_AT_name, DW_FORM_string) .. attr(DW_AT_low_pc, DW_FORM_addr) .. attr(DW_AT_high_pc, DW_FORM_addr) .. attr(DW_AT_linkage_name, DW_FORM_string)) -- equals DW_AT_name; gdb resolves the subprogram, not the gcc global array + --- @type integer local abbrev_variable = abbrev(ABBREV_VARIABLE, DW_TAG_variable, false, -- DW_CHILDREN_no attr( DW_AT_name, DW_FORM_string) .. attr(DW_AT_location, DW_FORM_exprloc) @@ -1435,20 +1853,24 @@ local function build_new_abbrev() -- rbind composite. -- DW_FORM_udata (0x0F, ULEB128) is declared at module scope. For small values (struct byte_size, member offsets) 1 byte is enough; -- we emit ULEB128 anyway for spec compliance with the DWARF abbrev encoding rules. + --- @type integer local abbrev_struct_type = abbrev(ABBREV_STRUCT_TYPE, DW_TAG_structure_type, true, -- DW_CHILDREN_yes attr( DW_AT_name, DW_FORM_string) .. attr(DW_AT_byte_size, DW_FORM_udata)) + --- @type integer local abbrev_member = abbrev(ABBREV_MEMBER, DW_TAG_member, false, -- DW_CHILDREN_no attr( DW_AT_name, DW_FORM_string) .. attr(DW_AT_data_member_location, DW_FORM_udata) .. attr(DW_AT_type, DW_FORM_ref4)) + --- @type integer local abbrev_bind_var = abbrev(ABBREV_BIND_VAR, DW_TAG_variable, false, -- DW_CHILDREN_no attr( DW_AT_name, DW_FORM_string) .. attr(DW_AT_location, DW_FORM_exprloc) .. attr(DW_AT_type, DW_FORM_ref4)) + --- @type integer local abbrev_base_type = abbrev(ABBREV_BASE_TYPE, DW_TAG_base_type, false, -- DW_CHILDREN_no attr( DW_AT_name, DW_FORM_string) .. attr(DW_AT_byte_size, DW_FORM_data1) @@ -1457,6 +1879,7 @@ local function build_new_abbrev() -- Abstract subprograms carry DW_AT_decl_file and DW_AT_decl_line for definition-site resolution, -- even when no inlined_subroutine instance currently maps to it. -- DW_FORM_udata is consistent with the call_file/call_line forms on abbrev 108. + --- @type integer local abbrev_abstract_subprogram = abbrev(ABBREV_ABSTRACT_SUBPROGRAM, DW_TAG_subprogram, false, -- DW_CHILDREN_no attr( DW_AT_name, DW_FORM_string) .. attr(DW_AT_inline, DW_FORM_data1) @@ -1464,6 +1887,7 @@ local function build_new_abbrev() .. attr(DW_AT_decl_file, DW_FORM_udata) .. attr(DW_AT_decl_line, DW_FORM_udata)) + --- @type integer local abbrev_inlined_subroutine = abbrev(ABBREV_INLINED_SUBROUTINE, DW_TAG_inlined_subroutine, false, -- DW_CHILDREN_no (emits no per-inlined-instance children; the PC range IS the inlining scope) attr( DW_AT_abstract_origin, DW_FORM_ref4) .. attr(DW_AT_low_pc, DW_FORM_addr) @@ -1474,6 +1898,7 @@ local function build_new_abbrev() -- bind_args with DW_FORM_sec_offset → .debug_loclists. -- The .debug_loclists section holds a sequence of DW_LLE entries; -- the first matching entry for a PC describes each field's value (tape memory DW_OP_breg24 or register DW_OP_regN). + --- @type integer local abbrev_bind_var_loclists = abbrev(ABBREV_BIND_VAR_LOCLIST, DW_TAG_variable, false, -- DW_CHILDREN_no attr( DW_AT_name, DW_FORM_string) .. attr(DW_AT_location, DW_FORM_sec_offset) @@ -1482,6 +1907,7 @@ local function build_new_abbrev() -- Typed-view pointer_type: DW_TAG_pointer_type, DW_AT_type = ref4 (no DW_AT_byte_size; the -- chain target carries the byte size via the base_type or struct_type we point at). -- MUST be in the appended table — see ABBREV_TYPED_VIEW_POINTER above. + --- @type integer local abbrev_typed_view_pointer = abbrev(ABBREV_TYPED_VIEW_POINTER, DW_TAG_pointer_type, false, -- DW_CHILDREN_no attr(DW_AT_type, DW_FORM_ref4)) @@ -1510,28 +1936,35 @@ end -- The CU name + comp_dir + one entry per RR_ debug-visible alias all go into one new blob. -- Register names are sourced from the merged registry filtered to MIPS GPR 0..31 -- (the same filter that build_inserted_children applies for the RR_ locals, so .debug_str entries stay in sync with .debug_info). --- @param atom_table table[] -- list of {name, addr, size_bytes, ...} --- @param registries table -- merged registries from collect_per_source_registries --- @return string, table -- (new_strings_blob, map_of_name_to_offset_in_new_blob) +--- @param atom_table DwarfAtom[] -- list of {name, addr, size_bytes, ...} +--- @param registries DwarfRegistries +--- @return string +--- @return table -- bag: name -> offset in the new blob local function build_new_strings(atom_table, registries) registries = registries or {} -- The CU name + comp_dir are the first two strings (offsets 0 and N1). -- Then each unique atom name + each register name follows. + --- @type string[] local strings = {} + --- @type table -- bag local map = {} -- CU name at offset 0 in the new blob strings[#strings + 1] = DEFAULT_CU_NAME .. "\0" map["__cu_name__"] = 0 + --- @type integer local cu_name_len = #strings[#strings] -- comp_dir at offset cu_name_len strings[#strings + 1] = DEFAULT_CU_COMP_DIR .. "\0" map["__comp_dir__"] = cu_name_len + --- @type integer local comp_dir_len = #strings[#strings] -- Atom names (one per unique atom) + --- @type integer, DwarfAtom for _, atom in ipairs(atom_table) do + --- @type string local name = atom.name if not map[name] then map[name] = #table.concat(strings) @@ -1543,14 +1976,18 @@ local function build_new_strings(atom_table, registries) -- filtered to MIPS GPR 0..31 — the same filter that build_inserted_children applies -- for the RR_ locals, so .debug_str entries stay in sync with .debug_info). -- Lua's pairs() is non-deterministic; sort the alias names first so the emitted .debug_str bytes are byte-identical across runs. + --- @type string[] local sorted_alias_names = {} + --- @type string, AliasEntry|nil for r_name, alias in pairs(registries.register_alias_registry or {}) do if alias.code and alias.code >= 0 and alias.code <= 31 then sorted_alias_names[#sorted_alias_names + 1] = r_name end end table.sort(sorted_alias_names) + --- @type integer, string for _, r_name in ipairs(sorted_alias_names) do + --- @type string local rr_name = "RR_" .. strip_r_prefix(r_name) if not map[rr_name] then map[rr_name] = #table.concat(strings) @@ -1605,10 +2042,10 @@ end --- Per-die section offsets are tracked via the running `next_offset` cursor (= section offset of the NEXT byte to emit). --- @param main_cu_offset integer -- 0-based section offset of the main CU's unit_length field --- @param main_cu_end_excl integer -- 0-based section offset of the first byte AFTER the main CU ---- @param atom_table table[] -- atoms (with atom.rbind set if rbind; atom.invocations set if mac_X(...) calls) ---- @param rbind_structs table -- {[binds_name] = {bytes, fields, atom_names}} ---- @param loclists_offsets table -- {[atom_name] = section-relative offset into the new .debug_loclists} ---- @param registries table -- merged registries from collect_per_source_registries +--- @param atom_table DwarfAtom[] -- atoms (with atom.rbind set if rbind; atom.invocations set if mac_X(...) calls) +--- @param rbind_structs table -- {[binds_name] = {bytes, fields, atom_names}} +--- @param loclists_offsets table -- {[atom_name] = section-relative offset into the new .debug_loclists} +--- @param registries DwarfRegistries -- merged registries from collect_per_source_registries --- @return string -- bytes to splice into the main CU just before its root terminator local function build_inserted_children(main_cu_offset, main_cu_end_excl, atom_table, rbind_structs, loclists_offsets, registries) rbind_structs = rbind_structs or {} @@ -1620,33 +2057,44 @@ local function build_inserted_children(main_cu_offset, main_cu_end_excl, atom_ta -- by_alias_order: sorted list of by_alias keys, for deterministic iteration order. -- Lua's pairs() order is implementation-defined and varies between runs; without sorting, the per-atom variable emission order -- would be non-deterministic and the .debug_info bytes would differ across builds. + --- @type table -- bag local by_alias = {} + --- @type string, AliasEntry|nil for r_name, alias in pairs(registries.register_alias_registry or {}) do if alias.code and alias.code >= 0 and alias.code <= 31 then by_alias[r_name] = alias end end + --- @type string[] local by_alias_order = {} + --- @type string for r_name in pairs(by_alias) do by_alias_order[#by_alias_order + 1] = r_name end table.sort(by_alias_order) -- Build a name->atom lookup for fast rbind_atom resolution during the per-atom phase/ctx propagation. -- Cheap (O(atom_table)) and built once. + --- @param atoms DwarfAtom[] + --- @return table local function build_atom_name_index(atoms) + --- @type table -- bag local m = {} + --- @type integer, AliasEntry|nil for _, a in ipairs(atoms or {}) do if a and a.name then m[a.name] = a end end return m end + --- @type table -- bag local atom_by_name_global = build_atom_name_index(atom_table) -- We insert IMMEDIATELY BEFORE the main CU's root children-terminator (the last byte of the main CU). -- The first emitted byte lives at section offset (main_cu_end_excl - 1). + --- @type integer local insertion_start = main_cu_end_excl - 1 -- Closure state: bytes + next_offset + the typed-view/structure/abstract offset caches. -- All step-emitters mutate this state in place. + --- @type DwarfEmitState local S = { bytes = {}, next_offset = insertion_start, -- 0-based section offset of the NEXT byte to emit @@ -1657,30 +2105,66 @@ local function build_inserted_children(main_cu_offset, main_cu_end_excl, atom_ta base_type_section_offset = nil, -- set by emit_unsigned_int_base_type } + --- @param s string + --- @return nil local function emit(s) S.bytes[#S.bytes + 1] = s S.next_offset = S.next_offset + #s end + --- @type table local FORM_WRITERS = { + --- @param emit fun(s: string) + --- @param v string|integer + --- @return nil string = function(emit, v) emit(v .. "\0") end, + --- @param emit fun(s: string) + --- @param v string|integer + --- @return nil data1 = function(emit, v) emit(string.char(v)) end, + --- @param emit fun(s: string) + --- @param v string|integer + --- @return nil udata = function(emit, v) emit(uleb128(v)) end, + --- @param emit fun(s: string) + --- @param v string|integer + --- @return nil addr = function(emit, v) emit(elf_dwarf.write_u32_le(v)) end, + --- @param emit fun(s: string) + --- @param v string|integer + --- @return nil ref4 = function(emit, v) emit(elf_dwarf.write_u32_le(v)) end, + --- @param emit fun(s: string) + --- @param v string|integer + --- @return nil sec_offset = function(emit, v) emit(elf_dwarf.write_u32_le(v)) end, + --- @param emit fun(s: string) + --- @param v string|integer + --- @return nil data2 = function(emit, v) emit(elf_dwarf.write_u16_le(v)) end, + --- @param emit fun(s: string) + --- @param v string|integer + --- @return nil exprloc = function(emit, v) emit(v) end, } + --- @param schema_name string + --- @param values DieValues + --- @return nil local function emit_die(schema_name, values) + --- @type DieSchema local row = DIE_SCHEMA[schema_name] emit(uleb128(row.abbrev)) + --- @type integer, DieSchemaAttr for _, attr in ipairs(row.attrs) do + --- @type string|integer local v = values[attr.key] + --- @type DieFormWriter local w = FORM_WRITERS[attr.form] if not w then error("emit_die: unknown form " .. tostring(attr.form)) end w(emit, v) end end + --- @param section_offset integer + --- @return integer local function ref4_of(section_offset) if section_offset == nil then return 0 end return section_offset - main_cu_offset @@ -1688,6 +2172,7 @@ local function build_inserted_children(main_cu_offset, main_cu_end_excl, atom_ta -- 1) Emit the base_type DIE first (member ref4s reference it). + --- @type integer local base_type_section_offset = S.next_offset emit_die("base_type", { name = "unsigned int", @@ -1696,6 +2181,7 @@ local function build_inserted_children(main_cu_offset, main_cu_end_excl, atom_ta }) -- The function body below reads S.next_offset directly via the `next_offset` function; -- this keeps offsets synchronized with emitted data. + --- @return integer local function next_offset() return S.next_offset end -- Typed local views. @@ -1705,15 +2191,20 @@ local function build_inserted_children(main_cu_offset, main_cu_end_excl, atom_ta -- For a field declared "V4_S2*" (depth=1), the chain is: -- V4_S2 (typedef, references base_type 4-byte unsigned) + ptr_to_V4_S2_1 (pointer_type, byte_size 4, refs the typedef) -- gdb walks: variable type = ptr_to_V4_S2_1 → V4_S2 → base_type, and displays "V4_S2 *" (the typedef's name + the pointer depth). + --- @type table -- bag local type_offsets = {} -- {[type_name] = section_offset for the typedef DIE} -- Always include the base_type "unsigned int" as the U4 target. type_offsets["U4"] = base_type_section_offset -- Collect every unique (type_name, max_pointer_depth) used by any rbind field. + --- @type table -- bag local used_typed_views = {} -- { [type_name] = max_depth } + --- @type integer, DwarfAtom for _, atom in ipairs(atom_table) do if atom.rbind and atom.rbind.fields then + --- @type integer, TypeField for _, f in ipairs(atom.rbind.fields) do if f.type_name and f.pointer_depth and f.pointer_depth > 0 then + --- @type integer local depth = used_typed_views[f.type_name] or 0 if f.pointer_depth > depth then used_typed_views[f.type_name] = f.pointer_depth @@ -1723,7 +2214,9 @@ local function build_inserted_children(main_cu_offset, main_cu_end_excl, atom_ta end end -- Sort for deterministic emission. + --- @type string[] local sorted_typed_types = {} + --- @type string for tn in pairs(used_typed_views) do sorted_typed_types[#sorted_typed_types + 1] = tn end table.sort(sorted_typed_types) -- For each non-U4 type, emit a typedef (DW_TAG_typedef) named after the type and referencing the base_type "unsigned int" (4 bytes). @@ -1740,14 +2233,20 @@ local function build_inserted_children(main_cu_offset, main_cu_end_excl, atom_ta -- Layout for V* / Rect_* / Reg_* / Slice / … comes from corpus.type_name_registry. -- scan_source parses Struct_() in math.h, math.atom.h, memory.h and fills field offset + byte_size. -- U1–U4 / S1–S4 / B1–B4 stay authored fundamentals (BUILTIN_BYTE_SIZES + base_type DIEs below). + --- @type table -- bag local type_reg = registries.type_name_registry or {} + --- @param tn string + --- @return DwarfTypeLayout|nil local function layout_from_registry(tn) + --- @type TypeNameEntry|nil local entry = type_reg[tn] if not entry or entry.kind ~= "struct" or not entry.fields or #entry.fields == 0 then return nil end if entry.byte_size == nil then return nil end + --- @type DwarfTypeLayoutMember[] local members = {} + --- @type integer, TypeField for _, f in ipairs(entry.fields) do if f.offset == nil or f.byte_size == nil then return nil end members[#members + 1] = { @@ -1760,19 +2259,30 @@ local function build_inserted_children(main_cu_offset, main_cu_end_excl, atom_ta end return { byte_size = entry.byte_size, members = members } end + --- @param tn string + --- @return integer local function encoding_for_type(tn) if type(tn) == "string" and tn:match("^S[124]$") then return 5 end return 7 end + --- @type integer local S2_TYPE_BYTE_SIZE = 2 + --- @type integer local S4_TYPE_BYTE_SIZE = 4 -- Pre-emit the signed base types (S2, S4) once if any typed view's member needs them. -- We emit per-typed-view lazily below; build a member-base-type offset cache (idempotent). + --- @type table -- bag local member_base_type_offsets = {} + --- @param tn string + --- @param byte_size integer + --- @param encoding integer + --- @return integer local function ensure_member_base_type(tn, byte_size, encoding) + --- @type string local key = tn .. "|" .. byte_size .. "|" .. encoding if member_base_type_offsets[key] then return member_base_type_offsets[key] end + --- @type integer local off = next_offset() emit_die("base_type", { name = tn, @@ -1787,24 +2297,33 @@ local function build_inserted_children(main_cu_offset, main_cu_end_excl, atom_ta ensure_member_base_type("S2", S2_TYPE_BYTE_SIZE, 5) ensure_member_base_type("S4", S4_TYPE_BYTE_SIZE, 5) + --- @type table -- bag local type_chain_offsets = {} + --- @type table -- bag local struct_die_offsets = {} + --- @param tn string + --- @return integer|nil local function emit_struct_layout(tn) if struct_die_offsets[tn] then return struct_die_offsets[tn] end + --- @type DwarfTypeLayout|nil local type_info = layout_from_registry(tn) if not type_info then return nil end + --- @type integer, DwarfTypeLayoutMember for _, m in ipairs(type_info.members) do if (m.pointer_depth or 0) == 0 and layout_from_registry(m.type_name) then emit_struct_layout(m.type_name) end end + --- @type integer local struct_offset = next_offset() struct_die_offsets[tn] = struct_offset emit_die("structure_type", { name = tn, byte_size = type_info.byte_size, }) + --- @type integer, DwarfTypeLayoutMember for _, m in ipairs(type_info.members) do + --- @type integer local member_type_off if (m.pointer_depth or 0) > 0 then member_type_off = type_chain_offsets[m.type_name .. "|" .. m.pointer_depth] @@ -1827,21 +2346,27 @@ local function build_inserted_children(main_cu_offset, main_cu_end_excl, atom_ta emit(string.char(DIE_CHILDREN_TERMINATOR)) return struct_offset end + --- @type integer, string for _, tn in ipairs(sorted_typed_types) do if tn ~= "U4" then + --- @type integer local depth = used_typed_views[tn] + --- @type integer local struct_offset = emit_struct_layout(tn) if not struct_offset then + --- @type integer local innermost_offset = next_offset() emit_die("base_type", { name = tn, byte_size = U4_BYTE_SIZE, encoding = DW_ATE_unsigned, }) + --- @type integer local outermost_offset = next_offset() emit_die("pointer_type", { type = ref4_of(innermost_offset) }) type_chain_offsets[tn .. "|" .. depth] = outermost_offset elseif depth == 1 then + --- @type integer local outermost_offset = next_offset() emit_die("pointer_type", { type = ref4_of(struct_offset) }) type_chain_offsets[tn .. "|" .. depth] = outermost_offset @@ -1858,6 +2383,7 @@ local function build_inserted_children(main_cu_offset, main_cu_end_excl, atom_ta -- The void base_type is emitted BEFORE any other typed chain so its ref4 pointer remains stable. -- Follow the SAME pattern as the typed-views chain above: capture the offset BEFORE the uleb tag -- (this is the ref4 target), emit the DIE bytes, then emit the pointer_type pointing at the offset. + --- @type integer local void_chain_offset = next_offset() emit_die("base_type", { name = "void", @@ -1870,6 +2396,7 @@ local function build_inserted_children(main_cu_offset, main_cu_end_excl, atom_ta -- The variable's DW_AT_type must reference the pointer_type, not the base_type. Patch below. -- Capture the pointer_type's offset (the last-thing-emitted DIE start) and overwrite the lookup. -- The pointer_type was emitted as: uleb(9) (1 byte) + 4-byte ref4 = 5 bytes. Its tag byte is at void_chain_offset + 8 (the base_type's 8 bytes: 1 tag + 5 name + 1 byte_size + 1 encoding). + --- @type integer local ptr_void_offset = void_chain_offset + 8 type_chain_offsets["void|1"] = ptr_void_offset @@ -1881,15 +2408,21 @@ local function build_inserted_children(main_cu_offset, main_cu_end_excl, atom_ta -- Once this chain is registered as `type_chain_offsets["U4|1"]`, step (e) of the per-RR_ precedence chain will resolve `atom_type(U4 *)` -- declarations on aliases like `R_PrimCursor` and `R_OtBase` to `U4 *` (gdb renders as `(unsigned int *)` with the value displayed in hex). emit_die("pointer_type", { type = ref4_of(base_type_section_offset) }) + --- @type integer local u4_chain_offset = next_offset() - 5 -- 1 (uleb tag) + 4 (ref4) = 5 bytes; capture the pointer_type's start offset type_chain_offsets["U4|1"] = u4_chain_offset -- 2) Emit one DW_TAG_structure_type per unique Binds_X. + --- @type table -- bag local struct_section_offsets = {} + --- @type string[] local sorted_struct_names = {} + --- @type string for k in pairs(rbind_structs) do sorted_struct_names[#sorted_struct_names + 1] = k end table.sort(sorted_struct_names) + --- @type integer, string for _, binds_name in ipairs(sorted_struct_names) do + --- @type DwarfRbindStruct local struct = rbind_structs[binds_name] struct_section_offsets[binds_name] = next_offset() @@ -1898,7 +2431,9 @@ local function build_inserted_children(main_cu_offset, main_cu_end_excl, atom_ta byte_size = struct.bytes, }) + --- @type integer, TypeField for _, field in ipairs(struct.fields) do + --- @type integer local field_type_offset if field.pointer_depth and field.pointer_depth > 0 then field_type_offset = type_chain_offsets[field.type_name .. "|" .. field.pointer_depth] @@ -1920,14 +2455,21 @@ local function build_inserted_children(main_cu_offset, main_cu_end_excl, atom_ta -- Each abstract DIE is a CU-level child (sibling of the per-atom subprograms below). -- The abstract DIE's section offset is later used by inlined_subroutine DIEs (which embed `DW_AT_abstract_origin = ref4 → abstract DIE`). -- Each abstract DIE also carries DW_AT_decl_file + DW_AT_decl_line pointing at the component's definition site (file path + body line). + --- @type table -- bag local component_defs = collect_component_defs(atom_table) + --- @type table -- bag local abstract_offsets = {} -- name -> section offset + --- @type string[] local sorted_comp_names = {} + --- @type string for name in pairs(component_defs) do sorted_comp_names[#sorted_comp_names + 1] = name end table.sort(sorted_comp_names) -- DW_INL_inlined (1) = "this subroutine was inlined" — accurate for the mac_* components. + --- @type integer local DW_INL_inlined = 0x01 + --- @type integer, string for _, comp_name in ipairs(sorted_comp_names) do + --- @type DwarfComponentSite local def = component_defs[comp_name] abstract_offsets[comp_name] = next_offset() emit_die("abstract_subprogram", { @@ -1943,6 +2485,7 @@ local function build_inserted_children(main_cu_offset, main_cu_end_excl, atom_ta -- Subprogram names match the written C ident (the ELF symbol). -- The gcc global `[]` is a DW_TAG_variable without children; our subprogram has the wave-context var children. -- gdb's symbol resolution picks our subprogram (it has low_pc/high_pc + children) over the gcc global for function-context lookups. + --- @type integer, DwarfAtom for _, atom in ipairs(atom_table) do emit_die("subprogram", { name = atom.name, @@ -1960,42 +2503,57 @@ local function build_inserted_children(main_cu_offset, main_cu_end_excl, atom_ta -- (e) enum-site atom_type() default: register_alias_registry[R_Name].default_type (per-alias fallback declared in lottes_tape.h) -- (f) void* fallback: the void_chain_offset built in section 1b; gdb renders `(void *) 0x...` (hex) -- An R_Name absent from the registry AND missed by all of (a..e) skips emission for that alias entirely. + --- @type table|nil -- bag local atom_view_ctx_fields = nil -- populated by step (b); map field_name -> field entry + --- @type table|nil -- bag local reg_to_field_ctx = nil -- populated by step (b); map GPR index -> field name + --- @type table|nil -- bag local atom_view_phase_fields = nil -- populated by step (d); map field_name -> field entry + --- @type table|nil -- bag local reg_to_field_phase = nil -- populated by step (d); map GPR index -> field name -- atom-name -> atom lookup is precomputed once as atom_by_name_global. + --- @type table -- bag local field_type_by_name = {} if atom.rbind and atom.rbind.fields then + --- @type integer, TypeField for _, f in ipairs(atom.rbind.fields) do if f.type_name then field_type_by_name[f.name] = f end end end + --- @type table -- bag local reg_to_field = {} if atom.rbind and atom.rbind.regs then + --- @type integer, DwarfLoadPair for _, pair in ipairs(atom.rbind.regs) do reg_to_field[pair.reg] = pair.field end end + --- @type AtomViewEntry|nil local atom_view = (registries.atom_views or {})[atom.name] -- step (b) inputs: this atom's `atom_ctx()` (resolved from the registries' atom_ctxs) + --- @type AtomCtxEntry|nil local this_ctx = registries.atom_ctxs and registries.atom_ctxs[atom.name] if this_ctx and this_ctx.rbind_atom then + --- @type DwarfAtom|nil local rbind = atom_by_name_global[this_ctx.rbind_atom] if rbind and rbind.rbind and rbind.rbind.fields then atom_view_ctx_fields = {} + --- @type integer, TypeField for _, f in ipairs(rbind.rbind.fields) do atom_view_ctx_fields[f.name] = f end if rbind.rbind.regs then reg_to_field_ctx = {} + --- @type integer, DwarfLoadPair for _, pair in ipairs(rbind.rbind.regs) do reg_to_field_ctx[pair.reg] = pair.field end end end end -- step (d) inputs: this atom's `atom_phase(