From fa598a41c632e5d4d69ba3bce8bf4978a70e97f7 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Fri, 10 Jul 2026 17:53:13 -0400 Subject: [PATCH] readability pass on word_count_eval.lua --- scripts/passes/static_analysis.lua | 78 ++++++++++- scripts/word_count_eval.lua | 202 ++++++++++++++++------------- 2 files changed, 186 insertions(+), 94 deletions(-) diff --git a/scripts/passes/static_analysis.lua b/scripts/passes/static_analysis.lua index c8e4f8f..d69eb93 100644 --- a/scripts/passes/static_analysis.lua +++ b/scripts/passes/static_analysis.lua @@ -1231,10 +1231,20 @@ local function validate(ctx, src) warnings[#warnings + 1] = { line = f.line, msg = f.msg } end end - info[#info + 1] = { - line = 0, - msg = string.format("scanned: %d atom bodies; %d findings", #atoms, #findings), - } + -- Per-source "scanned:" summary line. Includes the source basename + -- for traceability (the old format was just "scanned: N atom + -- bodies; M findings" which is unidentifiable when the module has + -- multiple sources). Sources with 0 atoms (pure-header files like + -- dsl.h, mips.h, etc.) are SKIPPED — the per-module header already + -- lists them in the "Sources:" section, and emitting a noisy + -- "0 atom bodies" line per header is just clutter. + if #atoms > 0 or #findings > 0 then + info[#info + 1] = { + line = 0, + msg = string.format("scanned: %s: %d atom bodies; %d findings", + src.basename, #atoms, #findings), + } + end -- Phase 3: cycle-budget summary line. Per-path min/max totals. if #atoms > 0 then @@ -1476,9 +1486,63 @@ local function emit_module_static_analysis_txt(ctx, dir, dir_sources, atoms, fin end add("") - add("── Info ────────────────────────────────────────────────") - for _, i_ in ipairs(info) do - add(string.format(" %s", i_.msg)) + add("── Per-source scan summary ──────────────────────────────") + -- One line per source that contributed atoms. The line includes + -- the source basename + per-source atom count + (if path-aware + -- cycle data is present) the min..max cycle range. Sources with + -- 0 atoms are skipped (they're just header files that declared + -- no MipsAtom_ — they're already listed in the module's + -- "Sources:" section above). + -- + -- TODO: per-source finding attribution. Currently we can't tell + -- which source a given error/warning came from (errors/warnings + -- only carry atom-name + line, not source-path). The per-atom + -- cycle section already shows which atoms are in which source + -- via the `(file_basename)` suffix. Adding source attribution to + -- error/warning would be a future enhancement. + for _, src in ipairs(dir_sources) do + local src_atoms = {} + for _, a in ipairs(atoms) do + if a.source_path == src.path then + src_atoms[#src_atoms + 1] = a + end + end + if #src_atoms == 0 then + goto continue + end + local atom_count = #src_atoms + local mn, mx = math.huge, -1 + for _, a in ipairs(src_atoms) do + local p = a.paths or {} + if (p.cycles_min or 0) < mn then mn = p.cycles_min or 0 end + if (p.cycles_max or 0) > mx then mx = p.cycles_max or 0 end + end + local path_str + if mx > 0 then + path_str = string.format(" cycles=%d..%d", mn, mx) + else + path_str = string.format(" %d cycles", mn) + end + add(string.format(" %-30s %d atom%s%s", + src.basename, atom_count, + atom_count == 1 and "" or "s", + path_str)) + ::continue:: + end + + -- Module-level findings summary (across all sources). + local total_errs = #errors + local total_warns = #warnings + add("") + add(string.format("Module findings: %d error(s), %d warning(s)", total_errs, total_warns)) + + -- Per-source "scanned:" info lines (each line includes the source + -- basename for traceability). + if #info > 0 then + add("") + for _, i_ in ipairs(info) do + add(string.format(" %s", i_.msg)) + end end write_file(out_path, table.concat(lines, "\n") .. "\n") diff --git a/scripts/word_count_eval.lua b/scripts/word_count_eval.lua index 07eb3ca..975f04e 100644 --- a/scripts/word_count_eval.lua +++ b/scripts/word_count_eval.lua @@ -1,39 +1,53 @@ --- word_count_eval.lua --- --- Word-counting logic for the tape-atom metaprogram pipeline. --- Used by: --- - passes/components.lua (compute_component_word_count) --- - passes/offsets.lua (scan_atom_body) --- - passes/annotation.lua (TAPE_WORDS <-> WORD_COUNT drift check) --- --- This module ALSO exposes M.run(ctx) — the "word-counts" pass entry in --- the PASSES table — which loads metadata.h + scans for existing --- *.macs.h files into ctx.shared.word_counts. --- --- Coding standard: tabs (1/level), EmmyLua annotations, no regex, --- Lua 5.3 compatible. +--- word_count_eval.lua — Word-counting logic for the tape-atom metaprogram +--- pipeline. +--- +--- Three responsibilities: +--- 1. **Public utilities** (used by `passes/components.lua`, +--- `passes/offsets.lua`, `passes/annotation.lua`): +--- - `M.count_token_words(token, wc)` — words emitted by one token +--- - `M.scan_dir(dir, suffix)` — glob walk for *.macs.h +--- - `M.count_body_words(body, wc)` — words emitted by an atom body +--- 2. **Pass entry** `M.run(ctx)` — loads metadata.h + *.macs.h into +--- `ctx.shared.word_counts` for downstream passes. +--- 3. **Internal helpers** for the body scanner. +--- +--- **Conventions**: tabs (1/level), EmmyLua annotations, no regex, +--- Lua 5.3 compatible. See +--- `C:\projects\Pikuma\ps1-ai\conductor\code_styleguides\lua.md`. -- ════════════════════════════════════════════════════════════════════════════ -- Module-scope requires + package.path setup -- ════════════════════════════════════════════════════════════════════════════ +-- Resolve `arg[0]` to an absolute-ish script directory so that +-- `require("duffle")` resolves against `scripts/` regardless of CWD. +-- Note: this boilerplate is duplicated in 6 other entry scripts; a +-- Phase-6 extraction target (`duffle.setup_package_path()`). local script_path = arg and arg[0] or "?" local last_sep = 0 -for i = 1, #script_path do - local c = script_path:sub(i, i) - if c == "/" or c == "\\" then last_sep = i end +for sep_pos = 1, #script_path do + local ch = script_path:sub(sep_pos, sep_pos) + if ch == "/" or ch == "\\" then last_sep = sep_pos end end local script_dir = last_sep == 0 and "./" or script_path:sub(1, last_sep) package.path = script_dir .. "?.lua;" .. script_dir .. "?/init.lua;" .. package.path package.cpath = "C:\\projects\\Pikuma\\ps1\\toolchain\\luajit-2.1\\lib\\lua\\5.1\\?.dll;" .. package.cpath local duffle = require("duffle") -local trim = duffle.trim -local read_ident = duffle.read_ident -local skip_ws_and_cmt = duffle.skip_ws_and_cmt -local load_word_counts = duffle.load_word_counts -local split_top_level_commas = duffle.split_top_level_commas -local is_space = duffle.is_space + +-- ════════════════════════════════════════════════════════════════════════════ +-- Constants +-- ════════════════════════════════════════════════════════════════════════════ + +-- Windows separator chars — used to convert `dir /b /s` output (which uses +-- `\`) into POSIX paths (which our scripts expect). +local PATH_SEP_BACKSLASH = "\\" +local PATH_SEP_FORWARD = "/" + +-- Glob command for Windows directory walk. `dir /b /s` lists all matching +-- files recursively with bare paths (no headers); `2>nul` discards the +-- "file not found" stderr when nothing matches. +local DIR_GLOB_CMD = 'dir /b /s "%s\\%s" 2>nul' -- ════════════════════════════════════════════════════════════════════════════ -- Type declarations @@ -42,28 +56,28 @@ local is_space = duffle.is_space --- @class WordCounts --- @field [string] integer -- macro name -> word count +--- @class SourceFile +--- @field path string -- absolute path to the source file +--- @field text string -- the full source text +--- @field dir string -- the directory containing the source +--- @field basename string -- filename without extension + --- @class PassCtx ---- @field sources SourceFile[] ---- @field metadata_path string ---- @field shared table ---- @field shared.word_counts WordCounts ---- @field out_root string ---- @field project_root string ---- @field upstream table ---- @field flags table ---- @field dry_run boolean ---- @field verbose boolean +--- @field sources SourceFile[] -- all source files in the build +--- @field metadata_path string -- path to word_count.metadata.h +--- @field shared table -- cross-pass shared state +--- @field shared.word_counts WordCounts -- populated by this pass +--- @field out_root string -- output root (e.g. "build/gen") +--- @field project_root string -- project root (e.g. "code/") +--- @field upstream table -- per-pass upstream outputs +--- @field flags table -- CLI flags +--- @field dry_run boolean -- if true, compute but don't write +--- @field verbose boolean -- if true, log diagnostic info --- @class PassResult ---- @field outputs table[] ---- @field errors table[] ---- @field warnings table[] - ---- @class SourceFile ---- @field path string ---- @field text string ---- @field dir string ---- @field basename string +--- @field outputs table[] -- {kind=, path=} entries describing emit files +--- @field errors table[] -- {line=, msg=} entries; build-stops +--- @field warnings table[] -- {line=, msg=} entries; build-succeeds -- ════════════════════════════════════════════════════════════════════════════ -- Module exports @@ -85,13 +99,13 @@ local M = {} --- @param wc WordCounts -- the shared word-count table --- @return integer function M.count_token_words(token, wc) - local s = trim(token) + local s = duffle.trim(token) if s == "" then return 0 end - local name, after = read_ident(s, 1) + local name, after = duffle.read_ident(s, 1) if not name then return 1 end if wc[name] then return wc[name] end - local j = skip_ws_and_cmt(s, after) - if s:sub(j, j) == "(" then + local paren_pos = duffle.skip_ws_and_cmt(s, after) + if s:sub(paren_pos, paren_pos) == "(" then io.stderr:write(" warning: unknown macro '" .. name .. "', assuming 1 word\n") end return 1 @@ -110,13 +124,13 @@ end --- @return string[] function M.scan_dir(dir, suffix) local results = {} - local p = io.popen('dir /b /s "' .. dir .. '\\' .. suffix .. '" 2>nul') - if not p then return results end - for raw_line in p:lines() do - local path = raw_line:gsub("\\", "/") + local pipe = io.popen(DIR_GLOB_CMD:format(dir, suffix)) + if not pipe then return results end + for raw_line in pipe:lines() do + local path = raw_line:gsub(PATH_SEP_BACKSLASH, PATH_SEP_FORWARD) results[#results + 1] = path end - p:close() + pipe:close() return results end @@ -134,13 +148,16 @@ end --- @param wc WordCounts -- the shared word-count table --- @return integer -- total words function M.count_body_words(body, wc) - local pos = 0 - for _, tok in ipairs(split_top_level_commas(body)) do - local k = 1 - local tlen = #tok - while k <= tlen and is_space(tok:sub(k, k)) do k = k + 1 end - local leading_ident = read_ident(tok, k) - if leading_ident == "atom_label" or leading_ident == "atom_offset" then + local total = 0 + for _, tok in ipairs(duffle.split_top_level_commas(body)) do + local pos = 1 + local tok_len = #tok + while pos <= tok_len and duffle.is_space(tok:sub(pos, pos)) do + pos = pos + 1 + end + local leading_ident = duffle.read_ident(tok, pos) + local is_marker = leading_ident == "atom_label" or leading_ident == "atom_offset" + if is_marker then -- Marker call: record at current pos, do NOT advance pos. -- But the source pattern may bundle the marker with the next -- instruction on a new line (no top-level comma between them). @@ -148,17 +165,16 @@ function M.count_body_words(body, wc) -- a real instruction that must still be counted. local marker_end = M.find_marker_call_end(tok) if marker_end > 0 and marker_end < #tok then - local rest = trim(tok:sub(marker_end + 1)) + local rest = duffle.trim(tok:sub(marker_end + 1)) if rest ~= "" then - local rest_words = M.count_token_words(rest, wc) - pos = pos + rest_words + total = total + M.count_token_words(rest, wc) end end else - pos = pos + M.count_token_words(tok, wc) + total = total + M.count_token_words(tok, wc) end end - return pos + return total end --- Find the end position (just past the closing ')') of the first @@ -168,34 +184,42 @@ end --- @param tok string --- @return integer -- 0 if no marker call found function M.find_marker_call_end(tok) - local i = 1 - local len = #tok - while i <= len do - i = skip_ws_and_cmt(tok, i) - if i > len then break end - local c = tok:sub(i, i) - if is_space(c) then - i = i + 1 - elseif c == "/" then + local pos = 1 + local tok_len = #tok + while pos <= tok_len do + pos = duffle.skip_ws_and_cmt(tok, pos) + if pos > tok_len then break end + local ch = tok:sub(pos, pos) + if duffle.is_space(ch) then + pos = pos + 1 + elseif ch == "/" then -- comment — skip past it (delegated to duffle.skip_str_or_cmt) - local nx = duffle.skip_str_or_cmt(tok, i) - if nx > i then i = nx else i = i + 1 end + local nx = duffle.skip_str_or_cmt(tok, pos) + pos = (nx > pos) and nx or (pos + 1) else - local ident, after = read_ident(tok, i) - if ident == "atom_label" or ident == "atom_offset" then - local j = skip_ws_and_cmt(tok, after) - if tok:sub(j, j) == "(" then - local _, end_paren = duffle.read_parens(tok, j) - return end_paren - 1 - end - return 0 - end - i = after or (i + 1) + local ident, after_ident = duffle.read_ident(tok, pos) + local marker_end = find_marker_end(tok, ident, after_ident) + if marker_end > 0 then return marker_end end + pos = after_ident or (pos + 1) end end return 0 end +-- (internal) If `ident` is `atom_label`/`atom_offset` followed by `(...)`, +-- return the position just past the closing ')'. Otherwise 0. +-- @param tok string +-- @param ident string|nil +-- @param after_ident integer +-- @return integer +local function find_marker_end(tok, ident, after_ident) + if ident ~= "atom_label" and ident ~= "atom_offset" then return 0 end + local open_paren = duffle.skip_ws_and_cmt(tok, after_ident) + if tok:sub(open_paren, open_paren) ~= "(" then return 0 end + local _, end_paren = duffle.read_parens(tok, open_paren) + return end_paren - 1 +end + -- ┌────────────────────────────────────────────────────────────────────┐ -- │ Pass entry: M.run(ctx) — "word-counts" pass │ -- └────────────────────────────────────────────────────────────────────┘ @@ -211,14 +235,18 @@ function M.run(ctx) local wc = {} -- 1. Load metadata.h (the encoding-macro source of truth). - local meta_counts = load_word_counts(ctx.metadata_path) + local meta_counts = duffle.load_word_counts(ctx.metadata_path) for name, count in pairs(meta_counts) do wc[name] = count end -- 2. Scan project_root recursively for *.macs.h files (component-macro source). local macs_files = M.scan_dir(ctx.project_root, "*.macs.h") for _, macs_path in ipairs(macs_files) do - local ok, mc = pcall(load_word_counts, macs_path) - if ok and type(mc) == "table" then + local ok, mc = pcall(duffle.load_word_counts, macs_path) + if not ok then + io.stderr:write(string.format("[word_count_eval] parse error in '%s': %s\n", macs_path, tostring(mc))) + elseif type(mc) ~= "table" then + io.stderr:write(string.format("[word_count_eval] '%s' did not return a table (got %s)\n", macs_path, type(mc))) + else for name, count in pairs(mc) do wc[name] = count end end end