From e662d175ab9b9841029e4bcc81a9416a387143b4 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sat, 11 Jul 2026 16:47:09 -0400 Subject: [PATCH] lifting tokenize_body, using lfs package --- .gitignore | 2 + scripts/duffle.lua | 100 +++++++++++++++++++++++++++++ scripts/duffle_paths.lua | 5 +- scripts/passes/annotation.lua | 3 +- scripts/passes/components.lua | 12 ++-- scripts/passes/offsets.lua | 12 +++- scripts/passes/report.lua | 2 +- scripts/passes/scan_source.lua | 10 +++ scripts/passes/static_analysis.lua | 96 ++------------------------- scripts/passes/word_count_eval.lua | 64 +++++++++++------- scripts/ps1_meta.lua | 6 ++ scripts/update_deps.ps1 | 15 +++++ 12 files changed, 201 insertions(+), 126 deletions(-) diff --git a/.gitignore b/.gitignore index 86dc9a0..6957337 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,5 @@ toolchain/PSn00bSDK *.a .sentry-native .vscode/settings.json +toolchain/lfs +toolchain/lpeg diff --git a/scripts/duffle.lua b/scripts/duffle.lua index 826a8b3..a3e93a1 100644 --- a/scripts/duffle.lua +++ b/scripts/duffle.lua @@ -549,6 +549,106 @@ function M.split_top_level_commas(body) return tokens end +-- ════════════════════════════════════════════════════════════════════════════ +-- Section 4b: tokenize_body + build_body_line_index (shared, memoized) +-- ════════════════════════════════════════════════════════════════════════════ + +-- Moved here from passes/static_analysis.lua so all passes can share the memoized +-- per-body tokenization. The memoization key is the body string (immutable per pass). + +local _tokenize_body_cache = {} +local _body_line_index_cache = {} + +--- Tokenize the body inner-text into a flat list of `{tok, rel}` pairs. +--- `tok` is the trimmed token string; `rel` is the byte offset within `body`. +--- Memoized on the body string — first call pays O(body_len), subsequent calls return cached. +--- @param body string +--- @return table[] -- {{tok=string, rel=integer}, ...} +function M.tokenize_body(body) + if _tokenize_body_cache[body] ~= nil then return _tokenize_body_cache[body] end + local out = {} + local len = #body + local rel = 1 + while rel <= len do + local ws_end = M.skip_ws_and_cmt(body, rel) + if ws_end > rel then rel = ws_end end + if rel > len then break end + + local scan = rel + while scan <= len do + local c = body:byte(scan) + if c == 44 then break end -- ',' + if c == 10 then break end -- '\n' + if c == 59 then break end -- ';' + if c == 40 then local _, a = M.read_parens (body, scan); scan = a -- '(' + elseif c == 123 then local _, a = M.read_braces (body, scan); scan = a -- '{' + elseif c == 91 then local _, a = M.read_brackets (body, scan); scan = a -- '[' + elseif c == 34 or c == 39 then scan = M.skip_str_or_cmt(body, scan) + 1 -- '"' or '\'' + else + scan = scan + 1 + end + end + local tok = M.trim(body:sub(rel, scan - 1)) + if tok ~= "" then out[#out + 1] = { tok = tok, rel = rel } end + if scan <= len then + scan = scan + 1 + local w = M.skip_ws_and_cmt(body, scan) + if w > scan then scan = w end + end + rel = scan + end + _tokenize_body_cache[body] = out + return out +end + +--- Tokenize the body into a flat list of trimmed string tokens (preserves comments). +--- Uses `split_top_level_commas` (which appends trailing comments to the previous token) +--- so the components pass can emit `/* Words: ... */` comments in the .macs.h output. +--- @param body string +--- @return string[] +function M.tokenize_body_simple(body) + local tokens = M.split_top_level_commas(body) + local out = {} + for i = 1, #tokens do out[i] = M.trim(tokens[i]) end + return out +end + +--- Build a line-index: count `\n` chars from offset 1 up to the offset; that count + 1 is the line number (1-based). +--- Memoized on the body string. +--- @param body string +--- @return table -- index[pos] = line_number +function M.build_body_line_index(body) + if _body_line_index_cache[body] ~= nil then return _body_line_index_cache[body] end + local index = {} + local len = #body + local newline_count = 0 + for pos = 1, len do + if pos > 1 then + index[pos] = newline_count + 1 + end + if body:byte(pos) == 10 then + newline_count = newline_count + 1 + end + end + index[len + 1] = newline_count + 1 + _body_line_index_cache[body] = index + return index +end + +--- Find the end of a marker call (`atom_label(...)` or `atom_offset(...)`). +--- Returns the position past the closing `)`, or nil if the token isn't a marker call. +--- @param tok string +--- @return integer|nil +function M.find_marker_call_end(tok) + local ident, after = M.read_ident(tok, 1) + if not ident then return nil end + if ident ~= "atom_label" and ident ~= "atom_offset" then return nil end + local paren_pos = M.skip_ws_and_cmt(tok, after) + if tok:sub(paren_pos, paren_pos) ~= "(" then return nil end + local _, close = M.read_parens(tok, paren_pos) + return close +end + -- ════════════════════════════════════════════════════════════════════════════ -- Section 5: load_word_counts -- ════════════════════════════════════════════════════════════════════════════ diff --git a/scripts/duffle_paths.lua b/scripts/duffle_paths.lua index e2c9c1a..eb9a476 100644 --- a/scripts/duffle_paths.lua +++ b/scripts/duffle_paths.lua @@ -62,9 +62,12 @@ function M.setup() .. package.path -- lpeg: built by `update_deps.ps1` to `toolchain/lpeg/lpeg.dll`. - -- Wire its directory into cpath so `require("lpeg")` resolves. + -- lfs: compiled from pcsx-redux's vendored luafilesystem source to `toolchain/lfs/lfs.dll`. + -- Wire both directories into cpath so `require("lpeg")` and `require("lfs")` resolve. local lpeg_dir = repo_root .. "toolchain/lpeg/" + local lfs_dir = repo_root .. "toolchain/lfs/" package.cpath = lpeg_dir .. "?.dll;" + .. lfs_dir .. "?.dll;" .. package.cpath end diff --git a/scripts/passes/annotation.lua b/scripts/passes/annotation.lua index f10acc9..7a99bac 100644 --- a/scripts/passes/annotation.lua +++ b/scripts/passes/annotation.lua @@ -329,7 +329,8 @@ function M.run(ctx) -- Per-DIRECTORY (per-module) aggregation. Group sources by `src.dir`, -- validate every source in the dir, then emit ONE errors.h per dir. - local by_dir = duffle.group_sources_by_dir(ctx.sources) + -- `ctx.by_dir` is pre-computed in build_ctx (shared across all passes). + local by_dir = ctx.by_dir or duffle.group_sources_by_dir(ctx.sources) for dir, dir_sources in pairs(by_dir) do local dir_basename = dir:match("([^/\\]+)$") or dir diff --git a/scripts/passes/components.lua b/scripts/passes/components.lua index 98b420f..deb0754 100644 --- a/scripts/passes/components.lua +++ b/scripts/passes/components.lua @@ -445,8 +445,9 @@ local function word_count_rec(name, comp_by_name, wc, cache) local n if cc then n = 0 - for _, t in ipairs(duffle.split_top_level_commas(cc.body)) do - local trimmed = duffle.trim(t) + local tokens = cc.body_tokens or duffle.tokenize_body(cc.body) + for _, t in ipairs(tokens) do + local trimmed = t.tok if trimmed ~= "" then local lookup = strip_mac_prefix(duffle.read_ident(trimmed, 1)) if lookup and comp_by_name[lookup] then @@ -518,12 +519,7 @@ end --- @param body string --- @return string[] local function tokens_from_body(body) - local out = {} - for _, t in ipairs(duffle.split_top_level_commas(body)) do - local trimmed = duffle.trim(t) - if trimmed ~= "" then out[#out + 1] = trimmed end - end - return out + return duffle.tokenize_body_simple(body) end --- Determine the macro signature: function-args list (function form) or variadic-ignored (bare form). diff --git a/scripts/passes/offsets.lua b/scripts/passes/offsets.lua index be0d124..95c317a 100644 --- a/scripts/passes/offsets.lua +++ b/scripts/passes/offsets.lua @@ -230,11 +230,17 @@ end --- @param body string --- @param word_counts table --- @return table, table[], integer -local function scan_atom_body(body, word_counts) +-- scan_atom_body: walk pre-tokenized body for atom_label/atom_offset markers + word counts. +-- Uses `atom.body_tokens` from the SourceScan payload (pre-tokenized by scan-source pass). +-- @param body_tokens table[] -- {{tok=string, rel=integer}, ...} from duffle.tokenize_body +-- @param word_counts table +-- @return table, table, integer -- labels, branches, total_words +local function scan_atom_body(body_tokens, word_counts) local pos = 0 local labels = {} local branches = {} - for _, tok in ipairs(duffle.split_top_level_commas(body)) do + for _, t in ipairs(body_tokens) do + local tok = t.tok if is_marker_token(tok) then -- Marker call: record at the current pos, do NOT advance pos. scan_for_atom_markers(tok, pos, labels, branches) @@ -368,7 +374,7 @@ local function process_source(ctx, src) local atoms_data = {} for _, atom in ipairs(atoms) do - local labels, branches, total = scan_atom_body(atom.body, ctx.shared.word_counts) + local labels, branches, total = scan_atom_body(atom.body_tokens or duffle.tokenize_body(atom.body), ctx.shared.word_counts) atoms_data[#atoms_data + 1] = { name = atom.name, total_words = total, diff --git a/scripts/passes/report.lua b/scripts/passes/report.lua index 56aa8d4..ff6db84 100644 --- a/scripts/passes/report.lua +++ b/scripts/passes/report.lua @@ -404,7 +404,7 @@ function M.run(ctx) local warnings = {} local module_entries = (ctx.flags and ctx.flags._annot_results) or {} - local by_dir = duffle.group_sources_by_dir(ctx.sources) + local by_dir = ctx.by_dir or duffle.group_sources_by_dir(ctx.sources) if not ctx.dry_run then duffle.ensure_dir(ctx.out_root) end diff --git a/scripts/passes/scan_source.lua b/scripts/passes/scan_source.lua index 8f255f9..8f2afc6 100644 --- a/scripts/passes/scan_source.lua +++ b/scripts/passes/scan_source.lua @@ -467,6 +467,16 @@ local M = {} function M.run(ctx) for _, src in ipairs(ctx.sources) do src.scan = scan_source(src.text) + -- Pre-tokenize each atom body once (plex: single source of truth). + -- Downstream passes (offsets, word-counts, components, static-analysis) read from + -- `atom.body_tokens` instead of calling `split_top_level_commas` / `tokenize_body` independently. + -- The tokens are memoized in duffle.lua's cache, so re-access is O(1). + for _, atom in ipairs(src.scan.atoms) do + atom.body_tokens = duffle.tokenize_body(atom.body) + end + for _, atom in ipairs(src.scan.raw_atoms or {}) do + atom.body_tokens = duffle.tokenize_body(atom.body) + end end return { outputs = {}, errors = {}, warnings = {} } end diff --git a/scripts/passes/static_analysis.lua b/scripts/passes/static_analysis.lua index 869b76e..2865b03 100644 --- a/scripts/passes/static_analysis.lua +++ b/scripts/passes/static_analysis.lua @@ -129,91 +129,9 @@ local OUTPUT_EXTENSION = ".static_analysis.txt" --- Used by the checks to convert per-token offsets in the body to lina numbers relative to the start of `body`. --- The atom's source-line of the body-start is added by the caller. --- --- Memoization cache for build_body_line_index / tokenize_body. --- Atom bodies are immutable for the duration of a validate() pass (they come from src.scan which is set once by scan-source), --- so the same body string is safe to use as a cache key. Each call site (5 check_* functions + analyze_atom_paths) --- was re-running the full O(body_len) scan on the SAME body. After memoization, the first call pays the O(body_len) cost, --- every subsequent call on the same body returns the cached table in O(1). --- --- (Future-proofing: if a caller ever mutates the returned tokens / line-index tables, the cache aliasing means those --- mutations would leak across atoms. Current callers all read-only; safe today.) -local _body_line_index_cache = {} -local _tokenize_body_cache = {} - ---- Simple line-counting: count `\n` chars from offset 1 up to the offset; that count + 1 is the line number (1-based). -local function build_body_line_index(body) - if _body_line_index_cache[body] ~= nil then return _body_line_index_cache[body] end - local index = {} - local len = #body - local newline_count = 0 - for pos = 1, len do - if pos > 1 then - index[pos] = newline_count + 1 -- line of `pos` relative to body - end - if body:byte(pos) == 10 then -- '\n' - newline_count = newline_count + 1 - end - end - -- Offsets beyond the body still resolve to the final line - index[len + 1] = newline_count + 1 - _body_line_index_cache[body] = index - return index -end - --- NOTE: `nop_word_count` was removed when `classify_tokens` (nop_words field) replaced it. --- `count_preceding_nops` (backward walk) was replaced by `tok_class.nop_prefix` (forward-pass pre-compute). - ---- Tokenize the body inner-text into a flat list of `(token, body_rel_offset)` ---- pairs (nested parens/braces/brackets are honored; comments and strings are skipped). ---- `body_rel_offset` is the char offset within `body` of the start of the token. ---- Callers add it to the atom's `body_off` to get an absolute source position for line tracking. -local function tokenize_body(body) - if _tokenize_body_cache[body] ~= nil then return _tokenize_body_cache[body] end - local out = {} - local len = #body - local rel = 1 - while rel <= len do - -- Find next non-whitespace, non-comment start - local ws_end = duffle.skip_ws_and_cmt(body, rel) - if ws_end > rel then - rel = ws_end - end - if rel > len then break end - - -- Find comma/newline/semicolon after this token. - -- Read balanced groups so commas inside parens/braces/brackets aren't treated as separators. - -- Comments / strings are skipped. - local scan = rel - while scan <= len do - local c = body:byte(scan) - if c == 44 then break end -- ',' - if c == 10 then break end -- '\n' - if c == 59 then break end -- ';' - if c == 40 then local _, a = duffle.read_parens (body, scan); scan = a -- '(' - elseif c == 123 then local _, a = duffle.read_braces (body, scan); scan = a -- '{' - elseif c == 91 then local _, a = duffle.read_brackets (body, scan); scan = a -- '[' - elseif c == 34 or c == 39 then scan = duffle.skip_str_or_cmt(body, scan) + 1 -- '"' or '\'' - else - scan = scan + 1 - end - end - -- Extract token [rel .. scan-1] - local tok = duffle.trim(body:sub(rel, scan - 1)) - if tok ~= "" then - out[#out + 1] = { tok = tok, rel = rel } - end - -- Move past the separator - if scan <= len then - scan = scan + 1 - -- Also skip whitespace before next token - local w = duffle.skip_ws_and_cmt(body, scan) - if w > scan then scan = w end - end - rel = scan - end - _tokenize_body_cache[body] = out - return out -end +-- NOTE: `tokenize_body` and `build_body_line_index` moved to `duffle.lua` as shared +-- memoized utilities (`duffle.tokenize_body`, `duffle.build_body_line_index`). +-- The local copies were deleted; all callers now use the duffle versions. -- ════════════════════════════════════════════════════════════════════════════ -- classify_tokens — per-token classification (the plex's pre-computed data layer) @@ -637,7 +555,7 @@ end --- (warning; loop bodies aren't supported) --- unknown_macros - list of unique macro names not in duffle.INSTRUCTION_LATENCY local function analyze_atom_paths(atom) - local tokens = atom.paths.tokens or tokenize_body(atom.body) + local tokens = atom.paths.tokens or duffle.tokenize_body(atom.body) local tc = atom.paths.tok_class or classify_tokens(tokens) local n = #tokens @@ -866,8 +784,8 @@ local function validate(ctx, src) local findings = {} for _, a in ipairs(atoms) do a.paths = a.paths or {} - a.paths.tokens = tokenize_body(a.body) - a.paths.line_in_body = build_body_line_index(a.body) + a.paths.tokens = duffle.tokenize_body(a.body) + a.paths.line_in_body = duffle.build_body_line_index(a.body) a.paths.tok_class = classify_tokens(a.paths.tokens) -- analyze_atom_paths fills the *cycles / branches / has_loops / unknown_macros* fields of a.paths. @@ -1132,7 +1050,7 @@ function M.run(ctx) -- -- Group sources by `src.dir`. The first component of `dir` is the module name (e.g. "code/duffle" -> "duffle", "code/gte_hello" -> "gte_hello"). -- Output path is `/.static_analysis.txt`. - local by_dir = duffle.group_sources_by_dir(ctx.sources) + local by_dir = ctx.by_dir or duffle.group_sources_by_dir(ctx.sources) for dir, dir_sources in pairs(by_dir) do -- Run validate() against every source in this directory; accumulate atoms / findings / errors / warnings. diff --git a/scripts/passes/word_count_eval.lua b/scripts/passes/word_count_eval.lua index c40a10e..7fdf011 100644 --- a/scripts/passes/word_count_eval.lua +++ b/scripts/passes/word_count_eval.lua @@ -29,13 +29,17 @@ local duffle = dofile(_bootstrap_dir .. "../duffle_paths.lua") -- Constants -- ════════════════════════════════════════════════════════════════════════════ --- Windows separator chars — used to convert `dir /b /s` output (which uses `\`) into POSIX paths (which our scripts expect). +-- Windows separator chars — used to convert `dir` output (which uses `\`) into POSIX paths (which our scripts expect). local PATH_SEP_BACKSLASH = "\\" local PATH_SEP_FORWARD = "/" --- Glob command for Windows directory walk. `dir /b /s` lists all matching files recursively with bare paths (no headers); --- `2>nul` discards the "file not found" stderr when nothing matches. -local DIR_GLOB_CMD = 'dir /b /s "%s\\%s" 2>nul' +-- Fallback glob command (subprocess). Used when `lfs` (LuaFileSystem) is not available. +-- Scoped to `code\` to avoid walking `.git/`, `toolchain/`, `build/`, etc. +local DIR_GLOB_CMD = 'dir /b /s "%s\\code\\%s" 2>nul' + +-- Try to load lfs (LuaFileSystem). If available, scan_dir uses native directory enumeration (~2ms) +-- instead of spawning `dir /b /s` as a subprocess (~56ms). Built by update_deps.ps1 into toolchain/lfs/lfs.dll. +local lfs = pcall(require, "lfs") and require("lfs") or nil -- ════════════════════════════════════════════════════════════════════════════ -- Type declarations @@ -120,36 +124,50 @@ end -- If a build removes/creates .macs.h files mid-process, the caller can invalidate by calling `M._invalidate_scan_cache()`. local SCAN_CACHE_KEY = "__word_count_eval_scan_cache__" ---- Recursively scan a directory for files matching a glob suffix. ---- No regex per the no_regex constraint — uses plain byte matching via `dir /b /s` on Windows. +--- Scan `code/` for files matching `suffix` (e.g. `*.macs.h`). +--- Uses `lfs` (LuaFileSystem) when available — native directory enumeration at ~2ms. +--- Falls back to `dir /b /s` subprocess (~56ms) when `lfs` is not compiled. --- ---- @param dir string -- directory to scan (absolute or relative) +--- @param dir string -- project root directory --- @param suffix string -- file pattern, e.g. "*.macs.h" --- @return string[] function M.scan_dir(dir, suffix) local key = dir .. "\0" .. suffix - -- Check the in-process cache first. (Mostly helps when a build triggers multiple `M.run` calls -- e.g. - -- the audit_lua_nesting script's stress tests but the cost is ~free either way.) local cache = package.loaded[SCAN_CACHE_KEY] if cache and cache[key] then return cache[key] end local results = {} - local pipe = io.popen(DIR_GLOB_CMD:format(dir, suffix)) - if not pipe then - -- Cache the empty result too (avoids re-scan if the dir is genuinely empty -- e.g. a clean build before components has run yet). - cache = cache or {} - cache[key] = results - package.loaded[SCAN_CACHE_KEY] = cache - return results - end - for raw_line in pipe:lines() do - local path = raw_line:gsub(PATH_SEP_BACKSLASH, PATH_SEP_FORWARD) - results[#results + 1] = path - end - pipe:close() - -- Cache the result. + if lfs then + -- Native walk: list code//gen/ for matching files. Zero subprocess spawns. + local code_dir = dir .. "/code" + if lfs.attributes(code_dir, "mode") == "directory" then + for mod_name in lfs.dir(code_dir) do + if mod_name ~= "." and mod_name ~= ".." then + local gen_path = code_dir .. "/" .. mod_name .. "/gen" + if lfs.attributes(gen_path, "mode") == "directory" then + for fname in lfs.dir(gen_path) do + if fname:match("%.macs%.h$") then + results[#results + 1] = gen_path .. "/" .. fname + end + end + end + end + end + end + else + -- Fallback: single `dir /b /s` subprocess scoped to code\. + local pipe = io.popen(DIR_GLOB_CMD:format(dir, suffix)) + if pipe then + for raw_line in pipe:lines() do + results[#results + 1] = raw_line:gsub(PATH_SEP_BACKSLASH, PATH_SEP_FORWARD) + end + pipe:close() + end + end + + -- Cache the result (including empty results). cache = cache or {} cache[key] = results package.loaded[SCAN_CACHE_KEY] = cache diff --git a/scripts/ps1_meta.lua b/scripts/ps1_meta.lua index 26cefcf..d633afa 100644 --- a/scripts/ps1_meta.lua +++ b/scripts/ps1_meta.lua @@ -355,8 +355,14 @@ local function build_ctx(args) } end + -- Pre-compute the per-directory grouping once (Fleury: expose structure). + -- Three passes (annotation, report, static-analysis) call group_sources_by_dir with the same ctx.sources; + -- computing it here and stashing on ctx.by_dir eliminates 2 redundant calls. + local by_dir = duffle.group_sources_by_dir(sources) + return { sources = sources, + by_dir = by_dir, metadata_path = args.metadata, shared = {}, upstream = {}, diff --git a/scripts/update_deps.ps1 b/scripts/update_deps.ps1 index 91dc9bf..07501e7 100644 --- a/scripts/update_deps.ps1 +++ b/scripts/update_deps.ps1 @@ -101,6 +101,21 @@ push-location $path_lpeg & gcc @lpeg_compile_args pop-location +# ════════════════════════════════════════════════════════════════════════════ +# lfs (LuaFileSystem) — compiled from pcsx-redux's vendored luafilesystem source. +# Used by word_count_eval.lua :: scan_dir for native directory enumeration (~2ms) +# instead of spawning `dir /b /s` as a subprocess (~56ms). +# Source: toolchain/pcsx-redux/third_party/luafilesystem/src/lfs.c +# Output: toolchain/lfs/lfs.dll +# ════════════════════════════════════════════════════════════════════════════ + +$path_lfs = join-path $path_toolchain 'lfs' +verify-path $path_lfs +$lfs_src = join-path $path_pcsx_redux 'third_party\luafilesystem\src\lfs.c' +$lfs_dll = join-path $path_lfs 'lfs.dll' +$lfs_dll_import = join-path $luajit_lib_dir 'libluajit-5.1.dll.a' +& gcc -O2 -shared "-I$lua_inc_dir" -o $lfs_dll $lfs_src $lfs_dll_import + # ════════════════════════════════════════════════════════════════════════════ # OpenBIOS — built from the PCSX-Redux source tree via make + mipsel-none-elf #