lifting tokenize_body, using lfs package

.gitignore

@@ -15,3 +15,5 @@ toolchain/PSn00bSDK
 *.a
 .sentry-native
 .vscode/settings.json
+toolchain/lfs
+toolchain/lpeg

scripts/duffle.lua

@@ -549,6 +549,106 @@ function M.split_top_level_commas(body)
 	return tokens
 end
 
+-- ════════════════════════════════════════════════════════════════════════════
+-- Section 4b: tokenize_body + build_body_line_index (shared, memoized)
+-- ════════════════════════════════════════════════════════════════════════════
+
+-- Moved here from passes/static_analysis.lua so all passes can share the memoized
+-- per-body tokenization. The memoization key is the body string (immutable per pass).
+
+local _tokenize_body_cache    = {}
+local _body_line_index_cache  = {}
+
+--- Tokenize the body inner-text into a flat list of `{tok, rel}` pairs.
+--- `tok` is the trimmed token string; `rel` is the byte offset within `body`.
+--- Memoized on the body string — first call pays O(body_len), subsequent calls return cached.
+--- @param body string
+--- @return table[]  -- {{tok=string, rel=integer}, ...}
+function M.tokenize_body(body)
+	if _tokenize_body_cache[body] ~= nil then return _tokenize_body_cache[body] end
+	local out = {}
+	local len     = #body
+	local rel     = 1
+	while rel <= len do
+		local ws_end = M.skip_ws_and_cmt(body, rel)
+		if ws_end > rel then rel = ws_end end
+		if rel > len then break end
+
+		local scan = rel
+		while scan <= len do
+			local c = body:byte(scan)
+			if     c == 44            then break end               -- ','
+			if     c == 10            then break end               -- '\n'
+			if     c == 59            then break end               -- ';'
+			if     c == 40            then local _, a = M.read_parens   (body, scan); scan = a -- '('
+			elseif c == 123           then local _, a = M.read_braces   (body, scan); scan = a -- '{'
+			elseif c == 91            then local _, a = M.read_brackets (body, scan); scan = a -- '['
+			elseif c == 34 or c == 39 then scan       = M.skip_str_or_cmt(body, scan) + 1     -- '"' or '\''
+			else
+				scan = scan + 1
+			end
+		end
+		local tok = M.trim(body:sub(rel, scan - 1))
+		if tok ~= "" then out[#out + 1] = { tok = tok, rel = rel } end
+		if scan <= len then
+			scan = scan + 1
+			local w = M.skip_ws_and_cmt(body, scan)
+			if w > scan then scan = w end
+		end
+		rel = scan
+	end
+	_tokenize_body_cache[body] = out
+	return out
+end
+
+--- Tokenize the body into a flat list of trimmed string tokens (preserves comments).
+--- Uses `split_top_level_commas` (which appends trailing comments to the previous token)
+--- so the components pass can emit `/* Words: ... */` comments in the .macs.h output.
+--- @param body string
+--- @return string[]
+function M.tokenize_body_simple(body)
+	local tokens = M.split_top_level_commas(body)
+	local out = {}
+	for i = 1, #tokens do out[i] = M.trim(tokens[i]) end
+	return out
+end
+
+--- Build a line-index: count `\n` chars from offset 1 up to the offset; that count + 1 is the line number (1-based).
+--- Memoized on the body string.
+--- @param body string
+--- @return table  -- index[pos] = line_number
+function M.build_body_line_index(body)
+	if _body_line_index_cache[body] ~= nil then return _body_line_index_cache[body] end
+	local index = {}
+	local len = #body
+	local newline_count = 0
+	for pos = 1, len do
+		if pos > 1 then
+			index[pos] = newline_count + 1
+		end
+		if body:byte(pos) == 10 then
+			newline_count = newline_count + 1
+		end
+	end
+	index[len + 1] = newline_count + 1
+	_body_line_index_cache[body] = index
+	return index
+end
+
+--- Find the end of a marker call (`atom_label(...)` or `atom_offset(...)`).
+--- Returns the position past the closing `)`, or nil if the token isn't a marker call.
+--- @param tok string
+--- @return integer|nil
+function M.find_marker_call_end(tok)
+	local ident, after = M.read_ident(tok, 1)
+	if not ident then return nil end
+	if ident ~= "atom_label" and ident ~= "atom_offset" then return nil end
+	local paren_pos = M.skip_ws_and_cmt(tok, after)
+	if tok:sub(paren_pos, paren_pos) ~= "(" then return nil end
+	local _, close = M.read_parens(tok, paren_pos)
+	return close
+end
+
 -- ════════════════════════════════════════════════════════════════════════════
 -- Section 5: load_word_counts
 -- ════════════════════════════════════════════════════════════════════════════

scripts/duffle_paths.lua

@@ -62,9 +62,12 @@ function M.setup()
 		.. package.path
 
 	-- lpeg: built by `update_deps.ps1` to `toolchain/lpeg/lpeg.dll`.
-	-- Wire its directory into cpath so `require("lpeg")` resolves.
+	-- lfs: compiled from pcsx-redux's vendored luafilesystem source to `toolchain/lfs/lfs.dll`.
+	-- Wire both directories into cpath so `require("lpeg")` and `require("lfs")` resolve.
 	local lpeg_dir = repo_root .. "toolchain/lpeg/"
+	local lfs_dir  = repo_root .. "toolchain/lfs/"
 	package.cpath = lpeg_dir .. "?.dll;"
+		.. lfs_dir  .. "?.dll;"
 		.. package.cpath
 end
 

scripts/passes/annotation.lua

@@ -329,7 +329,8 @@ function M.run(ctx)
 
 	-- Per-DIRECTORY (per-module) aggregation. Group sources by `src.dir`,
 	-- validate every source in the dir, then emit ONE errors.h per dir.
-	local by_dir = duffle.group_sources_by_dir(ctx.sources)
+	-- `ctx.by_dir` is pre-computed in build_ctx (shared across all passes).
+	local by_dir = ctx.by_dir or duffle.group_sources_by_dir(ctx.sources)
 
 	for dir, dir_sources in pairs(by_dir) do
 		local dir_basename = dir:match("([^/\\]+)$") or dir

scripts/passes/components.lua

@@ -445,8 +445,9 @@ local function word_count_rec(name, comp_by_name, wc, cache)
 	local n
 	if cc then
 		n = 0
-		for _, t in ipairs(duffle.split_top_level_commas(cc.body)) do
+		local tokens = cc.body_tokens or duffle.tokenize_body(cc.body)
-			local trimmed = duffle.trim(t)
+		for _, t in ipairs(tokens) do
+			local trimmed = t.tok
 			if trimmed ~= "" then
 				local lookup = strip_mac_prefix(duffle.read_ident(trimmed, 1))
 				if lookup and comp_by_name[lookup] then
@@ -518,12 +519,7 @@ end
 --- @param body string
 --- @return string[]
 local function tokens_from_body(body)
-	local out = {}
+	return duffle.tokenize_body_simple(body)
-	for _, t in ipairs(duffle.split_top_level_commas(body)) do
-		local trimmed = duffle.trim(t)
-		if trimmed ~= "" then out[#out + 1] = trimmed end
-	end
-	return out
 end
 
 --- Determine the macro signature: function-args list (function form) or variadic-ignored (bare form).

scripts/passes/offsets.lua

@@ -230,11 +230,17 @@ end
 --- @param body string
 --- @param word_counts table
 --- @return table<string, integer>, table[], integer
-local function scan_atom_body(body, word_counts)
+-- scan_atom_body: walk pre-tokenized body for atom_label/atom_offset markers + word counts.
+-- Uses `atom.body_tokens` from the SourceScan payload (pre-tokenized by scan-source pass).
+-- @param body_tokens table[]  -- {{tok=string, rel=integer}, ...} from duffle.tokenize_body
+-- @param word_counts table
+-- @return table, table, integer  -- labels, branches, total_words
+local function scan_atom_body(body_tokens, word_counts)
 	local pos      = 0
 	local labels   = {}
 	local branches = {}
-	for _, tok in ipairs(duffle.split_top_level_commas(body)) do
+	for _, t in ipairs(body_tokens) do
+		local tok = t.tok
 		if is_marker_token(tok) then
 			-- Marker call: record at the current pos, do NOT advance pos.
 			scan_for_atom_markers(tok, pos, labels, branches)
@@ -368,7 +374,7 @@ local function process_source(ctx, src)
 
 	local atoms_data = {}
 	for _, atom in ipairs(atoms) do
-		local labels, branches, total = scan_atom_body(atom.body, ctx.shared.word_counts)
+		local labels, branches, total = scan_atom_body(atom.body_tokens or duffle.tokenize_body(atom.body), ctx.shared.word_counts)
 		atoms_data[#atoms_data + 1] = {
 			name        = atom.name,
 			total_words = total,

scripts/passes/report.lua

@@ -404,7 +404,7 @@ function M.run(ctx)
 	local warnings = {}
 
 	local module_entries = (ctx.flags and ctx.flags._annot_results) or {}
-	local by_dir         = duffle.group_sources_by_dir(ctx.sources)
+	local by_dir         = ctx.by_dir or duffle.group_sources_by_dir(ctx.sources)
 
 	if not ctx.dry_run then duffle.ensure_dir(ctx.out_root) end
 

scripts/passes/scan_source.lua

@@ -467,6 +467,16 @@ local M = {}
 function M.run(ctx)
 	for _, src in ipairs(ctx.sources) do
 		src.scan = scan_source(src.text)
+		-- Pre-tokenize each atom body once (plex: single source of truth).
+		-- Downstream passes (offsets, word-counts, components, static-analysis) read from
+		-- `atom.body_tokens` instead of calling `split_top_level_commas` / `tokenize_body` independently.
+		-- The tokens are memoized in duffle.lua's cache, so re-access is O(1).
+		for _, atom in ipairs(src.scan.atoms) do
+			atom.body_tokens = duffle.tokenize_body(atom.body)
+		end
+		for _, atom in ipairs(src.scan.raw_atoms or {}) do
+			atom.body_tokens = duffle.tokenize_body(atom.body)
+		end
 	end
 	return { outputs = {}, errors = {}, warnings = {} }
 end

scripts/passes/static_analysis.lua

@@ -129,91 +129,9 @@ local OUTPUT_EXTENSION = ".static_analysis.txt"
 --- Used by the checks to convert per-token offsets in the body to lina numbers relative to the start of `body`. 
 --- The atom's source-line of the body-start is added by the caller.
 ---
--- Memoization cache for build_body_line_index / tokenize_body.
+-- NOTE: `tokenize_body` and `build_body_line_index` moved to `duffle.lua` as shared
--- Atom bodies are immutable for the duration of a validate() pass (they come from src.scan which is set once by scan-source),
+-- memoized utilities (`duffle.tokenize_body`, `duffle.build_body_line_index`).
--- so the same body string is safe to use as a cache key. Each call site (5 check_* functions + analyze_atom_paths)
+-- The local copies were deleted; all callers now use the duffle versions.
--- was re-running the full O(body_len) scan on the SAME body. After memoization, the first call pays the O(body_len) cost,
--- every subsequent call on the same body returns the cached table in O(1).
---
--- (Future-proofing: if a caller ever mutates the returned tokens / line-index tables, the cache aliasing means those
--- mutations would leak across atoms. Current callers all read-only; safe today.)
-local _body_line_index_cache = {}
-local _tokenize_body_cache    = {}
-
---- Simple line-counting: count `\n` chars from offset 1 up to the offset; that count + 1 is the line number (1-based).
-local function build_body_line_index(body)
-	if _body_line_index_cache[body] ~= nil then return _body_line_index_cache[body] end
-	local index = {}
-	local len = #body
-	local newline_count = 0
-	for pos = 1, len do
-		if pos > 1 then
-			index[pos] = newline_count + 1   -- line of `pos` relative to body
-		end
-		if body:byte(pos) == 10 then        -- '\n'
-			newline_count = newline_count + 1
-		end
-	end
-	-- Offsets beyond the body still resolve to the final line
-	index[len + 1] = newline_count + 1
-	_body_line_index_cache[body] = index
-	return index
-end
-
--- NOTE: `nop_word_count` was removed when `classify_tokens` (nop_words field) replaced it.
--- `count_preceding_nops` (backward walk) was replaced by `tok_class.nop_prefix` (forward-pass pre-compute).
-
---- Tokenize the body inner-text into a flat list of `(token, body_rel_offset)`
---- pairs (nested parens/braces/brackets are honored; comments and strings are skipped). 
---- `body_rel_offset` is the char offset within `body` of the start of the token. 
---- Callers add it to the atom's `body_off` to get an absolute source position for line tracking.
-local function tokenize_body(body)
-	if _tokenize_body_cache[body] ~= nil then return _tokenize_body_cache[body] end
-	local out = {}
-	local len     = #body
-	local rel     = 1
-	while rel <= len do
-		-- Find next non-whitespace, non-comment start
-		local ws_end = duffle.skip_ws_and_cmt(body, rel)
-		if ws_end > rel then
-			rel = ws_end
-		end
-		if rel > len then break end
-
-		-- Find comma/newline/semicolon after this token.
-		-- Read balanced groups so commas inside parens/braces/brackets aren't treated as separators.
-		-- Comments / strings are skipped.
-		local scan = rel
-		while scan <= len do
-			local c = body:byte(scan)
-			if     c == 44            then break end               -- ','
-			if     c == 10            then break end               -- '\n'
-			if     c == 59            then break end               -- ';'
-			if     c == 40            then local _, a = duffle.read_parens    (body, scan); scan = a -- '('
-			elseif c == 123           then local _, a = duffle.read_braces    (body, scan); scan = a -- '{'
-			elseif c == 91            then local _, a = duffle.read_brackets  (body, scan); scan = a -- '['
-			elseif c == 34 or c == 39 then scan       = duffle.skip_str_or_cmt(body, scan) + 1       -- '"' or '\''
-			else
-				scan = scan + 1
-			end
-		end
-		-- Extract token [rel .. scan-1]
-		local tok = duffle.trim(body:sub(rel, scan - 1))
-		if    tok ~= "" then
-			out[#out + 1] = { tok = tok, rel = rel }
-		end
-		-- Move past the separator
-		if scan <= len then
-			 scan = scan + 1
-			-- Also skip whitespace before next token
-			local w = duffle.skip_ws_and_cmt(body, scan)
-			if    w > scan then scan = w end
-		end
-		rel = scan
-	end
-	_tokenize_body_cache[body] = out
-	return out
-end
 
 -- ════════════════════════════════════════════════════════════════════════════
 -- classify_tokens — per-token classification (the plex's pre-computed data layer)
@@ -637,7 +555,7 @@ end
 ---                    (warning; loop bodies aren't supported)
 ---   unknown_macros - list of unique macro names not in duffle.INSTRUCTION_LATENCY
 local function analyze_atom_paths(atom)
-	local tokens   = atom.paths.tokens or tokenize_body(atom.body)
+	local tokens   = atom.paths.tokens or duffle.tokenize_body(atom.body)
 	local tc       = atom.paths.tok_class or classify_tokens(tokens)
 	local n        = #tokens
 
@@ -866,8 +784,8 @@ local function validate(ctx, src)
 	local findings = {}
 	for _, a in ipairs(atoms) do
 		a.paths                  = a.paths or {}
-		a.paths.tokens           = tokenize_body(a.body)
+		a.paths.tokens           = duffle.tokenize_body(a.body)
-		a.paths.line_in_body     = build_body_line_index(a.body)
+		a.paths.line_in_body     = duffle.build_body_line_index(a.body)
 		a.paths.tok_class        = classify_tokens(a.paths.tokens)
 
 		-- analyze_atom_paths fills the *cycles / branches / has_loops / unknown_macros* fields of a.paths.
@@ -1132,7 +1050,7 @@ function M.run(ctx)
 	--
 	-- Group sources by `src.dir`. The first component of `dir` is the module name (e.g. "code/duffle" -> "duffle", "code/gte_hello" -> "gte_hello").
 	-- Output path is `<out_root>/<module_basename>.static_analysis.txt`.
-	local by_dir = duffle.group_sources_by_dir(ctx.sources)
+	local by_dir = ctx.by_dir or duffle.group_sources_by_dir(ctx.sources)
 
 	for dir, dir_sources in pairs(by_dir) do
 		-- Run validate() against every source in this directory; accumulate atoms / findings / errors / warnings. 

scripts/passes/word_count_eval.lua

@@ -29,13 +29,17 @@ local duffle        = dofile(_bootstrap_dir .. "../duffle_paths.lua")
 -- Constants
 -- ════════════════════════════════════════════════════════════════════════════
 
--- Windows separator chars — used to convert `dir /b /s` output (which uses `\`) into POSIX paths (which our scripts expect).
+-- Windows separator chars — used to convert `dir` output (which uses `\`) into POSIX paths (which our scripts expect).
 local PATH_SEP_BACKSLASH = "\\"
 local PATH_SEP_FORWARD   = "/"
 
--- Glob command for Windows directory walk. `dir /b /s` lists all matching files recursively with bare paths (no headers); 
+-- Fallback glob command (subprocess). Used when `lfs` (LuaFileSystem) is not available.
--- `2>nul` discards the "file not found" stderr when nothing matches.
+-- Scoped to `code\` to avoid walking `.git/`, `toolchain/`, `build/`, etc.
-local DIR_GLOB_CMD = 'dir /b /s "%s\\%s" 2>nul'
+local DIR_GLOB_CMD = 'dir /b /s "%s\\code\\%s" 2>nul'
+
+-- Try to load lfs (LuaFileSystem). If available, scan_dir uses native directory enumeration (~2ms)
+-- instead of spawning `dir /b /s` as a subprocess (~56ms). Built by update_deps.ps1 into toolchain/lfs/lfs.dll.
+local lfs = pcall(require, "lfs") and require("lfs") or nil
 
 -- ════════════════════════════════════════════════════════════════════════════
 -- Type declarations
@@ -120,36 +124,50 @@ end
 -- If a build removes/creates .macs.h files mid-process, the caller can invalidate by calling `M._invalidate_scan_cache()`.
 local SCAN_CACHE_KEY = "__word_count_eval_scan_cache__"
 
---- Recursively scan a directory for files matching a glob suffix.
+--- Scan `code/` for files matching `suffix` (e.g. `*.macs.h`).
---- No regex per the no_regex constraint — uses plain byte matching via `dir /b /s` on Windows.
+--- Uses `lfs` (LuaFileSystem) when available — native directory enumeration at ~2ms.
+--- Falls back to `dir /b /s` subprocess (~56ms) when `lfs` is not compiled.
 ---
---- @param dir string        -- directory to scan (absolute or relative)
+--- @param dir string        -- project root directory
 --- @param suffix string     -- file pattern, e.g. "*.macs.h"
 --- @return string[]
 function M.scan_dir(dir, suffix)
 	local key = dir .. "\0" .. suffix
 
-	-- Check the in-process cache first. (Mostly helps when a build triggers multiple `M.run` calls -- e.g. 
-	-- the audit_lua_nesting script's stress tests but the cost is ~free either way.)
 	local cache = package.loaded[SCAN_CACHE_KEY]
 	if    cache and cache[key] then return cache[key] end
 
 	local results = {}
-	local pipe = io.popen(DIR_GLOB_CMD:format(dir, suffix))
-	if not pipe then
-		-- Cache the empty result too (avoids re-scan if the dir is genuinely empty -- e.g. a clean build before components has run yet).
-		cache      = cache or {}
-		cache[key] = results
-		package.loaded[SCAN_CACHE_KEY] = cache
-		return results
-	end
-	for raw_line in pipe:lines() do
-		local path = raw_line:gsub(PATH_SEP_BACKSLASH, PATH_SEP_FORWARD)
-		results[#results + 1] = path
-	end
-	pipe:close()
 
-	-- Cache the result.
+	if lfs then
+		-- Native walk: list code/<module>/gen/ for matching files. Zero subprocess spawns.
+		local code_dir = dir .. "/code"
+		if lfs.attributes(code_dir, "mode") == "directory" then
+			for mod_name in lfs.dir(code_dir) do
+				if mod_name ~= "." and mod_name ~= ".." then
+					local gen_path = code_dir .. "/" .. mod_name .. "/gen"
+					if lfs.attributes(gen_path, "mode") == "directory" then
+						for fname in lfs.dir(gen_path) do
+							if fname:match("%.macs%.h$") then
+								results[#results + 1] = gen_path .. "/" .. fname
+							end
+						end
+					end
+				end
+			end
+		end
+	else
+		-- Fallback: single `dir /b /s` subprocess scoped to code\.
+		local pipe = io.popen(DIR_GLOB_CMD:format(dir, suffix))
+		if pipe then
+			for raw_line in pipe:lines() do
+				results[#results + 1] = raw_line:gsub(PATH_SEP_BACKSLASH, PATH_SEP_FORWARD)
+			end
+			pipe:close()
+		end
+	end
+
+	-- Cache the result (including empty results).
 	cache      = cache or {}
 	cache[key] = results
 	package.loaded[SCAN_CACHE_KEY] = cache

scripts/ps1_meta.lua

@@ -355,8 +355,14 @@ local function build_ctx(args)
 		}
 	end
 
+	-- Pre-compute the per-directory grouping once (Fleury: expose structure).
+	-- Three passes (annotation, report, static-analysis) call group_sources_by_dir with the same ctx.sources;
+	-- computing it here and stashing on ctx.by_dir eliminates 2 redundant calls.
+	local by_dir = duffle.group_sources_by_dir(sources)
+
 	return {
 		sources       = sources,
+		by_dir        = by_dir,
 		metadata_path = args.metadata,
 		shared        = {},
 		upstream      = {},

scripts/update_deps.ps1

@@ -101,6 +101,21 @@ push-location $path_lpeg
 &	gcc @lpeg_compile_args
 pop-location
 
+# ════════════════════════════════════════════════════════════════════════════
+# lfs (LuaFileSystem) — compiled from pcsx-redux's vendored luafilesystem source.
+# Used by word_count_eval.lua :: scan_dir for native directory enumeration (~2ms)
+# instead of spawning `dir /b /s` as a subprocess (~56ms).
+# Source: toolchain/pcsx-redux/third_party/luafilesystem/src/lfs.c
+# Output: toolchain/lfs/lfs.dll
+# ════════════════════════════════════════════════════════════════════════════
+
+$path_lfs = join-path $path_toolchain 'lfs'
+verify-path $path_lfs
+$lfs_src = join-path $path_pcsx_redux 'third_party\luafilesystem\src\lfs.c'
+$lfs_dll = join-path $path_lfs 'lfs.dll'
+$lfs_dll_import = join-path $luajit_lib_dir 'libluajit-5.1.dll.a'
+& gcc -O2 -shared "-I$lua_inc_dir" -o $lfs_dll $lfs_src $lfs_dll_import
+
 # ════════════════════════════════════════════════════════════════════════════
 # OpenBIOS — built from the PCSX-Redux source tree via make + mipsel-none-elf
 #