lifting tokenize_body, using lfs package

scripts/passes/annotation.lua

@@ -329,7 +329,8 @@ function M.run(ctx)
 
 	-- Per-DIRECTORY (per-module) aggregation. Group sources by `src.dir`,
 	-- validate every source in the dir, then emit ONE errors.h per dir.
-	local by_dir = duffle.group_sources_by_dir(ctx.sources)
+	-- `ctx.by_dir` is pre-computed in build_ctx (shared across all passes).
+	local by_dir = ctx.by_dir or duffle.group_sources_by_dir(ctx.sources)
 
 	for dir, dir_sources in pairs(by_dir) do
 		local dir_basename = dir:match("([^/\\]+)$") or dir

scripts/passes/components.lua

@@ -445,8 +445,9 @@ local function word_count_rec(name, comp_by_name, wc, cache)
 	local n
 	if cc then
 		n = 0
-		for _, t in ipairs(duffle.split_top_level_commas(cc.body)) do
-			local trimmed = duffle.trim(t)
+		local tokens = cc.body_tokens or duffle.tokenize_body(cc.body)
+		for _, t in ipairs(tokens) do
+			local trimmed = t.tok
 			if trimmed ~= "" then
 				local lookup = strip_mac_prefix(duffle.read_ident(trimmed, 1))
 				if lookup and comp_by_name[lookup] then
@@ -518,12 +519,7 @@ end
 --- @param body string
 --- @return string[]
 local function tokens_from_body(body)
-	local out = {}
-	for _, t in ipairs(duffle.split_top_level_commas(body)) do
-		local trimmed = duffle.trim(t)
-		if trimmed ~= "" then out[#out + 1] = trimmed end
-	end
-	return out
+	return duffle.tokenize_body_simple(body)
 end
 
 --- Determine the macro signature: function-args list (function form) or variadic-ignored (bare form).

scripts/passes/offsets.lua

@@ -230,11 +230,17 @@ end
 --- @param body string
 --- @param word_counts table
 --- @return table<string, integer>, table[], integer
-local function scan_atom_body(body, word_counts)
+-- scan_atom_body: walk pre-tokenized body for atom_label/atom_offset markers + word counts.
+-- Uses `atom.body_tokens` from the SourceScan payload (pre-tokenized by scan-source pass).
+-- @param body_tokens table[]  -- {{tok=string, rel=integer}, ...} from duffle.tokenize_body
+-- @param word_counts table
+-- @return table, table, integer  -- labels, branches, total_words
+local function scan_atom_body(body_tokens, word_counts)
 	local pos      = 0
 	local labels   = {}
 	local branches = {}
-	for _, tok in ipairs(duffle.split_top_level_commas(body)) do
+	for _, t in ipairs(body_tokens) do
+		local tok = t.tok
 		if is_marker_token(tok) then
 			-- Marker call: record at the current pos, do NOT advance pos.
 			scan_for_atom_markers(tok, pos, labels, branches)
@@ -368,7 +374,7 @@ local function process_source(ctx, src)
 
 	local atoms_data = {}
 	for _, atom in ipairs(atoms) do
-		local labels, branches, total = scan_atom_body(atom.body, ctx.shared.word_counts)
+		local labels, branches, total = scan_atom_body(atom.body_tokens or duffle.tokenize_body(atom.body), ctx.shared.word_counts)
 		atoms_data[#atoms_data + 1] = {
 			name        = atom.name,
 			total_words = total,

scripts/passes/report.lua

@@ -404,7 +404,7 @@ function M.run(ctx)
 	local warnings = {}
 
 	local module_entries = (ctx.flags and ctx.flags._annot_results) or {}
-	local by_dir         = duffle.group_sources_by_dir(ctx.sources)
+	local by_dir         = ctx.by_dir or duffle.group_sources_by_dir(ctx.sources)
 
 	if not ctx.dry_run then duffle.ensure_dir(ctx.out_root) end
 

scripts/passes/scan_source.lua

@@ -467,6 +467,16 @@ local M = {}
 function M.run(ctx)
 	for _, src in ipairs(ctx.sources) do
 		src.scan = scan_source(src.text)
+		-- Pre-tokenize each atom body once (plex: single source of truth).
+		-- Downstream passes (offsets, word-counts, components, static-analysis) read from
+		-- `atom.body_tokens` instead of calling `split_top_level_commas` / `tokenize_body` independently.
+		-- The tokens are memoized in duffle.lua's cache, so re-access is O(1).
+		for _, atom in ipairs(src.scan.atoms) do
+			atom.body_tokens = duffle.tokenize_body(atom.body)
+		end
+		for _, atom in ipairs(src.scan.raw_atoms or {}) do
+			atom.body_tokens = duffle.tokenize_body(atom.body)
+		end
 	end
 	return { outputs = {}, errors = {}, warnings = {} }
 end

scripts/passes/static_analysis.lua

@@ -129,91 +129,9 @@ local OUTPUT_EXTENSION = ".static_analysis.txt"
 --- Used by the checks to convert per-token offsets in the body to lina numbers relative to the start of `body`. 
 --- The atom's source-line of the body-start is added by the caller.
 ---
--- Memoization cache for build_body_line_index / tokenize_body.
--- Atom bodies are immutable for the duration of a validate() pass (they come from src.scan which is set once by scan-source),
--- so the same body string is safe to use as a cache key. Each call site (5 check_* functions + analyze_atom_paths)
--- was re-running the full O(body_len) scan on the SAME body. After memoization, the first call pays the O(body_len) cost,
--- every subsequent call on the same body returns the cached table in O(1).
---
--- (Future-proofing: if a caller ever mutates the returned tokens / line-index tables, the cache aliasing means those
--- mutations would leak across atoms. Current callers all read-only; safe today.)
-local _body_line_index_cache = {}
-local _tokenize_body_cache    = {}
-
---- Simple line-counting: count `\n` chars from offset 1 up to the offset; that count + 1 is the line number (1-based).
-local function build_body_line_index(body)
-	if _body_line_index_cache[body] ~= nil then return _body_line_index_cache[body] end
-	local index = {}
-	local len = #body
-	local newline_count = 0
-	for pos = 1, len do
-		if pos > 1 then
-			index[pos] = newline_count + 1   -- line of `pos` relative to body
-		end
-		if body:byte(pos) == 10 then        -- '\n'
-			newline_count = newline_count + 1
-		end
-	end
-	-- Offsets beyond the body still resolve to the final line
-	index[len + 1] = newline_count + 1
-	_body_line_index_cache[body] = index
-	return index
-end
-
--- NOTE: `nop_word_count` was removed when `classify_tokens` (nop_words field) replaced it.
--- `count_preceding_nops` (backward walk) was replaced by `tok_class.nop_prefix` (forward-pass pre-compute).
-
---- Tokenize the body inner-text into a flat list of `(token, body_rel_offset)`
---- pairs (nested parens/braces/brackets are honored; comments and strings are skipped). 
---- `body_rel_offset` is the char offset within `body` of the start of the token. 
---- Callers add it to the atom's `body_off` to get an absolute source position for line tracking.
-local function tokenize_body(body)
-	if _tokenize_body_cache[body] ~= nil then return _tokenize_body_cache[body] end
-	local out = {}
-	local len     = #body
-	local rel     = 1
-	while rel <= len do
-		-- Find next non-whitespace, non-comment start
-		local ws_end = duffle.skip_ws_and_cmt(body, rel)
-		if ws_end > rel then
-			rel = ws_end
-		end
-		if rel > len then break end
-
-		-- Find comma/newline/semicolon after this token.
-		-- Read balanced groups so commas inside parens/braces/brackets aren't treated as separators.
-		-- Comments / strings are skipped.
-		local scan = rel
-		while scan <= len do
-			local c = body:byte(scan)
-			if     c == 44            then break end               -- ','
-			if     c == 10            then break end               -- '\n'
-			if     c == 59            then break end               -- ';'
-			if     c == 40            then local _, a = duffle.read_parens    (body, scan); scan = a -- '('
-			elseif c == 123           then local _, a = duffle.read_braces    (body, scan); scan = a -- '{'
-			elseif c == 91            then local _, a = duffle.read_brackets  (body, scan); scan = a -- '['
-			elseif c == 34 or c == 39 then scan       = duffle.skip_str_or_cmt(body, scan) + 1       -- '"' or '\''
-			else
-				scan = scan + 1
-			end
-		end
-		-- Extract token [rel .. scan-1]
-		local tok = duffle.trim(body:sub(rel, scan - 1))
-		if    tok ~= "" then
-			out[#out + 1] = { tok = tok, rel = rel }
-		end
-		-- Move past the separator
-		if scan <= len then
-			 scan = scan + 1
-			-- Also skip whitespace before next token
-			local w = duffle.skip_ws_and_cmt(body, scan)
-			if    w > scan then scan = w end
-		end
-		rel = scan
-	end
-	_tokenize_body_cache[body] = out
-	return out
-end
+-- NOTE: `tokenize_body` and `build_body_line_index` moved to `duffle.lua` as shared
+-- memoized utilities (`duffle.tokenize_body`, `duffle.build_body_line_index`).
+-- The local copies were deleted; all callers now use the duffle versions.
 
 -- ════════════════════════════════════════════════════════════════════════════
 -- classify_tokens — per-token classification (the plex's pre-computed data layer)
@@ -637,7 +555,7 @@ end
 ---                    (warning; loop bodies aren't supported)
 ---   unknown_macros - list of unique macro names not in duffle.INSTRUCTION_LATENCY
 local function analyze_atom_paths(atom)
-	local tokens   = atom.paths.tokens or tokenize_body(atom.body)
+	local tokens   = atom.paths.tokens or duffle.tokenize_body(atom.body)
 	local tc       = atom.paths.tok_class or classify_tokens(tokens)
 	local n        = #tokens
 
@@ -866,8 +784,8 @@ local function validate(ctx, src)
 	local findings = {}
 	for _, a in ipairs(atoms) do
 		a.paths                  = a.paths or {}
-		a.paths.tokens           = tokenize_body(a.body)
-		a.paths.line_in_body     = build_body_line_index(a.body)
+		a.paths.tokens           = duffle.tokenize_body(a.body)
+		a.paths.line_in_body     = duffle.build_body_line_index(a.body)
 		a.paths.tok_class        = classify_tokens(a.paths.tokens)
 
 		-- analyze_atom_paths fills the *cycles / branches / has_loops / unknown_macros* fields of a.paths.
@@ -1132,7 +1050,7 @@ function M.run(ctx)
 	--
 	-- Group sources by `src.dir`. The first component of `dir` is the module name (e.g. "code/duffle" -> "duffle", "code/gte_hello" -> "gte_hello").
 	-- Output path is `<out_root>/<module_basename>.static_analysis.txt`.
-	local by_dir = duffle.group_sources_by_dir(ctx.sources)
+	local by_dir = ctx.by_dir or duffle.group_sources_by_dir(ctx.sources)
 
 	for dir, dir_sources in pairs(by_dir) do
 		-- Run validate() against every source in this directory; accumulate atoms / findings / errors / warnings. 

scripts/passes/word_count_eval.lua

@@ -29,13 +29,17 @@ local duffle        = dofile(_bootstrap_dir .. "../duffle_paths.lua")
 -- Constants
 -- ════════════════════════════════════════════════════════════════════════════
 
--- Windows separator chars — used to convert `dir /b /s` output (which uses `\`) into POSIX paths (which our scripts expect).
+-- Windows separator chars — used to convert `dir` output (which uses `\`) into POSIX paths (which our scripts expect).
 local PATH_SEP_BACKSLASH = "\\"
 local PATH_SEP_FORWARD   = "/"
 
--- Glob command for Windows directory walk. `dir /b /s` lists all matching files recursively with bare paths (no headers); 
--- `2>nul` discards the "file not found" stderr when nothing matches.
-local DIR_GLOB_CMD = 'dir /b /s "%s\\%s" 2>nul'
+-- Fallback glob command (subprocess). Used when `lfs` (LuaFileSystem) is not available.
+-- Scoped to `code\` to avoid walking `.git/`, `toolchain/`, `build/`, etc.
+local DIR_GLOB_CMD = 'dir /b /s "%s\\code\\%s" 2>nul'
+
+-- Try to load lfs (LuaFileSystem). If available, scan_dir uses native directory enumeration (~2ms)
+-- instead of spawning `dir /b /s` as a subprocess (~56ms). Built by update_deps.ps1 into toolchain/lfs/lfs.dll.
+local lfs = pcall(require, "lfs") and require("lfs") or nil
 
 -- ════════════════════════════════════════════════════════════════════════════
 -- Type declarations
@@ -120,36 +124,50 @@ end
 -- If a build removes/creates .macs.h files mid-process, the caller can invalidate by calling `M._invalidate_scan_cache()`.
 local SCAN_CACHE_KEY = "__word_count_eval_scan_cache__"
 
---- Recursively scan a directory for files matching a glob suffix.
---- No regex per the no_regex constraint — uses plain byte matching via `dir /b /s` on Windows.
+--- Scan `code/` for files matching `suffix` (e.g. `*.macs.h`).
+--- Uses `lfs` (LuaFileSystem) when available — native directory enumeration at ~2ms.
+--- Falls back to `dir /b /s` subprocess (~56ms) when `lfs` is not compiled.
 ---
---- @param dir string        -- directory to scan (absolute or relative)
+--- @param dir string        -- project root directory
 --- @param suffix string     -- file pattern, e.g. "*.macs.h"
 --- @return string[]
 function M.scan_dir(dir, suffix)
 	local key = dir .. "\0" .. suffix
 
-	-- Check the in-process cache first. (Mostly helps when a build triggers multiple `M.run` calls -- e.g. 
-	-- the audit_lua_nesting script's stress tests but the cost is ~free either way.)
 	local cache = package.loaded[SCAN_CACHE_KEY]
 	if    cache and cache[key] then return cache[key] end
 
 	local results = {}
-	local pipe = io.popen(DIR_GLOB_CMD:format(dir, suffix))
-	if not pipe then
-		-- Cache the empty result too (avoids re-scan if the dir is genuinely empty -- e.g. a clean build before components has run yet).
-		cache      = cache or {}
-		cache[key] = results
-		package.loaded[SCAN_CACHE_KEY] = cache
-		return results
-	end
-	for raw_line in pipe:lines() do
-		local path = raw_line:gsub(PATH_SEP_BACKSLASH, PATH_SEP_FORWARD)
-		results[#results + 1] = path
-	end
-	pipe:close()
 
-	-- Cache the result.
+	if lfs then
+		-- Native walk: list code/<module>/gen/ for matching files. Zero subprocess spawns.
+		local code_dir = dir .. "/code"
+		if lfs.attributes(code_dir, "mode") == "directory" then
+			for mod_name in lfs.dir(code_dir) do
+				if mod_name ~= "." and mod_name ~= ".." then
+					local gen_path = code_dir .. "/" .. mod_name .. "/gen"
+					if lfs.attributes(gen_path, "mode") == "directory" then
+						for fname in lfs.dir(gen_path) do
+							if fname:match("%.macs%.h$") then
+								results[#results + 1] = gen_path .. "/" .. fname
+							end
+						end
+					end
+				end
+			end
+		end
+	else
+		-- Fallback: single `dir /b /s` subprocess scoped to code\.
+		local pipe = io.popen(DIR_GLOB_CMD:format(dir, suffix))
+		if pipe then
+			for raw_line in pipe:lines() do
+				results[#results + 1] = raw_line:gsub(PATH_SEP_BACKSLASH, PATH_SEP_FORWARD)
+			end
+			pipe:close()
+		end
+	end
+
+	-- Cache the result (including empty results).
 	cache      = cache or {}
 	cache[key] = results
 	package.loaded[SCAN_CACHE_KEY] = cache