finally starting to approach decent performance.

scripts/duffle.lua

@@ -20,6 +20,12 @@
 
 local M = {}
 
+-- Optional native extension: lfs (LuaFileSystem). When present, ensure_dir uses
+-- lfs.attributes + lfs.mkdir instead of spawning `cmd.exe mkdir` — saves ~55ms per
+-- unique directory on Windows. Built by `update_deps.ps1` to `toolchain/lfs/lfs.dll`
+-- and wired into package.cpath by `scripts/duffle_paths.lua`.
+local lfs = pcall(require, "lfs") and require("lfs") or nil
+
 -- ════════════════════════════════════════════════════════════════════════════
 -- Cross-file type aliases
 -- ════════════════════════════════════════════════════════════════════════════
@@ -308,21 +314,37 @@ end
 -- Convert a (possibly relative) path to an absolute path, using CWD if needed.
 -- Normalizes forward slashes to backslashes on Windows.
 -- Used for byte-identical emit: the // Source: comment line uses the absolute path.
+--
+-- The CWD is memoized (one `io.popen("cd")` per process — ~50ms on Windows).
+-- Without the cache, calling this per-source in the components pass added ~1.5s to a 30-source build.
 -- @param path string
 -- @return string
+local _absolute_path_cache = {}
+
 function M.to_absolute_path(path)
+	if _absolute_path_cache[path] then return _absolute_path_cache[path] end
 	if #path >= 2 and path:sub(2, 2) == ":" then
 		-- Already absolute; normalize slashes for consistency.
-		return (path:gsub("/", "\\"))
+		local result = (path:gsub("/", "\\"))
+		_absolute_path_cache[path] = result
+		return result
 	end
-	local p = io.popen("cd")
+	-- Native: lfs.currentdir() is ~0ms vs io.popen("cd") at ~50ms per call.
-	if not p then return path end
+	local cwd
-	local cwd = p:read("*l")
+	if lfs then
-	p:close()
+		cwd = lfs.currentdir()
-	if not cwd then return path end
+	else
+		local p = io.popen("cd")
+		if not p then _absolute_path_cache[path] = path; return path end
+		cwd = p:read("*l")
+		p:close()
+	end
+	if not cwd then _absolute_path_cache[path] = path; return path end
 	cwd = cwd:gsub("/", "\\")
 	local tail = (path:gsub("/", "\\"))
-	return cwd .. "\\" .. tail
+	local result = cwd .. "\\" .. tail
+	_absolute_path_cache[path] = result
+	return result
 end
 
 -- Cache of directories already verified to exist in this process.
@@ -334,8 +356,15 @@ local _ensured_dirs = {}
 function M.ensure_dir(path)
 	if _ensured_dirs[path] then return end
 	_ensured_dirs[path] = true
-	local is_win = package.config:sub(1, 1) == "\\"
+	if lfs then
-	os.execute(is_win and ('if not exist "' .. path .. '" mkdir "' .. path .. '"') or ('mkdir -p "' .. path .. '" 2>/dev/null'))
+		-- Native: ~0ms when dir exists (the common case). lfs.mkdir on a new dir is ~2ms (no shell spawn).
+		-- Falls through silently if lfs.mkdir fails (e.g. permission denied); the subsequent write_file will surface the error.
+		if lfs.attributes(path, "mode") ~= "directory" then lfs.mkdir(path) end
+	else
+		-- Fallback: shell mkdir. Slow (~55ms per call on Windows due to cmd.exe spawn) but works without lfs.
+		local is_win = package.config:sub(1, 1) == "\\"
+		os.execute(is_win and ('if not exist "' .. path .. '" mkdir "' .. path .. '"') or ('mkdir -p "' .. path .. '" 2>/dev/null'))
+	end
 end
 
 -- Test helper: clear the cache (used by tests + between process runs).

scripts/passes/components.lua

@@ -350,6 +350,8 @@ end
 
 -- Project pre-scanned MipsAtomComp_ / MipsAtomComp_Proc_ entries into Component shape.
 -- Does per-source backward lookups for args (preceding function decl) and comment (preceding comment block).
+-- Carries `body_tokens` forward from scan-source so word_count_rec reads from the precomputed table
+-- instead of calling duffle.tokenize_body again.
 -- @param source string  -- the full source text (needed for backward lookups)
 -- @param scan  table   -- SourceScan from duffle.scan_source
 -- @return Component[]
@@ -360,11 +362,12 @@ local function project_components(source, scan)
 			local args    = find_function_args_for(source, a.raw_name, a.ident_pos)
 			local comment = preceding_comment_block(source, a.ident_pos)
 			out[#out + 1] = {
-				line    = a.line,
+				line        = a.line,
-				name    = a.name,
+				name        = a.name,
-				body    = a.body,
+				body        = a.body,
-				args    = args,
+				body_tokens = a.body_tokens,
-				comment = comment,
+				args        = args,
+				comment     = comment,
 			}
 		end
 	end
@@ -445,7 +448,7 @@ local function word_count_rec(name, comp_by_name, wc, cache)
 	local n
 	if cc then
 		n = 0
-		local tokens = cc.body_tokens or duffle.tokenize_body(cc.body)
+		local tokens = cc.body_tokens
 		for _, t in ipairs(tokens) do
 			local trimmed = t.tok
 			if trimmed ~= "" then

scripts/passes/offsets.lua

@@ -164,44 +164,6 @@ local function scan_for_atom_markers(token, at_pos, labels, branches)
 	end
 end
 
---- Find the end position (just past the closing ')') of the first atom_label/atom_offset call in `tok`. Returns 0 if no such call.
---- @param tok string
---- @return integer  -- 0 if no marker call found; otherwise end-1 (just past ')')
-local function find_marker_call_end(tok)
-	local pos     = 1
-	local tok_len = #tok
-	while pos <= tok_len do
-		pos = duffle.skip_ws_and_cmt(tok, pos)
-		if pos > tok_len then break end
-		local ch = tok:sub(pos, pos)
-		if duffle.is_space(ch) then
-			pos = pos + 1
-		elseif ch == "/" then
-			-- comment — skip past it (delegated to duffle.skip_str_or_cmt)
-			local nx = duffle.skip_str_or_cmt(tok, pos)
-			pos = (nx > pos) and nx or (pos + 1)
-		else
-			local ident, after_ident = duffle.read_ident(tok, pos)
-			-- scan: <ident>
-			if ident == LABEL_MARKER or ident == OFFSET_MARKER then
-				-- scan: atom_label(<name>)  OR  atom_offset(<tag>, <target>)
-				local open_paren = duffle.skip_ws_and_cmt(tok, after_ident)
-				if tok:sub(open_paren, open_paren) == "(" then
-					local _, end_paren = duffle.read_parens(tok, open_paren)
-					return end_paren - 1
-				end
-				return 0
-			end
-			pos = after_ident or (pos + 1)
-		end
-	end
-	return 0
-end
-
--- ════════════════════════════════════════════════════════════════════════════
--- Per-atom body scan (for atom_label / atom_offset markers)
--- ════════════════════════════════════════════════════════════════════════════
-
 -- (internal) Count words emitted by the rest of `tok` after a marker call
 -- (the marker call itself emits 0 words, but the source pattern may bundle the marker with the next instruction on the same line,
 -- separated by no top-level comma).
@@ -210,9 +172,12 @@ end
 -- @param word_counts table
 -- @return integer
 local function count_marker_rest(tok, word_counts)
-	local marker_end = find_marker_call_end(tok)
+	-- duffle.find_marker_call_end returns the position PAST the closing `)` of the marker call
-	if marker_end <= 0 or marker_end >= #tok then return 0 end
+	-- (or nil if `tok` isn't a marker call). Canonical impl in duffle.lua is faster than the
-	local rest = duffle.trim(tok:sub(marker_end + 1))
+	-- file-local copy that used to live here (byte-indexed, no `tok:sub` per char).
+	local marker_end = duffle.find_marker_call_end(tok)
+	if not marker_end or marker_end >= #tok then return 0 end
+	local rest = duffle.trim(tok:sub(marker_end))
 	if rest == "" then return 0 end
 	return count_token_words(rest, word_counts)
 end
@@ -348,17 +313,19 @@ end
 
 local M = {}
 
--- Project the pre-scanned SourceScan entries into the {name, body} shape this pass needs.
+-- Project the pre-scanned SourceScan entries into the {name, body, body_tokens} shape this pass needs.
 -- MipsAtom_ entries have kind="atom"; MipsCode code_<name> entries have kind="raw_atom".
+-- `body_tokens` is set by scan-source on every `scan.atoms[i]` / `scan.raw_atoms[i]`; we carry it forward
+-- so `scan_atom_body` reads from the precomputed table directly (no per-atom tokenize_body fallback).
 -- @param scan table  -- SourceScan from duffle.scan_source
--- @return table[]  -- list of {name=, body=}
+-- @return table[]  -- list of {name=, body=, body_tokens=}
 local function project_atoms(scan)
 	local out = {}
 	for _, a in ipairs(scan.atoms) do
-		out[#out + 1] = { name = a.raw_name, body = a.body }
+		out[#out + 1] = { name = a.raw_name, body = a.body, body_tokens = a.body_tokens }
 	end
 	for _, a in ipairs(scan.raw_atoms) do
-		out[#out + 1] = { name = a.name, body = a.body }
+		out[#out + 1] = { name = a.name, body = a.body, body_tokens = a.body_tokens }
 	end
 	return out
 end
@@ -374,7 +341,7 @@ local function process_source(ctx, src)
 
 	local atoms_data = {}
 	for _, atom in ipairs(atoms) do
-		local labels, branches, total = scan_atom_body(atom.body_tokens or duffle.tokenize_body(atom.body), ctx.shared.word_counts)
+		local labels, branches, total = scan_atom_body(atom.body_tokens, ctx.shared.word_counts)
 		atoms_data[#atoms_data + 1] = {
 			name        = atom.name,
 			total_words = total,

scripts/passes/static_analysis.lua

@@ -784,7 +784,7 @@ local function validate(ctx, src)
 	local findings = {}
 	for _, a in ipairs(atoms) do
 		a.paths                  = a.paths or {}
-		a.paths.tokens           = duffle.tokenize_body(a.body)
+		a.paths.tokens           = a.body_tokens
 		a.paths.line_in_body     = duffle.build_body_line_index(a.body)
 		a.paths.tok_class        = classify_tokens(a.paths.tokens)
 

scripts/passes/word_count_eval.lua

@@ -4,7 +4,6 @@
 ---   1. **Public utilities** (used by `passes/components.lua`, `passes/offsets.lua`, `passes/annotation.lua`):
 ---        - `M.count_token_words(token, wc)` — words emitted by one token
 ---        - `M.scan_dir(dir, suffix)` — glob walk for *.macs.h
----        - `M.count_body_words(body, wc)` — words emitted by an atom body
 ---   2. **Pass entry** `M.run(ctx)` — loads metadata.h + *.macs.h into `ctx.shared.word_counts` for downstream passes.
 ---   3. **Internal helpers** for the body scanner.
 ---
@@ -178,88 +177,6 @@ end
 --- Invalidate the scan cache (call after creating new .macs.h files in the same Lua process — usually not needed).
 function M._invalidate_scan_cache() package.loaded[SCAN_CACHE_KEY] = nil end
 
--- ┌────────────────────────────────────────────────────────────────────┐
--- │ Shared utility: count_body_words                                   │
--- └────────────────────────────────────────────────────────────────────┘
-
---- Count words emitted by an entire atom body (a brace-delimited block).
---- Splits by top-level commas; for each token, delegates to count_token_words.
---- Handles `atom_label(name)` / `atom_offset(tag, name)` markers 
---- (record at current pos, do NOT advance pos; if the marker call bundles an instruction after it, count that instruction too).
----
---- @param body string         -- brace-delimited atom body (without braces)
---- @param wc WordCounts       -- the shared word-count table
---- @return integer            -- total words
-function M.count_body_words(body, wc)
-	local total = 0
-	for _, tok in ipairs(duffle.split_top_level_commas(body)) do
-		local pos     = 1
-		local tok_len = #tok
-		while pos <= tok_len and duffle.is_space(tok:sub(pos, pos)) do
-			pos = pos + 1
-		end
-		local leading_ident = duffle.read_ident(tok, pos)
-		local is_marker = leading_ident == "atom_label" or leading_ident == "atom_offset"
-		if is_marker then
-			-- Marker call: record at current pos, do NOT advance pos.
-			-- But the source pattern may bundle the marker with the next instruction on a new line (no top-level comma between them).
-			-- In that case, the rest of `tok` after the marker call is a real instruction that must still be counted.
-			local marker_end = M.find_marker_call_end(tok)
-			if marker_end > 0 and marker_end < #tok then
-				local rest = duffle.trim(tok:sub(marker_end + 1))
-				if rest ~= "" then
-					total = total + M.count_token_words(rest, wc)
-				end
-			end
-		else
-			total = total + M.count_token_words(tok, wc)
-		end
-	end
-	return total
-end
-
---- Find the end position (just past the closing ')') of the first atom_label/atom_offset call in `tok`. Returns 0 if no such call.
---- Internal helper for count_body_words.
----
---- @param tok string
---- @return integer  -- 0 if no marker call found
-function M.find_marker_call_end(tok)
-	local pos     = 1
-	local tok_len = #tok
-	while pos <= tok_len do
-		pos = duffle.skip_ws_and_cmt(tok, pos)
-		if pos > tok_len then break end
-		local ch = tok:sub(pos, pos)
-		if duffle.is_space(ch) then
-			pos = pos + 1
-		elseif ch == "/" then
-			-- comment — skip past it (delegated to duffle.skip_str_or_cmt)
-			local nx = duffle.skip_str_or_cmt(tok, pos)
-			pos      = (nx > pos) and nx or (pos + 1)
-		else
-			local ident, after_ident = duffle.read_ident(tok, pos)
-			local marker_end         = find_marker_end(tok, ident, after_ident)
-			if marker_end > 0 then return marker_end end
-			pos = after_ident or (pos + 1)
-		end
-	end
-	return 0
-end
-
--- (internal) If `ident` is `atom_label`/`atom_offset` followed by `(...)`, return the position just past the closing ')'. 
--- Otherwise 0.
--- @param tok string
--- @param ident string|nil
--- @param after_ident integer
--- @return integer
-local function find_marker_end(tok, ident, after_ident)
-	if ident ~= "atom_label" and ident ~= "atom_offset" then return 0 end
-	local open_paren = duffle.skip_ws_and_cmt(tok, after_ident)
-	if tok:sub(open_paren, open_paren) ~= "(" then return 0 end
-	local _, end_paren = duffle.read_parens(tok, open_paren)
-	return end_paren - 1
-end
-
 -- ┌────────────────────────────────────────────────────────────────────┐
 -- │ Pass entry: M.run(ctx) — "word-counts" pass                        │
 -- └────────────────────────────────────────────────────────────────────┘