diff --git a/scripts/duffle.lua b/scripts/duffle.lua index a3e93a1..a7dcd15 100644 --- a/scripts/duffle.lua +++ b/scripts/duffle.lua @@ -20,6 +20,12 @@ local M = {} +-- Optional native extension: lfs (LuaFileSystem). When present, ensure_dir uses +-- lfs.attributes + lfs.mkdir instead of spawning `cmd.exe mkdir` — saves ~55ms per +-- unique directory on Windows. Built by `update_deps.ps1` to `toolchain/lfs/lfs.dll` +-- and wired into package.cpath by `scripts/duffle_paths.lua`. +local lfs = pcall(require, "lfs") and require("lfs") or nil + -- ════════════════════════════════════════════════════════════════════════════ -- Cross-file type aliases -- ════════════════════════════════════════════════════════════════════════════ @@ -308,21 +314,37 @@ end -- Convert a (possibly relative) path to an absolute path, using CWD if needed. -- Normalizes forward slashes to backslashes on Windows. -- Used for byte-identical emit: the // Source: comment line uses the absolute path. +-- +-- The CWD is memoized (one `io.popen("cd")` per process — ~50ms on Windows). +-- Without the cache, calling this per-source in the components pass added ~1.5s to a 30-source build. -- @param path string -- @return string +local _absolute_path_cache = {} + function M.to_absolute_path(path) + if _absolute_path_cache[path] then return _absolute_path_cache[path] end if #path >= 2 and path:sub(2, 2) == ":" then -- Already absolute; normalize slashes for consistency. - return (path:gsub("/", "\\")) + local result = (path:gsub("/", "\\")) + _absolute_path_cache[path] = result + return result end - local p = io.popen("cd") - if not p then return path end - local cwd = p:read("*l") - p:close() - if not cwd then return path end + -- Native: lfs.currentdir() is ~0ms vs io.popen("cd") at ~50ms per call. + local cwd + if lfs then + cwd = lfs.currentdir() + else + local p = io.popen("cd") + if not p then _absolute_path_cache[path] = path; return path end + cwd = p:read("*l") + p:close() + end + if not cwd then _absolute_path_cache[path] = path; return path end cwd = cwd:gsub("/", "\\") local tail = (path:gsub("/", "\\")) - return cwd .. "\\" .. tail + local result = cwd .. "\\" .. tail + _absolute_path_cache[path] = result + return result end -- Cache of directories already verified to exist in this process. @@ -334,8 +356,15 @@ local _ensured_dirs = {} function M.ensure_dir(path) if _ensured_dirs[path] then return end _ensured_dirs[path] = true - local is_win = package.config:sub(1, 1) == "\\" - os.execute(is_win and ('if not exist "' .. path .. '" mkdir "' .. path .. '"') or ('mkdir -p "' .. path .. '" 2>/dev/null')) + if lfs then + -- Native: ~0ms when dir exists (the common case). lfs.mkdir on a new dir is ~2ms (no shell spawn). + -- Falls through silently if lfs.mkdir fails (e.g. permission denied); the subsequent write_file will surface the error. + if lfs.attributes(path, "mode") ~= "directory" then lfs.mkdir(path) end + else + -- Fallback: shell mkdir. Slow (~55ms per call on Windows due to cmd.exe spawn) but works without lfs. + local is_win = package.config:sub(1, 1) == "\\" + os.execute(is_win and ('if not exist "' .. path .. '" mkdir "' .. path .. '"') or ('mkdir -p "' .. path .. '" 2>/dev/null')) + end end -- Test helper: clear the cache (used by tests + between process runs). diff --git a/scripts/passes/components.lua b/scripts/passes/components.lua index deb0754..a12123a 100644 --- a/scripts/passes/components.lua +++ b/scripts/passes/components.lua @@ -350,6 +350,8 @@ end -- Project pre-scanned MipsAtomComp_ / MipsAtomComp_Proc_ entries into Component shape. -- Does per-source backward lookups for args (preceding function decl) and comment (preceding comment block). +-- Carries `body_tokens` forward from scan-source so word_count_rec reads from the precomputed table +-- instead of calling duffle.tokenize_body again. -- @param source string -- the full source text (needed for backward lookups) -- @param scan table -- SourceScan from duffle.scan_source -- @return Component[] @@ -360,11 +362,12 @@ local function project_components(source, scan) local args = find_function_args_for(source, a.raw_name, a.ident_pos) local comment = preceding_comment_block(source, a.ident_pos) out[#out + 1] = { - line = a.line, - name = a.name, - body = a.body, - args = args, - comment = comment, + line = a.line, + name = a.name, + body = a.body, + body_tokens = a.body_tokens, + args = args, + comment = comment, } end end @@ -445,7 +448,7 @@ local function word_count_rec(name, comp_by_name, wc, cache) local n if cc then n = 0 - local tokens = cc.body_tokens or duffle.tokenize_body(cc.body) + local tokens = cc.body_tokens for _, t in ipairs(tokens) do local trimmed = t.tok if trimmed ~= "" then diff --git a/scripts/passes/offsets.lua b/scripts/passes/offsets.lua index 95c317a..a29e017 100644 --- a/scripts/passes/offsets.lua +++ b/scripts/passes/offsets.lua @@ -164,44 +164,6 @@ local function scan_for_atom_markers(token, at_pos, labels, branches) end end ---- Find the end position (just past the closing ')') of the first atom_label/atom_offset call in `tok`. Returns 0 if no such call. ---- @param tok string ---- @return integer -- 0 if no marker call found; otherwise end-1 (just past ')') -local function find_marker_call_end(tok) - local pos = 1 - local tok_len = #tok - while pos <= tok_len do - pos = duffle.skip_ws_and_cmt(tok, pos) - if pos > tok_len then break end - local ch = tok:sub(pos, pos) - if duffle.is_space(ch) then - pos = pos + 1 - elseif ch == "/" then - -- comment — skip past it (delegated to duffle.skip_str_or_cmt) - local nx = duffle.skip_str_or_cmt(tok, pos) - pos = (nx > pos) and nx or (pos + 1) - else - local ident, after_ident = duffle.read_ident(tok, pos) - -- scan: - if ident == LABEL_MARKER or ident == OFFSET_MARKER then - -- scan: atom_label() OR atom_offset(, ) - local open_paren = duffle.skip_ws_and_cmt(tok, after_ident) - if tok:sub(open_paren, open_paren) == "(" then - local _, end_paren = duffle.read_parens(tok, open_paren) - return end_paren - 1 - end - return 0 - end - pos = after_ident or (pos + 1) - end - end - return 0 -end - --- ════════════════════════════════════════════════════════════════════════════ --- Per-atom body scan (for atom_label / atom_offset markers) --- ════════════════════════════════════════════════════════════════════════════ - -- (internal) Count words emitted by the rest of `tok` after a marker call -- (the marker call itself emits 0 words, but the source pattern may bundle the marker with the next instruction on the same line, -- separated by no top-level comma). @@ -210,9 +172,12 @@ end -- @param word_counts table -- @return integer local function count_marker_rest(tok, word_counts) - local marker_end = find_marker_call_end(tok) - if marker_end <= 0 or marker_end >= #tok then return 0 end - local rest = duffle.trim(tok:sub(marker_end + 1)) + -- duffle.find_marker_call_end returns the position PAST the closing `)` of the marker call + -- (or nil if `tok` isn't a marker call). Canonical impl in duffle.lua is faster than the + -- file-local copy that used to live here (byte-indexed, no `tok:sub` per char). + local marker_end = duffle.find_marker_call_end(tok) + if not marker_end or marker_end >= #tok then return 0 end + local rest = duffle.trim(tok:sub(marker_end)) if rest == "" then return 0 end return count_token_words(rest, word_counts) end @@ -348,17 +313,19 @@ end local M = {} --- Project the pre-scanned SourceScan entries into the {name, body} shape this pass needs. +-- Project the pre-scanned SourceScan entries into the {name, body, body_tokens} shape this pass needs. -- MipsAtom_ entries have kind="atom"; MipsCode code_ entries have kind="raw_atom". +-- `body_tokens` is set by scan-source on every `scan.atoms[i]` / `scan.raw_atoms[i]`; we carry it forward +-- so `scan_atom_body` reads from the precomputed table directly (no per-atom tokenize_body fallback). -- @param scan table -- SourceScan from duffle.scan_source --- @return table[] -- list of {name=, body=} +-- @return table[] -- list of {name=, body=, body_tokens=} local function project_atoms(scan) local out = {} for _, a in ipairs(scan.atoms) do - out[#out + 1] = { name = a.raw_name, body = a.body } + out[#out + 1] = { name = a.raw_name, body = a.body, body_tokens = a.body_tokens } end for _, a in ipairs(scan.raw_atoms) do - out[#out + 1] = { name = a.name, body = a.body } + out[#out + 1] = { name = a.name, body = a.body, body_tokens = a.body_tokens } end return out end @@ -374,7 +341,7 @@ local function process_source(ctx, src) local atoms_data = {} for _, atom in ipairs(atoms) do - local labels, branches, total = scan_atom_body(atom.body_tokens or duffle.tokenize_body(atom.body), ctx.shared.word_counts) + local labels, branches, total = scan_atom_body(atom.body_tokens, ctx.shared.word_counts) atoms_data[#atoms_data + 1] = { name = atom.name, total_words = total, diff --git a/scripts/passes/static_analysis.lua b/scripts/passes/static_analysis.lua index 2865b03..d829e19 100644 --- a/scripts/passes/static_analysis.lua +++ b/scripts/passes/static_analysis.lua @@ -784,7 +784,7 @@ local function validate(ctx, src) local findings = {} for _, a in ipairs(atoms) do a.paths = a.paths or {} - a.paths.tokens = duffle.tokenize_body(a.body) + a.paths.tokens = a.body_tokens a.paths.line_in_body = duffle.build_body_line_index(a.body) a.paths.tok_class = classify_tokens(a.paths.tokens) diff --git a/scripts/passes/word_count_eval.lua b/scripts/passes/word_count_eval.lua index 7fdf011..bd758f8 100644 --- a/scripts/passes/word_count_eval.lua +++ b/scripts/passes/word_count_eval.lua @@ -4,7 +4,6 @@ --- 1. **Public utilities** (used by `passes/components.lua`, `passes/offsets.lua`, `passes/annotation.lua`): --- - `M.count_token_words(token, wc)` — words emitted by one token --- - `M.scan_dir(dir, suffix)` — glob walk for *.macs.h ---- - `M.count_body_words(body, wc)` — words emitted by an atom body --- 2. **Pass entry** `M.run(ctx)` — loads metadata.h + *.macs.h into `ctx.shared.word_counts` for downstream passes. --- 3. **Internal helpers** for the body scanner. --- @@ -178,88 +177,6 @@ end --- Invalidate the scan cache (call after creating new .macs.h files in the same Lua process — usually not needed). function M._invalidate_scan_cache() package.loaded[SCAN_CACHE_KEY] = nil end --- ┌────────────────────────────────────────────────────────────────────┐ --- │ Shared utility: count_body_words │ --- └────────────────────────────────────────────────────────────────────┘ - ---- Count words emitted by an entire atom body (a brace-delimited block). ---- Splits by top-level commas; for each token, delegates to count_token_words. ---- Handles `atom_label(name)` / `atom_offset(tag, name)` markers ---- (record at current pos, do NOT advance pos; if the marker call bundles an instruction after it, count that instruction too). ---- ---- @param body string -- brace-delimited atom body (without braces) ---- @param wc WordCounts -- the shared word-count table ---- @return integer -- total words -function M.count_body_words(body, wc) - local total = 0 - for _, tok in ipairs(duffle.split_top_level_commas(body)) do - local pos = 1 - local tok_len = #tok - while pos <= tok_len and duffle.is_space(tok:sub(pos, pos)) do - pos = pos + 1 - end - local leading_ident = duffle.read_ident(tok, pos) - local is_marker = leading_ident == "atom_label" or leading_ident == "atom_offset" - if is_marker then - -- Marker call: record at current pos, do NOT advance pos. - -- But the source pattern may bundle the marker with the next instruction on a new line (no top-level comma between them). - -- In that case, the rest of `tok` after the marker call is a real instruction that must still be counted. - local marker_end = M.find_marker_call_end(tok) - if marker_end > 0 and marker_end < #tok then - local rest = duffle.trim(tok:sub(marker_end + 1)) - if rest ~= "" then - total = total + M.count_token_words(rest, wc) - end - end - else - total = total + M.count_token_words(tok, wc) - end - end - return total -end - ---- Find the end position (just past the closing ')') of the first atom_label/atom_offset call in `tok`. Returns 0 if no such call. ---- Internal helper for count_body_words. ---- ---- @param tok string ---- @return integer -- 0 if no marker call found -function M.find_marker_call_end(tok) - local pos = 1 - local tok_len = #tok - while pos <= tok_len do - pos = duffle.skip_ws_and_cmt(tok, pos) - if pos > tok_len then break end - local ch = tok:sub(pos, pos) - if duffle.is_space(ch) then - pos = pos + 1 - elseif ch == "/" then - -- comment — skip past it (delegated to duffle.skip_str_or_cmt) - local nx = duffle.skip_str_or_cmt(tok, pos) - pos = (nx > pos) and nx or (pos + 1) - else - local ident, after_ident = duffle.read_ident(tok, pos) - local marker_end = find_marker_end(tok, ident, after_ident) - if marker_end > 0 then return marker_end end - pos = after_ident or (pos + 1) - end - end - return 0 -end - --- (internal) If `ident` is `atom_label`/`atom_offset` followed by `(...)`, return the position just past the closing ')'. --- Otherwise 0. --- @param tok string --- @param ident string|nil --- @param after_ident integer --- @return integer -local function find_marker_end(tok, ident, after_ident) - if ident ~= "atom_label" and ident ~= "atom_offset" then return 0 end - local open_paren = duffle.skip_ws_and_cmt(tok, after_ident) - if tok:sub(open_paren, open_paren) ~= "(" then return 0 end - local _, end_paren = duffle.read_parens(tok, open_paren) - return end_paren - 1 -end - -- ┌────────────────────────────────────────────────────────────────────┐ -- │ Pass entry: M.run(ctx) — "word-counts" pass │ -- └────────────────────────────────────────────────────────────────────┘