metaprogram improvements

scripts/build_psyq.ps1

@@ -343,7 +343,7 @@ function generate-TapeAtomOffsets {param([Parameter(Mandatory=$true)] [string[]]
     }
 
     write-host "AtomOffsets  $($sources.Count) source(s)" -ForegroundColor Magenta
-    & lua $gen_atom_offsets_script $metadata @sources
+    & luajit $gen_atom_offsets_script $metadata @sources
     if ($LASTEXITCODE -ne 0) {
         write-error "Atom offset generation failed. Aborting."
         exit 1
@@ -382,7 +382,7 @@ function generate-TapeAtomAnnotations {param([Parameter(Mandatory=$true)] [strin
     }
 
     write-host "AtomAnnotations $($sources.Count) source(s)" -ForegroundColor Magenta
-    & lua $gen_atom_annot_script $metadata @sources
+    & luajit $gen_atom_annot_script $metadata @sources
     if ($LASTEXITCODE -ne 0) {
         write-error "Atom annotation generation failed. Aborting."
         exit 1

scripts/duffle.lua

@@ -0,0 +1,478 @@
+-- duffle.lua
+--
+-- Shared primitives + domain tables for the tape-atom metaprograms.
+-- Both `tape_atom_annotation_pass.lua` and `tape_atom.offset_gen.meta.lua`
+-- `require("duffle")` for these.
+--
+-- 5.3-compatible Lua (no 5.4/5.5-only features):
+--   - no <close> / <toclose>
+--   - no continue keyword
+--   - no string.dump improvements
+--   - LuaJIT 5.1+extensions model is the primary target
+--
+-- No :match / :gmatch regex use anywhere; all delimiter-splitting is
+-- hand-rolled or via LPeg (the regex-free PEG library).
+--
+-- Phase 3: the hot lexer primitives are LPeg-backed where it pays off.
+-- The hand-rolled variants remain for callers that need a fallback.
+
+local M = {}
+
+-- ════════════════════════════════════════════════════════════════════════════
+-- Section 0: LPeg patterns (compiled once at module load)
+-- ════════════════════════════════════════════════════════════════════════════
+--
+-- LPeg is a PEG library (no regex). All patterns below are first-class
+-- pattern values; they're cheap to build and reuse.
+--
+-- Note: lpeg is required lazily because Lua 5.5 may not have it on its
+-- cpath at the same location as LuaJIT. We attempt the require and fall
+-- back to the hand-rolled implementations if it fails.
+
+local lpeg_ok, lpeg = pcall(require, "lpeg")
+local lpeg_lib = nil
+local lpeg_alpha_pat, lpeg_alnum_pat, lpeg_ident_pat
+local lpeg_str_or_cmt_pat, lpeg_ws_and_cmt_pat
+local lpeg_scan_to_target_pat  -- generic "anything but target or balanced group" matcher
+
+if lpeg_ok then
+    lpeg_lib = lpeg
+    local P, S, R = lpeg.P, lpeg.S, lpeg.R
+
+    -- Character class patterns
+    local alpha_pat = R("AZ", "az") + P("_")
+    local digit_pat = R("09")
+    lpeg_alnum_pat = alpha_pat + digit_pat
+
+    -- Identifier: alpha followed by zero+ alnum. Capture as a string.
+    lpeg_alpha_pat = alpha_pat
+    lpeg_ident_pat = lpeg.C(alpha_pat * lpeg_alnum_pat^0)
+
+    -- String literal: "..." with backslash escapes.
+    local lpeg_str_pat = P('"') * (P(1) - S('"\\') + P('\\') * P(1))^0 * P('"')
+    -- Char literal: '...' with backslash escapes.
+    local lpeg_chr_pat = P("'") * (P(1) - S("'\\") + P('\\') * P(1))^0 * P("'")
+    -- Line comment: // ... to end-of-line.
+    local lpeg_line_cmt_pat = P("//") * (P(1) - S("\n"))^0
+    -- Block comment: /* ... */ (no nesting per C standard).
+    local lpeg_block_cmt_pat = P("/*") * (P(1) - P("*/"))^0 * P("*/")
+    -- String or comment (any of the four forms).
+    lpeg_str_or_cmt_pat = lpeg_str_pat + lpeg_chr_pat + lpeg_line_cmt_pat + lpeg_block_cmt_pat
+
+    -- Whitespace + comment skipper: zero+ (whitespace run | string | comment).
+    local ws_pat = S(" \t\n\r\v\f")
+    lpeg_ws_and_cmt_pat = (ws_pat + lpeg_str_or_cmt_pat)^0
+
+    -- Generic "skip until target, but step over balanced groups" matcher.
+    -- Used by scan_to_char for non-ident / non-bracket chars.
+    -- We accept any single char except the target.
+    -- The balanced-group stepping is handled by the caller (via read_balanced).
+    lpeg_scan_to_target_pat = function(target)
+        return (P(1) - P(target))^0
+    end
+end
+
+-- ════════════════════════════════════════════════════════════════════════════
+-- Section 1: character classification (byte-based for hot loops)
+-- ════════════════════════════════════════════════════════════════════════════
+--
+-- Two APIs:
+--   is_space(c), is_alpha(c), etc. — accept a single-char STRING (legacy)
+--   is_space_byte(b), is_alpha_byte(b), etc. — accept a single-byte INTEGER
+--
+-- The byte-based versions are 5-10x faster in tight loops because they
+-- avoid the string allocation per s:sub(i, i) call.
+
+-- Whitespace characters per C locale.
+function M.is_space_byte(b)
+    return b == 32  or b == 9 or b == 10 or b == 13 or b == 11 or b == 12
+end
+
+-- Letters (a-z, A-Z) and underscore.
+function M.is_alpha_byte(b)
+    if not b then return false end
+    if b >= 97 and b <= 122 then return true  end  -- 'a'..'z'
+    if b >= 65 and b <= 90  then return true  end  -- 'A'..'Z'
+    return b == 95                                    -- '_'
+end
+
+-- Single digit.
+function M.is_digit_byte(b) return b and b >= 48 and b <= 57 end
+
+-- Letter OR digit OR underscore.
+function M.is_alnum_byte(b) return M.is_alpha_byte(b) or M.is_digit_byte(b) end
+
+-- String-based wrappers (kept for callers that already have a single-char
+-- string; the byte versions are what the hot loops should call).
+function M.is_space(c)
+    if type(c) == "number" then return M.is_space_byte(c) end
+    return c == " "  or c == "\t" or c == "\n" or c == "\r" or c == "\v" or c == "\f"
+end
+function M.is_alpha(c)
+    if type(c) == "number" then return M.is_alpha_byte(c) end
+    if not c or #c == 0 then return false end
+    if c >= "a" and c <= "z" then return true  end
+    if c >= "A" and c <= "Z" then return true  end
+    return c == "_"
+end
+function M.is_digit(c)
+    if type(c) == "number" then return M.is_digit_byte(c) end
+    return c and c >= "0" and c <= "9"
+end
+function M.is_alnum(c) return M.is_alpha(c) or M.is_digit(c) end
+
+-- ════════════════════════════════════════════════════════════════════════════
+-- Section 2: string primitives
+-- ════════════════════════════════════════════════════════════════════════════
+
+-- Trim leading and trailing whitespace from a string.
+function M.trim(s)
+    local a = 1
+    while a <= #s and M.is_space_byte(s:byte(a)) do a = a + 1 end
+    local b = #s
+    while b >= a and M.is_space_byte(s:byte(b)) do b = b - 1 end
+    return s:sub(a, b)
+end
+
+-- Linear-search for a single-byte target in a string.
+-- (Phase 3 retained this for places where LPeg is overkill.)
+function M.find_byte(haystack, target, start)
+    for i = start or 1, #haystack do
+        if haystack:byte(i) == target then return i end
+    end
+    return nil
+end
+
+-- Returns the directory portion of a path.
+function M.dirname(path)
+    local last_sep = 0
+    for i = 1, #path do
+        if path:byte(i) == 47 or path:byte(i) == 92 then last_sep = i end  -- '/' or '\\'
+    end
+    if last_sep == 0 then return "." end
+    return path:sub(1, last_sep - 1)
+end
+
+-- Returns the basename of a path, with the file extension stripped.
+function M.basename_no_ext(path)
+    local last_sep = 0
+    for i = 1, #path do
+        if path:byte(i) == 47 or path:byte(i) == 92 then last_sep = i end
+    end
+    local a = last_sep + 1
+    local last_dot = #path + 1
+    for i = #path, a, -1 do
+        if path:byte(i) == 46 then last_dot = i; break end  -- '.'
+    end
+    return path:sub(a, last_dot - 1)
+end
+
+-- ════════════════════════════════════════════════════════════════════════════
+-- Section 3: I/O primitives
+-- ════════════════════════════════════════════════════════════════════════════
+
+function M.read_file(path)
+    local f = io.open(path, "r")
+    if not f then error("Cannot open " .. path) end
+    local content = f:read("*a")
+    f:close()
+    return content
+end
+
+function M.write_file(path, content)
+    local f = io.open(path, "w")
+    if not f then error("Cannot write " .. path) end
+    f:write(content)
+    f:close()
+end
+
+function M.ensure_dir(path)
+    local is_win = package.config:sub(1, 1) == "\\"
+    os.execute(is_win and ('if not exist "' .. path .. '" mkdir "' .. path .. '"')
+                  or ('mkdir -p "' .. path .. '" 2>/dev/null'))
+end
+
+-- ════════════════════════════════════════════════════════════════════════════
+-- Section 4: C-language scanner primitives
+-- ════════════════════════════════════════════════════════════════════════════
+
+-- LPeg-backed skipper when LPeg is available, hand-rolled fallback otherwise.
+-- Returns position just past the construct, or `i` unchanged if no
+-- string/comment starts at position i.
+function M.skip_str_or_cmt(s, i)
+    if lpeg_ok then
+        local new_pos = lpeg.match(lpeg_str_or_cmt_pat, s, i)
+        if new_pos then return new_pos end
+        return i
+    end
+    -- Hand-rolled fallback (kept for builds where LPeg isn't available).
+    local c = s:byte(i)
+    if c == 34 or c == 39 then  -- '"' or '\''
+        i = i + 1
+        while i <= #s do
+            local b = s:byte(i)
+            if b == 92 then i = i + 2  -- '\\'
+            elseif b == c then return i + 1
+            else i = i + 1 end
+        end
+        return #s + 1
+    elseif c == 47 then  -- '/'
+        local nx = s:byte(i + 1)
+        if nx == 47 then  -- '//'
+            while i <= #s and s:byte(i) ~= 10 do i = i + 1 end
+            return i
+        elseif nx == 42 then  -- '/*'
+            i = i + 2
+            while i <= #s - 1 do
+                if s:byte(i) == 42 and s:byte(i + 1) == 47 then  -- '*/'
+                    return i + 2
+                end
+                i = i + 1
+            end
+            return #s + 1
+        end
+    end
+    return i
+end
+
+-- Skip whitespace AND C-style comments starting at position i.
+-- LPeg-backed when available; ~5-10x faster than the hand-rolled version.
+function M.skip_ws_and_cmt(s, i)
+    if lpeg_ok then
+        local new_pos = lpeg.match(lpeg_ws_and_cmt_pat, s, i)
+        if new_pos then return new_pos end
+        return i
+    end
+    -- Hand-rolled fallback.
+    local len = #s
+    while i <= len do
+        if M.is_space_byte(s:byte(i)) then
+            i = i + 1
+        else
+            local nx = M.skip_str_or_cmt(s, i)
+            if nx > i then i = nx else break end
+        end
+    end
+    return i
+end
+
+-- Read a C-style identifier (alpha followed by zero+ alnum) starting at
+-- position i. Returns the identifier string + the position just past it,
+-- or nil + i if no identifier starts here.
+function M.read_ident(s, i)
+    if lpeg_ok then
+        local result = lpeg.match(lpeg_ident_pat, s, i)
+        if result then return result, i + #result end
+        return nil, i
+    end
+    -- Hand-rolled fallback.
+    if not M.is_alpha_byte(s:byte(i)) then return nil, i end
+    local a = i
+    i = i + 1
+    while i <= #s and M.is_alnum_byte(s:byte(i)) do i = i + 1 end
+    return s:sub(a, i - 1), i
+end
+
+-- Read a balanced-delimited group (parens, braces, or brackets) starting
+-- at position i. Returns the inner content (between the delimiters) +
+-- the position just past the closing delimiter, or nil + i if `s[i]`
+-- isn't `open_char`.
+--
+-- (Hand-rolled; the depth counting makes pure LPeg awkward here.)
+function M.read_balanced(s, open_char, close_char, i)
+    local open_byte = open_char:byte()
+    if s:byte(i) ~= open_byte then return nil, i end
+    i = i + 1
+    local len   = #s
+    local depth = 1
+    local a     = i
+    while i <= len and depth > 0 do
+        local c = s:byte(i)
+        if c == open_byte then
+            depth = depth + 1
+            i = i + 1
+        elseif c == close_char:byte() then
+            depth = depth - 1
+            if depth == 0 then break end
+            i = i + 1
+        else
+            local nx = M.skip_str_or_cmt(s, i)
+            if nx > i then i = nx else i = i + 1 end
+        end
+    end
+    return s:sub(a, i - 1), i + 1
+end
+
+-- Convenience specializations of read_balanced.
+M.read_parens   = function(s, i) return M.read_balanced(s, "(", ")", i) end
+M.read_braces   = function(s, i) return M.read_balanced(s, "{", "}", i) end
+M.read_brackets = function(s, i) return M.read_balanced(s, "[", "]", i) end
+
+-- Scan forward from position `start` until we find a specific single byte
+-- `target`, transparently stepping over balanced parens/braces/brackets.
+-- Returns the position of `target`, or nil if not found.
+function M.scan_to_char(s, target, start)
+    local target_byte = target:byte()
+    local i = start
+    while i <= #s do
+        local c = s:byte(i)
+        if c == target_byte then return i end
+        if c == 40 then local _, a = M.read_balanced(s, "(", ")", i); i = a  -- '('
+        elseif c == 123 then local _, a = M.read_balanced(s, "{", "}", i); i = a  -- '{'
+        elseif c == 91 then local _, a = M.read_balanced(s, "[", "]", i); i = a  -- '['
+        else
+            local nx = M.skip_str_or_cmt(s, i)
+            if nx > i then i = nx else i = i + 1 end
+        end
+    end
+    return nil
+end
+
+-- Split a brace-body into top-level comma-separated tokens. Honors nested
+-- parens/braces/brackets and skips strings/comments.
+function M.split_top_level_commas(body)
+    local tokens = {}
+    local i = 1
+    local token_start = 1
+    while i <= #body do
+        local c = body:byte(i)
+        if c == 40 then local _, a = M.read_parens(body, i); i = a    -- '('
+        elseif c == 123 then local _, a = M.read_braces(body, i); i = a  -- '{'
+        elseif c == 91 then local _, a = M.read_brackets(body, i); i = a  -- '['
+        elseif c == 44 then  -- ','
+            tokens[#tokens + 1] = body:sub(token_start, i - 1)
+            i = i + 1
+            token_start = i
+        else
+            local nx = M.skip_str_or_cmt(body, i)
+            if nx > i then i = nx; token_start = nx else i = i + 1 end
+        end
+    end
+    local last = body:sub(token_start)
+    if M.trim(last) ~= "" then tokens[#tokens + 1] = last end
+    return tokens
+end
+
+-- ════════════════════════════════════════════════════════════════════════════
+-- Section 5: load_word_counts
+-- ════════════════════════════════════════════════════════════════════════════
+
+function M.load_word_counts(metadata_path)
+    local counts  = {}
+    local content = M.read_file(metadata_path)
+    local len     = #content
+    local i       = 1
+    local prefix  = "WORD_COUNT("
+    while i <= len do
+        local nl       = M.find_byte(content, 10, i)  -- '\n'
+        local line_end = nl or (len + 1)
+        local line     = content:sub(i, line_end - 1)
+        local trimmed  = M.trim(line)
+        if trimmed:sub(1, #prefix) == prefix and trimmed:sub(-1) == ")" then
+            local inner = trimmed:sub(#prefix + 1, #trimmed - 1)
+            local comma = M.find_byte(inner, 44, 1)  -- ','
+            if comma then
+                counts[M.trim(inner:sub(1, comma - 1))] =
+                    tonumber(M.trim(inner:sub(comma + 1)))
+            end
+        end
+        i = line_end + 1
+    end
+    return counts
+end
+
+-- ════════════════════════════════════════════════════════════════════════════
+-- Section 6: LineIndex (perf fix — replaces the per-call rescan line_of)
+-- ════════════════════════════════════════════════════════════════════════════
+
+function M.LineIndex(source)
+    local positions = {}
+    local n = 0
+    for i = 1, #source do
+        if source:byte(i) == 10 then  -- '\n'
+            n = n + 1
+            positions[n] = i
+        end
+    end
+    local function line_of(pos)
+        local lo, hi = 1, n
+        while lo <= hi do
+            local mid = math.floor((lo + hi) / 2)
+            if positions[mid] <= pos then
+                lo = mid + 1
+            else
+                hi = mid - 1
+            end
+        end
+        return hi + 1
+    end
+    return line_of
+end
+
+-- ════════════════════════════════════════════════════════════════════════════
+-- Section 7: domain tables
+-- ════════════════════════════════════════════════════════════════════════════
+
+M.WAVE_CONTEXT_REGS = {
+    ["R_PrimCursor"] = { alias = "R_T7", size = 4, role = "output cursor (prim arena)" },
+    ["R_FaceCursor"] = { alias = "R_T4", size = 4, role = "input cursor (face array)" },
+    ["R_VertBase"]   = { alias = "R_T5", size = 4, role = "base pointer (vertex array)" },
+    ["R_OtBase"]     = { alias = "R_T6", size = 4, role = "base pointer (ordering table)" },
+}
+
+M.MACRO_EXPANSION = {
+    ["phase_init"]      = "init",
+    ["phase_bind"]      = "bind",
+    ["phase_setup"]     = "setup",
+    ["phase_work"]      = "work",
+    ["phase_commit"]    = "commit",
+    ["phase_terminate"] = "terminate",
+    ["REGION_PRIM_ARENA"]   = "prim_arena",
+    ["REGION_FACE_ARENA"]   = "face_arena",
+    ["REGION_VERTEX_ARENA"] = "vertex_arena",
+    ["REGION_OT_ARENA"]     = "ot_arena",
+    ["REGION_HEAP_3D"]      = "heap_3d_models",
+    ["REGION_CDROM_STREAM"] = "cdrom_stream",
+    ["REGION_VRAM"]         = "vram_heap",
+    ["CADENCE_FRAME"]       = "frame",
+    ["CADENCE_ONCE"]        = "once",
+    ["CADENCE_ONDEMAND"]    = "ondemand",
+}
+
+M.KNOWN_PHASES = {
+    ["init"]      = true, ["bind"]   = true, ["setup"]    = true,
+    ["work"]      = true, ["commit"] = true, ["terminate"] = true,
+}
+M.KNOWN_REGIONS = {
+    ["prim_arena"]     = true, ["face_arena"]     = true,
+    ["vertex_arena"]   = true, ["ot_arena"]       = true,
+    ["heap_3d_models"] = true, ["cdrom_stream"]   = true,
+    ["vram_heap"]      = true,
+}
+M.KNOWN_CADENCES = {
+    ["frame"]    = true, ["once"] = true, ["ondemand"] = true,
+}
+
+M.TAPE_ATOM_MACROS = {
+    ["atom_annot"]     = { kind = "work",      binds = false },
+    ["atom_bind"]      = { kind = "bind",      binds = true  },
+    ["atom_setup"]     = { kind = "setup",     binds = false },
+    ["atom_commit"]    = { kind = "commit",    binds = false },
+    ["atom_init"]      = { kind = "init",      binds = false },
+    ["atom_terminate"] = { kind = "terminate", binds = false },
+}
+
+M.ATOM_PRAGMA_KINDS = {
+    ["resource"] = { kind = "string" },
+    ["region"]   = { kind = "ident",  allowed = M.KNOWN_REGIONS },
+    ["group"]    = { kind = "ident" },
+    ["cadence"]  = { kind = "ident",  allowed = M.KNOWN_CADENCES },
+    ["async"]    = { kind = "ident",  allowed = { ["true"] = true, ["false"] = true } },
+}
+
+-- Expose the lpeg_ok flag so callers can detect the LPeg-back path.
+-- True when LPeg was successfully required and the patterns above were
+-- compiled at module load time. False when running in fallback mode.
+M.lpeg_ok = lpeg_ok
+
+return M

scripts/tape_atom.offset_gen.meta.lua

@@ -20,58 +20,50 @@
 --   #pragma endregion <basename>
 --
 -- Usage:
---   lua gen_atom_offsets.lua <metadata.h> <source1> [source2 ...]
+--   luajit gen_atom_offsets.lua <metadata.h> <source1> [source2 ...]
 
--- ============================================================
+-- Make require("duffle") resolve to the sibling duffle.lua in this dir,
--- Character classification
+-- AND make require("lpeg") find the vendored LPeg DLL in the toolchain.
--- ============================================================
+-- Both prepends are explicit (no :match / no Lua pattern — plain byte scan).
-
+local script_path = arg[0]
-local function is_space(c) return c == " " or c == "\t" or c == "\n" or c == "\r" or c == "\v" or c == "\f" end
+local last_sep = 0
-local function is_alpha(c)
+for i = 1, #script_path do
-	if not c or #c == 0      then return false end
+    local c = script_path:sub(i, i)
-	if c >= "a" and c <= "z" then return true  end
+    if c == "/" or c == "\\" then last_sep = i end
-	if c >= "A" and c <= "Z" then return true  end
-	return c == "_"
 end
+local script_dir = last_sep == 0 and "./" or script_path:sub(1, last_sep)
+package.path   = script_dir .. "?.lua;" .. script_dir .. "?/init.lua;" .. package.path
+package.cpath = "C:\\projects\\Pikuma\\ps1\\toolchain\\luajit-2.1\\lib\\lua\\5.1\\?.dll;" .. package.cpath
 
-local function is_digit(c) return c and c >= "0" and c <= "9" end
+-- Shared primitives + domain tables live in scripts/duffle.lua.
-local function is_alnum(c) return is_alpha(c) or is_digit(c) end
+local duffle = require("duffle")
+
+-- Local aliases so the rest of this file reads cleanly. These resolve
+-- to the same functions in duffle.lua (5.3-compatible, no regex).
+local is_space        = duffle.is_space
+local is_alpha        = duffle.is_alpha
+local is_alnum        = duffle.is_alnum
+local trim            = duffle.trim
+local find_byte       = duffle.find_byte
+local read_file       = duffle.read_file
+local write_file      = duffle.write_file
+local ensure_dir      = duffle.ensure_dir
+local dirname         = duffle.dirname
+local basename_no_ext = duffle.basename_no_ext
+local skip_str_or_cmt = duffle.skip_str_or_cmt
+local skip_ws_and_cmt = duffle.skip_ws_and_cmt
+local read_ident      = duffle.read_ident
+local read_parens     = duffle.read_parens
+local read_braces     = duffle.read_braces
+local read_brackets   = duffle.read_brackets
+local scan_to_char    = duffle.scan_to_char
+local split_top_level_commas = duffle.split_top_level_commas
+local load_word_counts = duffle.load_word_counts
 
 -- ============================================================
--- I/O
+-- Offset-gen-specific helpers (not in duffle.lua)
 -- ============================================================
 
-local function read_file(path)
-	local  f = io.open(path, "r")
-	if not f then error("Cannot open " .. path) end
-	local content = f:read("*a")
-	f:close()
-	return content
-end
-
-local function write_file(path, content)
-	local  f = io.open(path, "w")
-	if not f then error("Cannot write " .. path) end
-	f:write(content)
-	f:close()
-end
-
--- PowerShell aliases `mkdir` to New-Item, which treats `-p` as a path, so guard the call.
-local function ensure_dir(path)
-	local is_win = package.config:sub(1, 1) == "\\"
-	os.execute(is_win and ('if not exist "' .. path .. '" mkdir "' .. path .. '"') or ('mkdir -p "' .. path .. '" 2>/dev/null'))
-end
-
--- ============================================================
--- String primitives
--- ============================================================
-
-local function trim(s)
-	local a = 1;  while a <= #s and is_space(s:sub(a, a)) do a = a + 1 end
-	local b = #s; while b >= a  and is_space(s:sub(b, b)) do b = b - 1 end
-	return s:sub(a, b)
-end
-
 local function starts_with(s, prefix)
     if #s < #prefix then return false end
     for i = 1, #prefix do
@@ -89,37 +81,6 @@ local function ends_with(s, suffix)
     return true
 end
 
-local function find_byte(haystack, target, start)
-	for i = start or 1, #haystack do
-		if haystack:sub(i, i) == target then return i end
-	end
-	return nil
-end
-
-local function dirname(path)
-	local last_sep = 0
-	for i = 1, #path do
-		local c = path:sub(i, i)
-		if c == "/" or c == "\\" then last_sep = i end
-	end
-	if last_sep == 0 then return "." end
-	return path:sub(1, last_sep - 1)
-end
-
-local function basename_no_ext(path)
-	local last_sep = 0
-	for i = 1, #path do
-		local c = path:sub(i, i)
-		if c == "/" or c == "\\" then last_sep = i end
-	end
-	local a = last_sep + 1
-	local last_dot = #path + 1
-	for i = #path, a, -1 do
-		if path:sub(i, i) == "." then last_dot = i; break end
-	end
-	return path:sub(a, last_dot - 1)
-end
-
 local function to_upper(s) return s:upper() end
 local function to_alnum_underscore(s)
     local out = ""
@@ -132,104 +93,6 @@ local function to_alnum_underscore(s)
 end
 local function pad_right(s, w) return s .. string.rep(" ", w - #s) end
 
--- ============================================================
--- Lexer helpers
--- ============================================================
-
--- If position i starts a C string literal ("..."), char literal ('.'),
--- // line comment, or /* block comment, advance past it and return the
--- position just after the construct (or #s+1 if unterminated).
--- Otherwise return i unchanged.
-local function skip_str_or_cmt(s, i)
-	local c = s:sub(i, i)
-	if    c == '"' or c == "'" then
-		i = i + 1
-		while i <= #s do
-			if s:sub(i, i) == "\\" then i = i + 2
-			elseif s:sub(i, i) == c then return i + 1
-			else i = i + 1 end
-		end
-		return #s + 1
-	elseif c == "/" then
-		local nx = s:sub(i+1, i+1)
-		if nx == "/" then
-			while i <= #s and s:sub(i, i) ~= "\n" do i = i + 1 end
-			return i
-		elseif nx == "*" then
-			i = i + 2
-			while i <= #s - 1 do
-				if s:sub(i, i) == "*" and s:sub(i+1, i+1) == "/" then
-					return i + 2
-				end
-				i = i + 1
-			end
-			return #s + 1
-		end
-	end
-	return i
-end
-
-local function skip_ws_and_cmt(s, i)
-	while i <= #s do
-		if is_space(s:sub(i, i)) then i = i + 1
-		else
-			local nx = skip_str_or_cmt(s, i)
-			if nx > i then i = nx else break end
-		end
-	end
-	return i
-end
-
-local function read_ident(source, i)
-	if not is_alpha(source:sub(i, i)) then return nil, i end
-	local a = i
-	i = i + 1
-	while i <= #source and is_alnum(source:sub(i, i)) do i = i + 1 end
-	return source:sub(a, i - 1), i
-end
-
-local function read_balanced(s, open_char, close_char, i)
-	if s:sub(i, i) ~= open_char then return nil, i end
-	i = i + 1
-	local len   = #s
-	local depth = 1
-	local a     = i
-	while i <= len and depth > 0 do
-		local c = s:sub(i, i)
-		if c == open_char then
-			depth = depth + 1
-			i = i + 1
-		elseif c == close_char then
-			depth = depth - 1
-			if depth == 0 then break end
-			i = i + 1
-		else
-			local nx = skip_str_or_cmt(s, i)
-			if nx > i then i = nx else i = i + 1 end
-		end
-	end
-	return s:sub(a, i - 1), i + 1
-end
-
-local read_parens   = function(s, i) return read_balanced(s, "(", ")", i) end
-local read_braces   = function(s, i) return read_balanced(s, "{", "}", i) end
-local read_brackets = function(s, i) return read_balanced(s, "[", "]", i) end
-
-local function scan_to_char(s, target, start)
-	local i = start
-	while i <= #s do
-		local c = s:sub(i, i)
-		if    c == target then return i end
-		if     c == "(" then local _, a = read_balanced(s, "(", ")", i); i = a
-		elseif c == "{" then local _, a = read_balanced(s, "{", "}", i); i = a
-		elseif c == "[" then local _, a = read_balanced(s, "[", "]", i); i = a
-		else
-			local nx = skip_str_or_cmt(s, i)
-			if nx > i then i = nx else i = i + 1 end
-		end
-	end
-end
-
 -- ============================================================
 -- Extract comma-separated identifier args from a parenthesized group
 -- after a function-like macro call.
@@ -260,33 +123,6 @@ local function extract_ident_args(token, after_ident)
     return args, after_paren
 end
 
--- ============================================================
--- Load WORD_COUNT manifest
--- ============================================================
-
-local function load_word_counts(metadata_path)
-	local counts  = {}
-	local content = read_file(metadata_path)
-	local len     = #content
-	local i       = 1
-	local prefix  = "WORD_COUNT("
-	while i <= len do
-		local nl       = find_byte(content, "\n", i)
-		local line_end = nl or (len + 1)
-		local line     = content:sub(i, line_end - 1)
-		local trimmed  = trim(line)
-		if starts_with(trimmed, prefix) and ends_with(trimmed, ")") then
-			local inner = trimmed:sub(#prefix + 1, #trimmed - 1)
-			local comma = find_byte(inner, ",", 1)
-			if comma then
-				counts[trim(inner:sub(1, comma - 1))] = tonumber(trim(inner:sub(comma + 1)))
-			end
-		end
-		i = line_end + 1
-	end
-	return counts
-end
-
 -- ============================================================
 -- Count words for a single comma-separated token
 -- ============================================================
@@ -304,33 +140,6 @@ local function word_count_of_token(token, wc)
     return 1
 end
 
--- ============================================================
--- Split brace-body into top-level comma-separated tokens
--- ============================================================
-
-local function split_top_level_commas(body)
-	local tokens = {}
-	local i = 1
-	local token_start = 1
-	while i <= #body do
-		local c = body:sub(i, i)
-		if c == "(" then local _, a = read_parens(body, i); i = a
-		elseif c == "{" then local _, a = read_braces(body, i); i = a
-		elseif c == "[" then local _, a = read_brackets(body, i); i = a
-		elseif c == "," then
-			table.insert(tokens, body:sub(token_start, i - 1))
-			i = i + 1
-			token_start = i
-		else
-			local nx = skip_str_or_cmt(body, i)
-			if nx > i then i = nx else i = i + 1 end
-		end
-	end
-	local last = body:sub(token_start)
-	if trim(last) ~= "" then table.insert(tokens, last) end
-	return tokens
-end
-
 -- ============================================================
 -- Scan token for atom_label/atom_offset markers, walking through
 -- balanced groups transparently (so nested calls are found)
@@ -367,6 +176,34 @@ end
 -- Scan atom body, count words, find markers
 -- ============================================================
 
+-- Find the end position (just past the closing ')') of the first
+-- atom_label/atom_offset call in `tok`. Returns 0 if no such call.
+local function find_marker_call_end(tok)
+    local i   = 1
+    local len = #tok
+    while i <= len do
+        i = skip_ws_and_cmt(tok, i)
+        if i > len then break end
+        local c = tok:sub(i, i)
+        if is_alpha(c) then
+            local ident, after = read_ident(tok, i)
+            if ident == "atom_label" or ident == "atom_offset" then
+                local j = skip_ws_and_cmt(tok, after)
+                if tok:sub(j, j) == "(" then
+                    local _, end_paren = read_parens(tok, j)
+                    return end_paren - 1
+                end
+                return 0
+            end
+            i = after
+        else
+            local nx = skip_str_or_cmt(tok, i)
+            if nx > i then i = nx else i = i + 1 end
+        end
+    end
+    return 0
+end
+
 local function scan_atom_body(body, word_counts)
     local pos      = 0
     local labels   = {}
@@ -377,7 +214,20 @@ local function scan_atom_body(body, word_counts)
         while k <= tlen and is_space(tok:sub(k, k)) do k = k + 1 end
         local leading_ident = read_ident(tok, k)
         if leading_ident == "atom_label" or leading_ident == "atom_offset" then
+            -- Marker call: record at the current pos, do NOT advance pos.
+            -- But the source pattern may bundle the marker with the next
+            -- instruction on a new line (no top-level comma between them).
+            -- In that case, the rest of `tok` after the marker call is a
+            -- real instruction that must still be counted.
             scan_for_atom_markers(tok, pos, labels, branches)
+            local marker_end = find_marker_call_end(tok)
+            if marker_end > 0 and marker_end < #tok then
+                local rest = trim(tok:sub(marker_end + 1))
+                if rest ~= "" then
+                    local rest_words = word_count_of_token(rest, word_counts)
+                    pos = pos + rest_words
+                end
+            end
         else
             local words = word_count_of_token(tok, word_counts)
             scan_for_atom_markers(tok, pos, labels, branches)
@@ -577,7 +427,7 @@ end
 
 local function main(args)
     if #args < 2 then
-		print("Usage: gen_atom_offsets.lua <metadata.h> <source1> [source2 ...]")
+        print("Usage: luajit gen_atom_offsets.lua <metadata.h> <source1> [source2 ...]")
         os.exit(1)
     end
     local word_counts = load_word_counts(args[1])