lua metaprogram adjustments

code/duffle/gen/lottes_tape.offsets.h

@@ -1,4 +1,4 @@
-// Auto-generated by gen_atom_offsets.lua — DO NOT EDIT
+// Auto-generated by tape_atom_offset_gen.meta.lua — DO NOT EDIT
 // Source: C:\projects\Pikuma\ps1\code\duffle\lottes_tape.h
 #pragma once
 

code/gte_hello/gen/hello_gte_tape.offsets.h

@@ -1,4 +1,4 @@
-// Auto-generated by gen_atom_offsets.lua — DO NOT EDIT
+// Auto-generated by tape_atom_offset_gen.meta.lua — DO NOT EDIT
 // Source: C:\projects\Pikuma\ps1\code\gte_hello\hello_gte_tape.c
 #pragma once
 

scripts/tape_atom.offset_gen.meta.lua

@@ -1,5 +1,5 @@
 #!/usr/bin/env lua
--- gen_atom_offsets.lua — hand-rolled, no patterns, no regex, no region pragmas.
+-- tape_atom_offset_gen.lua
 --
 -- Finds every `MipsAtom_(name) { ... }` declaration in the given sources,
 -- counts the words in each body using the WORD_COUNT manifest, computes
@@ -26,10 +26,7 @@
 -- Character classification
 -- ============================================================
 
-local function is_space(c)
+local function is_space(c) return c == " " or c == "\t" or c == "\n" or c == "\r" or c == "\v" or c == "\f" end
-    return c == " " or c == "\t" or c == "\n" or c == "\r" or c == "\v" or c == "\f"
-end
-
 local function is_alpha(c)
 	if not c or #c == 0      then return false end
 	if c >= "a" and c <= "z" then return true  end
@@ -37,13 +34,8 @@ local function is_alpha(c)
 	return c == "_"
 end
 
-local function is_digit(c)
+local function is_digit(c) return c and c >= "0" and c <= "9" end
-    return c and c >= "0" and c <= "9"
+local function is_alnum(c) return is_alpha(c) or is_digit(c) end
-end
-
-local function is_alnum(c)
-    return is_alpha(c) or is_digit(c)
-end
 
 -- ============================================================
 -- I/O
@@ -64,19 +56,15 @@ local function write_file(path, content)
 	f:close()
 end
 
-local function ensure_dir(path)
+local function ensure_dir(path) os.execute('mkdir -p "' .. path .. '"') end
-    os.execute('mkdir -p "' .. path .. '"')
-end
 
 -- ============================================================
 -- String primitives
 -- ============================================================
 
 local function trim(s)
-    local a = 1
+	local a = 1;  while a <= #s and is_space(s:sub(a, a)) do a = a + 1 end
-    while a <= #s and is_space(s:sub(a, a)) do a = a + 1 end
+	local b = #s; while b >= a  and is_space(s:sub(b, b)) do b = b - 1 end
-    local b = #s
-    while b >= a and is_space(s:sub(b, b)) do b = b - 1 end
 	return s:sub(a, b)
 end
 
@@ -128,19 +116,7 @@ local function basename_no_ext(path)
 	return path:sub(a, last_dot - 1)
 end
 
-local function to_upper(s)
+local function to_upper(s) return s:upper() end
-    local out = ""
-    for i = 1, #s do
-        local code = string.byte(s, i)
-        if code >= 97 and code <= 122 then
-            out = out .. string.char(code - 32)
-        else
-            out = out .. s:sub(i, i)
-        end
-    end
-    return out
-end
-
 local function to_alnum_underscore(s)
 	local out = ""
 	for i = 1, #s do
@@ -150,35 +126,51 @@ local function to_alnum_underscore(s)
 	end
 	return out
 end
-
+local function pad_right(s, w) return s .. string.rep(" ", w - #s) end
-local function pad_right(s, width)
-    while #s < width do s = s .. " " end
-    return s
-end
 
 -- ============================================================
 -- Lexer helpers
 -- ============================================================
 
-local function skip_ws_and_comments(source, i)
+-- If position i starts a C string literal ("..."), char literal ('.'),
-    local len = #source
+-- // line comment, or /* block comment, advance past it and return the
-    while i <= len do
+-- position just after the construct (or #s+1 if unterminated).
-        local c = source:sub(i, i)
+-- Otherwise return i unchanged.
-        if is_space(c) then
+local function skip_str_or_cmt(s, i)
+	local c = s:sub(i, i)
+	if    c == '"' or c == "'" then
 		i = i + 1
-        elseif c == "/" and source:sub(i+1, i+1) == "/" then
+		while i <= #s do
-            while i <= len and source:sub(i, i) ~= "\n" do i = i + 1 end
+			if s:sub(i, i) == "\\" then i = i + 2
-        elseif c == "/" and source:sub(i+1, i+1) == "*" then
+			elseif s:sub(i, i) == c then return i + 1
+			else i = i + 1 end
+		end
+		return #s + 1
+	elseif c == "/" then
+		local nx = s:sub(i+1, i+1)
+		if nx == "/" then
+			while i <= #s and s:sub(i, i) ~= "\n" do i = i + 1 end
+			return i
+		elseif nx == "*" then
 			i = i + 2
-            while i <= len - 1 do
+			while i <= #s - 1 do
-                if source:sub(i, i) == "*" and source:sub(i+1, i+1) == "/" then
+				if s:sub(i, i) == "*" and s:sub(i+1, i+1) == "/" then
-                    i = i + 2
+					return i + 2
-                    break
 				end
 				i = i + 1
 			end
+			return #s + 1
+		end
+	end
+	return i
+end
+
+local function skip_ws_and_cmt(s, i)
+	while i <= #s do
+		if is_space(s:sub(i, i)) then i = i + 1
 		else
-            break
+			local nx = skip_str_or_cmt(s, i)
+			if nx > i then i = nx else break end
 		end
 	end
 	return i
@@ -192,14 +184,14 @@ local function read_ident(source, i)
 	return source:sub(a, i - 1), i
 end
 
-local function read_balanced(source, open_char, close_char, i)
+local function read_balanced(s, open_char, close_char, i)
-    if source:sub(i, i) ~= open_char then return nil, i end
+	if s:sub(i, i) ~= open_char then return nil, i end
 	i = i + 1
-    local len = #source
+	local len   = #s
 	local depth = 1
 	local a     = i
 	while i <= len and depth > 0 do
-        local c = source:sub(i, i)
+		local c = s:sub(i, i)
 		if c == open_char then
 			depth = depth + 1
 			i = i + 1
@@ -207,81 +199,31 @@ local function read_balanced(source, open_char, close_char, i)
 			depth = depth - 1
 			if depth == 0 then break end
 			i = i + 1
-        elseif c == '"' then
-            i = i + 1
-            while i <= len do
-                if source:sub(i, i) == "\\" then i = i + 2
-                elseif source:sub(i, i) == '"' then i = i + 1; break
-                else i = i + 1 end
-            end
-        elseif c == "'" then
-            i = i + 1
-            while i <= len do
-                if source:sub(i, i) == "\\" then i = i + 2
-                elseif source:sub(i, i) == "'" then i = i + 1; break
-                else i = i + 1 end
-            end
-        elseif c == "/" and source:sub(i+1, i+1) == "/" then
-            while i <= len and source:sub(i, i) ~= "\n" do i = i + 1 end
-        elseif c == "/" and source:sub(i+1, i+1) == "*" then
-            i = i + 2
-            while i <= len - 1 do
-                if source:sub(i, i) == "*" and source:sub(i+1, i+1) == "/" then
-                    i = i + 2
-                    break
-                end
-                i = i + 1
-            end
 		else
-            i = i + 1
+			local nx = skip_str_or_cmt(s, i)
+			if nx > i then i = nx else i = i + 1 end
 		end
 	end
-    return source:sub(a, i - 1), i + 1
+	return s:sub(a, i - 1), i + 1
 end
 
-local function read_parens(source, i)  return read_balanced(source, "(", ")", i) end
+local read_parens   = function(s, i) return read_balanced(s, "(", ")", i) end
-local function read_braces(source, i)  return read_balanced(source, "{", "}", i) end
+local read_braces   = function(s, i) return read_balanced(s, "{", "}", i) end
-local function read_brackets(source, i) return read_balanced(source, "[", "]", i) end
+local read_brackets = function(s, i) return read_balanced(s, "[", "]", i) end
 
-local function scan_to_char(source, target, start)
+local function scan_to_char(s, target, start)
-    local len = #source
 	local i = start
-    while i <= len do
+	while i <= #s do
-        local c = source:sub(i, i)
+		local c = s:sub(i, i)
-        if c == target then return i
+		if    c == target then return i end
-        elseif c == "(" then local _, a = read_parens(source, i); i = a
+		if     c == "(" then local _, a = read_balanced(s, "(", ")", i); i = a
-        elseif c == "{" then local _, a = read_braces(source, i); i = a
+		elseif c == "{" then local _, a = read_balanced(s, "{", "}", i); i = a
-        elseif c == "[" then local _, a = read_brackets(source, i); i = a
+		elseif c == "[" then local _, a = read_balanced(s, "[", "]", i); i = a
-        elseif c == '"' then
-            i = i + 1
-            while i <= len do
-                if source:sub(i, i) == "\\" then i = i + 2
-                elseif source:sub(i, i) == '"' then i = i + 1; break
-                else i = i + 1 end
-            end
-        elseif c == "'" then
-            i = i + 1
-            while i <= len do
-                if source:sub(i, i) == "\\" then i = i + 2
-                elseif source:sub(i, i) == "'" then i = i + 1; break
-                else i = i + 1 end
-            end
-        elseif c == "/" and source:sub(i+1, i+1) == "/" then
-            while i <= len and source:sub(i, i) ~= "\n" do i = i + 1 end
-        elseif c == "/" and source:sub(i+1, i+1) == "*" then
-            i = i + 2
-            while i <= len - 1 do
-                if source:sub(i, i) == "*" and source:sub(i+1, i+1) == "/" then
-                    i = i + 2
-                    break
-                end
-                i = i + 1
-            end
 		else
-            i = i + 1
+			local nx = skip_str_or_cmt(s, i)
+			if nx > i then i = nx else i = i + 1 end
 		end
 	end
-    return nil
 end
 
 -- ============================================================
@@ -290,7 +232,7 @@ end
 -- ============================================================
 
 local function extract_ident_args(token, after_ident)
-    local arg_start = skip_ws_and_comments(token, after_ident)
+	local arg_start = skip_ws_and_cmt(token, after_ident)
 	if token:sub(arg_start, arg_start) ~= "(" then return {}, nil end
 	local inner, after_paren = read_parens(token, arg_start)
 
@@ -298,7 +240,7 @@ local function extract_ident_args(token, after_ident)
 	local n    = 1
 	local len  = #inner
 	while n <= len do
-        n = skip_ws_and_comments(inner, n)
+		   n = skip_ws_and_cmt(inner, n)
 		if n > len then break end
 		local ident, after = read_ident(inner, n)
 		if ident and ident ~= "" then
@@ -307,7 +249,7 @@ local function extract_ident_args(token, after_ident)
 		else
 			n = n + 1
 		end
-        n = skip_ws_and_comments(inner, n)
+		n = skip_ws_and_cmt(inner, n)
 		if n <= len and inner:sub(n, n) == "," then n = n + 1 end
 	end
 
@@ -333,8 +275,7 @@ local function load_word_counts(metadata_path)
 			local inner = trimmed:sub(#prefix + 1, #trimmed - 1)
 			local comma = find_byte(inner, ",", 1)
 			if comma then
-                counts[trim(inner:sub(1, comma - 1))] =
+				counts[trim(inner:sub(1, comma - 1))] = tonumber(trim(inner:sub(comma + 1)))
-                    tonumber(trim(inner:sub(comma + 1)))
 			end
 		end
 		i = line_end + 1
@@ -346,19 +287,15 @@ end
 -- Count words for a single comma-separated token
 -- ============================================================
 
-local function word_count_of_token(token, word_counts)
+local function word_count_of_token(token, wc)
-    local i = 1
+	local s = trim(token)
-    local len = #token
+	if    s == "" then return 0 end
-    while i <= len and is_space(token:sub(i, i)) do i = i + 1 end
+	local name, after = read_ident(s, 1)
-    if i > len then return 0 end
-    local name, after = read_ident(token, i)
 	if not name then return 1        end
-    local j = skip_ws_and_comments(token, after)
+	if wc[name] then return wc[name] end
-    if token:sub(j, j) == "(" then
+	local j = skip_ws_and_cmt(s, after)
-        local wc = word_counts[name]
+	if s:sub(j, j) == "(" then
-        if wc then return wc end
 		io.stderr:write("  warning: unknown macro '" .. name .. "', assuming 1 word\n")
-        return 1
 	end
 	return 1
 end
@@ -369,48 +306,23 @@ end
 
 local function split_top_level_commas(body)
 	local tokens = {}
-    local len = #body
 	local i = 1
 	local token_start = 1
-    while i <= len do
+	while i <= #body do
 		local c = body:sub(i, i)
 		if c == "(" then local _, a = read_parens(body, i); i = a
 		elseif c == "{" then local _, a = read_braces(body, i); i = a
 		elseif c == "[" then local _, a = read_brackets(body, i); i = a
-        elseif c == '"' then
-            i = i + 1
-            while i <= len do
-                if body:sub(i, i) == "\\" then i = i + 2
-                elseif body:sub(i, i) == '"' then i = i + 1; break
-                else i = i + 1 end
-            end
-        elseif c == "'" then
-            i = i + 1
-            while i <= len do
-                if body:sub(i, i) == "\\" then i = i + 2
-                elseif body:sub(i, i) == "'" then i = i + 1; break
-                else i = i + 1 end
-            end
-        elseif c == "/" and body:sub(i+1, i+1) == "/" then
-            while i <= len and body:sub(i, i) ~= "\n" do i = i + 1 end
-        elseif c == "/" and body:sub(i+1, i+1) == "*" then
-            i = i + 2
-            while i <= len - 1 do
-                if body:sub(i, i) == "*" and body:sub(i+1, i+1) == "/" then
-                    i = i + 2
-                    break
-                end
-                i = i + 1
-            end
 		elseif c == "," then
 			table.insert(tokens, body:sub(token_start, i - 1))
 			i = i + 1
 			token_start = i
 		else
-            i = i + 1
+			local nx = skip_str_or_cmt(body, i)
+			if nx > i then i = nx else i = i + 1 end
 		end
 	end
-    local last = body:sub(token_start, len)
+	local last = body:sub(token_start)
 	if trim(last) ~= "" then table.insert(tokens, last) end
 	return tokens
 end
@@ -424,35 +336,10 @@ local function scan_for_atom_markers(token, at_pos, labels, branches)
 	local i   = 1
 	local len = #token
 	while i <= len do
-        i = skip_ws_and_comments(token, i)
+		   i = skip_ws_and_cmt(token, i)
 		if i > len then break end
 		local c = token:sub(i, i)
-        if c == '"' then
+		if is_alpha(c) then
-            i = i + 1
-            while i <= len do
-                if token:sub(i, i) == "\\" then i = i + 2
-                elseif token:sub(i, i) == '"' then i = i + 1; break
-                else i = i + 1 end
-            end
-        elseif c == "'" then
-            i = i + 1
-            while i <= len do
-                if token:sub(i, i) == "\\" then i = i + 2
-                elseif token:sub(i, i) == "'" then i = i + 1; break
-                else i = i + 1 end
-            end
-        elseif c == "/" and token:sub(i+1, i+1) == "/" then
-            while i <= len and token:sub(i, i) ~= "\n" do i = i + 1 end
-        elseif c == "/" and token:sub(i+1, i+1) == "*" then
-            i = i + 2
-            while i <= len - 1 do
-                if token:sub(i, i) == "*" and token:sub(i+1, i+1) == "/" then
-                    i = i + 2
-                    break
-                end
-                i = i + 1
-            end
-        elseif is_alpha(c) then
 			local ident, after = read_ident(token, i)
 			if ident == "atom_label" then
 				local args, after_paren = extract_ident_args(token, after)
@@ -460,19 +347,14 @@ local function scan_for_atom_markers(token, at_pos, labels, branches)
 				if after_paren then i = after_paren else i = after end
 			elseif ident == "atom_offset" then
 				local args, after_paren = extract_ident_args(token, after)
-                if #args >= 2 then
+				if #args >= 2 then table.insert(branches, {pos = at_pos, target = args[2], tag = args[1]}) end
-                    table.insert(branches, {
-                        pos = at_pos,
-                        target = args[2],
-                        tag = args[1]
-                    })
-                end
 				if after_paren then i = after_paren else i = after end
 			else
 				i = after
 			end
 		else
-            i = i + 1
+			local nx = skip_str_or_cmt(token, i)
+			if nx > i then i = nx else i = i + 1 end
 		end
 	end
 end
@@ -485,13 +367,11 @@ local function scan_atom_body(body, word_counts)
 	local pos      = 0
 	local labels   = {}
 	local branches = {}
-
 	for _, tok in ipairs(split_top_level_commas(body)) do
 		local k    = 1
 		local tlen = #tok
 		while k <= tlen and is_space(tok:sub(k, k)) do k = k + 1 end
-        local leading_ident, leading_after = read_ident(tok, k)
+		local leading_ident = read_ident(tok, k)
-
 		if leading_ident == "atom_label" or leading_ident == "atom_offset" then
 			scan_for_atom_markers(tok, pos, labels, branches)
 		else
@@ -500,7 +380,6 @@ local function scan_atom_body(body, word_counts)
 			pos = pos + words
 		end
 	end
-
 	return labels, branches, pos
 end
 
@@ -508,14 +387,6 @@ end
 -- Find every MipsAtom_(name) { ... } in a source
 -- ============================================================
 
-local function has_prefix(s, prefix)
-    if #s < #prefix then return false end
-    for i = 1, #prefix do
-        if s:sub(i, i) ~= prefix:sub(i, i) then return false end
-    end
-    return true
-end
-
 local function skip_qualifiers(source, i)
 	local keywords = {
 		["static"]  = true, ["const"]    = true, ["volatile"] = true,
@@ -524,7 +395,7 @@ local function skip_qualifiers(source, i)
 		["internal"]= true, ["LP_"]      = true, ["global"] = true, ["gkknown"] = true
 	}
 	while true do
-        i = skip_ws_and_comments(source, i)
+		i = skip_ws_and_cmt(source, i)
 		local ident, after = read_ident(source, i)
 		if not ident       then                return i end
 		if keywords[ident] then i = after else return i end
@@ -537,7 +408,7 @@ local function find_atoms(source_text)
 	local i     = 1
 
 	local function try_wrapped(after_pos)
-        local paren_pos = skip_ws_and_comments(source_text, after_pos)
+		local paren_pos = skip_ws_and_cmt(source_text, after_pos)
 		if source_text:sub(paren_pos, paren_pos) ~= "(" then return nil end
 		local inner, after_paren = read_parens(source_text, paren_pos)
 		local n = 1
@@ -553,10 +424,10 @@ local function find_atoms(source_text)
 	end
 
 	local function try_raw(after_pos)
-        local next_pos = skip_ws_and_comments(source_text, after_pos)
+		local next_pos = skip_ws_and_cmt(source_text, after_pos)
 		local next_ident, next_after = read_ident(source_text, next_pos)
 		if not next_ident then return nil end
-        if not has_prefix(next_ident, "code_") then return nil end
+		if not starts_with(next_ident, "code_") then return nil end
 		if #next_ident <= 5 then return nil end
 		local atom_name = next_ident:sub(6)
 		local brace_pos = scan_to_char(source_text, "{", next_after)
@@ -566,11 +437,8 @@ local function find_atoms(source_text)
 	end
 
 	while i <= len do
-        i = skip_ws_and_comments(source_text, i)
+		i = skip_ws_and_cmt(source_text, i); if i > len then break end
-        if i > len then break end
+		i = skip_qualifiers(source_text, i); if i > len then break end
-        i = skip_qualifiers(source_text, i)
-        if i > len then break end
-
 		local ident, after = read_ident(source_text, i)
 		if not ident then
 			i = i + 1
@@ -606,14 +474,9 @@ local function compute_offsets(labels, branches)
 	for _, br in ipairs(branches) do
 		local target = labels[br.target]
 		if not target then
-            error("Branch target '" .. br.target ..
+			error("Branch target '" .. br.target .. "' has no atom_label (at word " .. br.pos .. ")")
-                  "' has no atom_label (at word " .. br.pos .. ")")
 		end
-        table.insert(results, {
+		table.insert(results, {target = br.target, tag = br.tag, offset = target - br.pos - 1 })
-            target = br.target,
-            tag = br.tag,
-            offset = target - br.pos - 1
-        })
 	end
 	return results
 end
@@ -639,13 +502,10 @@ local function generate_header(source_path, atoms_data)
 	add("#undef atom_offset")
 	add("#define atom_offset(tag, name)  atom_offset_##tag##_##name")
 	add("")
-
 	for _, atom in ipairs(atoms_data) do
 		if #atom.offsets > 0 then
 			add("// --- atom: " .. atom.name .. " (" .. atom.total_words .. " words) ---")
 			add("")
-
-            -- Build constant list once
 			local consts = {}
 			for _, r in ipairs(atom.offsets) do
 				table.insert(consts, {
@@ -654,27 +514,14 @@ local function generate_header(source_path, atoms_data)
 					value      = r.offset
 				})
 			end
-
+			for _, c in ipairs(consts) do add("#define " .. pad_right(c.macro_name, 44) .. "  " .. c.value .. "") end
-            -- Macro form: single source of truth for the literal value.
-            -- Underscore prefix keeps it out of the C namespace so the
-            -- enum can use the same identifier name without conflict.
-            for _, c in ipairs(consts) do
-                add("#define " .. pad_right(c.macro_name, 44) .. "  " .. c.value .. "")
-            end
 			add("")
-
-            -- Enum form: C code uses the natural name, value comes
-            -- from the underscore-prefixed macro so the literal is
-            -- defined in exactly one place.
 			add("enum {")
-            for _, c in ipairs(consts) do
+			for _, c in ipairs(consts) do add("    " .. c.enum_name .. " = " .. c.macro_name .. ",") end
-                add("    " .. c.enum_name .. " = " .. c.macro_name .. ",")
-            end
 			add("};")
 			add("")
 		end
 	end
-
 	add("#pragma endregion " .. basename)
 	add("")
 	return table.concat(lines, "\n") .. "\n"
@@ -729,11 +576,8 @@ local function main(args)
 		print("Usage: gen_atom_offsets.lua <metadata.h> <source1> [source2 ...]")
 		os.exit(1)
 	end
-    local metadata_path = args[1]
+	local word_counts = load_word_counts(args[1])
-    local sources = {}
+	for i = 2, #args do process_source(args[i], word_counts) end
-    for i = 2, #args do table.insert(sources, args[i]) end
-    local word_counts = load_word_counts(metadata_path)
-    for _, src in ipairs(sources) do process_source(src, word_counts) end
 end
 
 main({...})