More minor adjustments while reviewing for the article.

This commit is contained in:
ed
2026-09-05 18:52:14 -04:00
parent 1faf3539d8
commit 0e9034cbd4
10 changed files with 296 additions and 214 deletions
@@ -30,7 +30,7 @@
"editorHoverWidget.background": "#2c334b"
},
"semanticTokenColors": {
"comment": { "foreground": "#868686", "fontStyle": "italic" },
"comment": { "foreground": "#868686", }, //"fontStyle": "italic" },
"keyword": { "foreground": "#d8bd5b" },
"string": { "foreground": "#d46a54" },
"number": { "foreground": "#b5cea8" },
@@ -81,7 +81,7 @@
"tapeDelaySlot": { "foreground": "#ff5647" }
},
"tokenColors": [
{ "scope": ["comment", "comment.block", "comment.line", "comment.block.documentation"], "settings": { "foreground": "#868686", "fontStyle": "italic" } },
{ "scope": ["comment", "comment.block", "comment.line", "comment.block.documentation"], "settings": { "foreground": "#868686", } }, //"fontStyle": "italic" } },
{ "scope": ["keyword", "keyword.control", "keyword.other"], "settings": { "foreground": "#d8bd5b" } },
{ "scope": ["string", "string.quoted"], "settings": { "foreground": "#d46a54" } },
{ "scope": ["string.quoted.other"], "settings": { "foreground": "#d69d85" } },
+118 -73
View File
@@ -26,67 +26,103 @@ const TOKEN_TYPES = [
"macro",
];
const TOKEN_MODIFIERS = ["declaration", "tapeRead", "tapeWrite", "tapeAuto"];
const TOKEN_TYPE_INDEX = new Map(TOKEN_TYPES.map((name, index) => [name, index]));
const TOKEN_MODIFIERS = ["declaration", "tapeRead", "tapeWrite", "tapeAuto"];
const TOKEN_TYPE_INDEX = new Map(TOKEN_TYPES.map((name, index) => [name, index]));
const TOKEN_MODIFIER_INDEX = new Map(TOKEN_MODIFIERS.map((name, index) => [name, index]));
const ATOM_KEYWORDS = new Set(["MipsAtom_", "MipsAtom_Proc_"]);
const COMPONENT_KEYWORDS = new Set(["MipsAtomComp_", "MipsAtomComp_Proc_"]);
const ANNOTATIONS = new Set([
"atom_info", "atom_bind", "atom_reads", "atom_writes", "atom_label",
"atom_offset", "atom_reg", "atom_type", "atom_ctx", "atom_phase",
"atom_auto_reg", "phase_auto_reg", "atom_dbg_skip",
"atom_info",
"atom_bind",
"atom_reads",
"atom_writes",
"atom_label",
"atom_offset",
"atom_reg",
"atom_type",
"atom_ctx",
"atom_phase",
"atom_auto_reg",
"phase_auto_reg",
"atom_dbg_skip",
]);
const DSL_KEYWORDS = new Set([
"FI_", "I_", "NI_", "Relative_", "Struct_", "Enum_", "Union_", "Array_",
"Slice_", "TypeR_", "TypeV_", "align_", "internal", "local_persist", "global",
"RO_", "LP_", "gknown", "expect_", "cexpr_",
"enum", "struct", "union",
"offset_of", "static_assert", "typeof", "typeof_ptr", "typeof_same",
"glue", "tmpl",
"A_", "FI_", "I_", "NI_",
"Array_", "Enum_", "Proc_", "Relative_", "Struct_", "Union_", "Slice_",
// "TypeR_", "TypeV_",
"align_",
"internal", "local_persist", "global",
"RO_", "LP_",
"gknown", "expect_", "cexpr_",
"O_", "OA_", "S_", "C_", "T_", "T_same", "R_", "V_",
"r_", "v_", "rt_", "vt_",
"asm", "asm_words", "asm_rpins", "asm_clobber",
"O_", "S_", "C_", "T_", "tmpl", "glue", "r_", "v_", "rt_", "vt_",
"rgcc", "r_use", "r_set", "r_mod", "r_imm", "r_mem",
"u1_", "u2_", "u4_", "u8_", "s1_", "s2_", "s4_", "s8_",
"u1_r", "u2_r", "u4_r", "u8_r", "u1_v", "u2_v", "u4_v", "u8_v",
"b1_", "b2_", "b4_", "b8_",
"u1_", "u2_", "u4_", "u8_",
"s1_", "s2_", "s4_", "s8_",
"b1_r", "b2_r", "b4_r", "b8_r",
"b1_v", "b2_v", "b4_v", "b8_v",
"u1_r", "u2_r", "u4_r", "u8_r",
"u1_v", "u2_v", "u4_v", "u8_v",
"u4_lo", "u4_hi",
]);
const DELAY_SLOT_KEYWORDS = new Set(["LdSlot_", "BdSlot_", "DmaSlot_", "GteDelay_"]);
const DELAY_SLOT_KEYWORDS = new Set([
"LdSlot_",
"BdSlot_",
"DmaSlot_",
"GteDelay_"
]);
const CONTROL_FLOW_PREFIXES = /^(?:branch_|jump_|call_)/;
const ROLE_TO_TYPE = {
atomName: "tapeAtomName",
atomName: "tapeAtomName",
componentName: "tapeComponentName",
bindType: "tapeBindType",
duffleType: "tapeDuffleType",
gprRegister: "tapeGprRegister",
cop2Register: "tapeCop2Register",
bindType: "tapeBindType",
duffleType: "tapeDuffleType",
gprRegister: "tapeGprRegister",
cop2Register: "tapeCop2Register",
};
function registerType(name, index) {
const kind = index.registers.get(name);
if (kind === "gpr" || /^R_[A-Za-z0-9_]+$/.test(name)) return "tapeGprRegister";
if (kind === "cop2" || /^(?:C2_|gte_cr_)[A-Za-z0-9_]+$/.test(name)) return "tapeCop2Register";
if (kind === "gpr" || /^R_[A-Za-z0-9_] + $/.test(name)) return "tapeGprRegister";
if (kind === "cop2" || /^(?:C2_|gte_cr_)[A-Za-z0-9_] + $/.test(name)) return "tapeCop2Register";
return null;
}
function instructionType(name, index) {
const domain = index.macros.get(name);
if (domain === "control") return "tapeControlFlow";
if (domain === "cpu") return "tapeCpuInstruction";
if (domain === "gte") return "tapeGteInstruction";
if (domain === "gpu") return "tapeGpuInstruction";
if (domain === "cpu") return "tapeCpuInstruction";
if (domain === "gte") return "tapeGteInstruction";
if (domain === "gpu") return "tapeGpuInstruction";
if (domain === "component") {
if (/^mac_gte_/.test(name)) return "tapeGteInstruction";
if (/^mac_gp/.test(name)) return "tapeGpuInstruction";
if (/^mac_/.test(name)) return "tapeComponentInstruction";
if (/^mac_gp/.test(name)) return "tapeGpuInstruction";
if (/^mac_/.test(name)) return "tapeComponentInstruction";
return "macro";
}
if (domain === "utility") return "macro";
if (domain === "utility") return "macro";
if (/^gte_(?!cr_)/.test(name)) return "tapeGteInstruction";
if (/^gp[01]_/.test(name)) return "tapeGpuInstruction";
if (/^mac_gte_/.test(name)) return "tapeGteInstruction";
if (/^mac_gp/.test(name)) return "tapeGpuInstruction";
if (/^mac_/.test(name)) return "tapeComponentInstruction";
if (/^gp[01]_/.test(name)) return "tapeGpuInstruction";
if (/^mac_gte_/.test(name)) return "tapeGteInstruction";
if (/^mac_gp/.test(name)) return "tapeGpuInstruction";
if (/^mac_/.test(name)) return "tapeComponentInstruction";
return null;
}
@@ -100,13 +136,11 @@ function modifierMask(modifiers) {
}
function isRegUseAccess(tokens, tokenIndex) {
const prev = tokens[tokenIndex - 1];
if (!prev || prev.text !== ".") return false;
const prevPrev = tokens[tokenIndex - 2];
if (!prevPrev || prevPrev.kind !== "identifier") return false;
const next = tokens[tokenIndex + 1];
const prev = tokens[tokenIndex - 1]; if (!prev || prev.text !== ".") return false;
const prevPrev = tokens[tokenIndex - 2]; if (!prevPrev || prevPrev.kind !== "identifier") return false;
const next = tokens[tokenIndex + 1];
if (next && next.text === ".") return false;
if (prevPrev.text === "r") return true;
if (prevPrev.text === "r") return true;
const prev3 = tokens[tokenIndex - 3];
const prev4 = tokens[tokenIndex - 4];
if (prev3 && prev3.text === "." && prev4 && prev4.kind === "identifier" && prev4.text === "r") return true;
@@ -115,81 +149,92 @@ function isRegUseAccess(tokens, tokenIndex) {
function classifyDocument(source, filePath, workspaceIndex, shouldCancel = () => false) {
const scanned = scanSource(source, filePath);
const index = mergeIndexes(workspaceIndex, scanned.index);
const spans = [];
const index = mergeIndexes(workspaceIndex, scanned.index);
const spans = [];
for (let tokenIndex = 0; tokenIndex < scanned.tokens.length; tokenIndex += 1) {
if (shouldCancel()) break;
const token = scanned.tokens[tokenIndex];
if (token.kind !== "identifier") continue;
const token = scanned.tokens[tokenIndex]; if (token.kind !== "identifier") continue;
let type = null;
let modifiers = [];
let type = null;
let modifiers = [];
const declaration = scanned.declarations.get(token.start);
const context = nearestCall(scanned.contexts, tokenIndex);
const context = nearestCall(scanned.contexts, tokenIndex);
if (declaration) {
type = ROLE_TO_TYPE[declaration.role] || null;
type = ROLE_TO_TYPE[declaration.role] || null;
modifiers = declaration.modifiers.slice();
} else if (ATOM_KEYWORDS.has(token.text)) {
}
else if (ATOM_KEYWORDS.has(token.text)) {
type = "tapeAtomKeyword";
} else if (COMPONENT_KEYWORDS.has(token.text)) {
}
else if (COMPONENT_KEYWORDS.has(token.text)) {
type = "keyword";
} else if (ANNOTATIONS.has(token.text)) {
}
else if (ANNOTATIONS.has(token.text)) {
type = "tapeAnnotation";
} else if (context && context.callee === "atom_bind" && context.argIndex === 0) {
}
else if (context && context.callee === "atom_bind" && context.argIndex === 0) {
type = "tapeBindType";
} else if (context && context.callee === "atom_phase" && context.argIndex === 0) {
}
else if (context && context.callee === "atom_phase" && context.argIndex === 0) {
type = "tapePhase";
modifiers = ["declaration"];
} else if (context && context.callee === "atom_ctx" && context.argIndex === 0) {
}
else if (context && context.callee === "atom_ctx" && context.argIndex === 0) {
type = "tapeAtomName";
} else if (context && context.callee === "atom_label" && context.argIndex === 0) {
}
else if (context && context.callee === "atom_label" && context.argIndex === 0) {
type = "tapeLabel";
modifiers = ["declaration"];
} else if (context && context.callee === "atom_offset" && context.argIndex <= 1) {
}
else if (context && context.callee === "atom_offset" && context.argIndex <= 1) {
type = "tapeLabel";
} else if (context && context.callee === "atom_reads") {
}
else if (context && context.callee === "atom_reads") {
type = registerType(token.text, index);
if (type) modifiers = ["tapeRead"];
} else if (context && context.callee === "atom_writes") {
}
else if (context && context.callee === "atom_writes") {
type = registerType(token.text, index);
if (type) modifiers = ["tapeWrite"];
} else if (context && context.callee === "atom_auto_reg") {
}
else if (context && context.callee === "atom_auto_reg") {
if (context.argIndex === 0) type = "tapeAtomName";
if (context.argIndex === 1) {
type = "tapeGprRegister";
type = "tapeGprRegister";
modifiers = ["declaration", "tapeAuto"];
}
} else if (context && context.callee === "phase_auto_reg") {
}
else if (context && context.callee === "phase_auto_reg") {
if (context.argIndex === 0) type = "tapePhase";
if (context.argIndex === 1) {
type = "tapeGprRegister";
type = "tapeGprRegister";
modifiers = ["declaration", "tapeAuto"];
}
}
if (!type && index.bindTypes.has(token.text)) type = "tapeBindType";
if (!type && DSL_KEYWORDS.has(token.text)) type = "keyword";
if (!type && index.types.has(token.text)) type = "tapeDuffleType";
if (!type && index.attributes.has(token.text)) type = "tapeAttribute";
if (!type) type = registerType(token.text, index);
if (!type && DELAY_SLOT_KEYWORDS.has(token.text)) type = "tapeDelaySlot";
if (!type) {
if (! type && index.bindTypes.has(token.text)) type = "tapeBindType";
if (! type && DSL_KEYWORDS.has(token.text)) type = "keyword";
if (! type && index.types.has(token.text)) type = "tapeDuffleType";
if (! type && index.attributes.has(token.text)) type = "tapeAttribute";
if (! type) type = registerType(token.text, index);
if (! type && DELAY_SLOT_KEYWORDS.has(token.text)) type = "tapeDelaySlot";
if (! type) {
const domain = index.macros.get(token.text);
if (domain === "control" || (domain && CONTROL_FLOW_PREFIXES.test(token.text))) {
type = "tapeControlFlow";
}
}
if (!type && isRegUseAccess(scanned.tokens, tokenIndex)) type = "tapeGprRegister";
if (!type) type = instructionType(token.text, index);
if (!type && /^(?:Slice_|A[0-9]+_)/.test(token.text)) type = "tapeDuffleType";
if (!type && /_[RV]$/.test(token.text)) type = "tapeDuffleType";
if (!type && index.atoms.has(token.text)) type = "tapeAtomName";
if (!type && index.components.has(token.text)) type = "tapeComponentName";
if (!type && index.phases.has(token.text)) type = "tapePhase";
if (!type && index.labels.has(token.text)) type = "tapeLabel";
if (!type) continue;
if (! type && isRegUseAccess(scanned.tokens, tokenIndex)) type = "tapeGprRegister";
if (! type) type = instructionType(token.text, index);
if (! type && /^(?:Slice_|A[0-9]+_)/.test(token.text)) type = "tapeDuffleType";
if (! type && /_[RV]$/.test(token.text)) type = "tapeDuffleType";
if (! type && index.atoms.has(token.text)) type = "tapeAtomName";
if (! type && index.components.has(token.text)) type = "tapeComponentName";
if (! type && index.phases.has(token.text)) type = "tapePhase";
if (! type && index.labels.has(token.text)) type = "tapeLabel";
if (! type) continue;
spans.push({
text: token.text,
+22 -18
View File
@@ -2,10 +2,10 @@
const vscode = require("vscode");
const { TOKEN_MODIFIERS, TOKEN_TYPES, classifyDocument } = require("./classifier");
const { createIndex, mergeIndexes, scanSource } = require("./source-index");
const { createIndex, mergeIndexes, scanSource } = require("./source-index");
const SOURCE_GLOB = "**/*.{c,h,cc,cpp,cxx,hh,hpp,hxx}";
const EXCLUDE_GLOB = "**/{gen,build,.slop_cache,toolchain,node_modules}/**";
const SOURCE_GLOB = "**/*.{c,h,cc,cpp,cxx,hh,hpp,hxx}";
const EXCLUDE_GLOB = "**/{gen,build,.slop_cache,toolchain,node_modules}/**";
const EXCLUDED_SEGMENTS = new Set(["gen", "build", ".slop_cache", "toolchain", "node_modules"]);
function isExcluded(uri) {
@@ -18,28 +18,29 @@ function formatError(filePath, error) {
}
async function activate(context) {
const output = vscode.window.createOutputChannel("Tape Atom DSL");
const output = vscode.window.createOutputChannel("Tape Atom DSL");
const emitter = new vscode.EventEmitter();
const legend = new vscode.SemanticTokensLegend(TOKEN_TYPES, TOKEN_MODIFIERS);
let workspaceIndex = createIndex();
const legend = new vscode.SemanticTokensLegend(TOKEN_TYPES, TOKEN_MODIFIERS);
let workspaceIndex = createIndex();
let rebuildGeneration = 0;
let debounceHandle = null;
let debounceHandle = null;
async function rebuildIndex() {
const generation = ++rebuildGeneration;
const files = await vscode.workspace.findFiles(SOURCE_GLOB, EXCLUDE_GLOB);
let nextIndex = createIndex();
const files = await vscode.workspace.findFiles(SOURCE_GLOB, EXCLUDE_GLOB);
let nextIndex = createIndex();
for (const uri of files) {
if (generation !== rebuildGeneration) return;
if (isExcluded(uri)) continue;
try {
const bytes = await vscode.workspace.fs.readFile(uri);
const bytes = await vscode.workspace.fs.readFile(uri);
const source = Buffer.from(bytes).toString("utf8");
const result = scanSource(source, uri.fsPath);
nextIndex = mergeIndexes(nextIndex, result.index);
nextIndex = mergeIndexes(nextIndex, result.index);
for (const error of result.errors) output.appendLine(formatError(uri.fsPath, error));
} catch (error) {
}
catch (error) {
output.appendLine(`${uri.fsPath}: ${error.stack || error.message || error}`);
}
}
@@ -53,9 +54,11 @@ async function activate(context) {
if (uri && isExcluded(uri)) return;
if (debounceHandle !== null) clearTimeout(debounceHandle);
debounceHandle = setTimeout(() => {
debounceHandle = null;
rebuildIndex().catch((error) => output.appendLine(error.stack || String(error)));
}, 100);
debounceHandle = null;
rebuildIndex().catch((error) => output.appendLine(error.stack || String(error)));
},
100
);
}
const provider = {
@@ -77,7 +80,8 @@ async function activate(context) {
output.appendLine(formatError(document.uri.fsPath || document.uri.toString(), error));
}
return builder.build();
} catch (error) {
}
catch (error) {
output.appendLine(`${document.uri}: ${error.stack || error.message || error}`);
return new vscode.SemanticTokensBuilder(legend).build();
}
@@ -85,8 +89,8 @@ async function activate(context) {
};
const selector = [
{ language: "c", scheme: "file" },
{ language: "c", scheme: "untitled" },
{ language: "c", scheme: "file" },
{ language: "c", scheme: "untitled" },
{ language: "cpp", scheme: "file" },
{ language: "cpp", scheme: "untitled" },
];
+38 -31
View File
@@ -10,29 +10,30 @@ function isIdentifierContinue(code) {
return isIdentifierStart(code) || (code >= 48 && code <= 57);
}
function lex(source) {
function lex(source)
{
if (typeof source !== "string") throw new TypeError("source must be a string");
const tokens = [];
const errors = [];
let offset = 0;
let line = 0;
let character = 0;
const tokens = [];
const errors = [];
let offset = 0;
let line = 0;
let character = 0;
function advance() {
if (source[offset] === "\r" && source[offset + 1] === "\n") {
offset += 2;
line += 1;
offset += 2;
line += 1;
character = 0;
return;
}
if (source[offset] === "\n") {
offset += 1;
line += 1;
offset += 1;
line += 1;
character = 0;
return;
}
offset += 1;
offset += 1;
character += 1;
}
@@ -47,7 +48,8 @@ function lex(source) {
});
}
while (offset < source.length) {
while (offset < source.length)
{
const ch = source[offset];
if (/\s/.test(ch)) {
@@ -60,12 +62,14 @@ function lex(source) {
continue;
}
if (ch === "/" && source[offset + 1] === "*") {
if (ch === "/" && source[offset + 1] === "*")
{
const start = offset;
advance();
advance();
let closed = false;
while (offset < source.length) {
while (offset < source.length)
{
if (source[offset] === "*" && source[offset + 1] === "/") {
advance();
advance();
@@ -78,12 +82,14 @@ function lex(source) {
continue;
}
if (ch === "\"" || ch === "'") {
if (ch === "\"" || ch === "'")
{
const quote = ch;
const start = offset;
advance();
let closed = false;
while (offset < source.length) {
while (offset < source.length)
{
if (source[offset] === "\\") {
advance();
if (offset < source.length) advance();
@@ -97,14 +103,14 @@ function lex(source) {
if (source[offset] === "\n" || source[offset] === "\r") break;
advance();
}
if (!closed) errors.push({ kind: "unterminated-literal", offset: start });
if (! closed) errors.push({ kind: "unterminated-literal", offset: start });
continue;
}
const code = source.charCodeAt(offset);
if (isIdentifierStart(code)) {
const start = offset;
const startLine = line;
const start = offset;
const startLine = line;
const startCharacter = character;
advance();
while (offset < source.length && isIdentifierContinue(source.charCodeAt(offset))) advance();
@@ -112,8 +118,8 @@ function lex(source) {
continue;
}
const start = offset;
const startLine = line;
const start = offset;
const startLine = line;
const startCharacter = character;
advance();
pushToken("punctuation", start, startLine, startCharacter);
@@ -124,9 +130,9 @@ function lex(source) {
function buildCallContexts(tokens) {
const contexts = Array.from({ length: tokens.length }, () => []);
const calls = [];
const calls = [];
const errors = [];
const stack = [];
const stack = [];
for (let tokenIndex = 0; tokenIndex < tokens.length; tokenIndex += 1) {
const token = tokens[tokenIndex];
@@ -134,7 +140,8 @@ function buildCallContexts(tokens) {
if (token.text === ")") {
if (stack.length === 0) {
errors.push({ kind: "unmatched-close-paren", offset: token.start });
} else {
}
else {
const frame = stack.pop();
if (frame.callee !== null) calls.push({ ...frame, closeTokenIndex: tokenIndex });
}
@@ -143,20 +150,20 @@ function buildCallContexts(tokens) {
contexts[tokenIndex] = stack
.filter((frame) => frame.callee !== null)
.map((frame) => ({
callee: frame.callee,
callee: frame.callee,
calleeTokenIndex: frame.calleeTokenIndex,
openTokenIndex: frame.openTokenIndex,
argIndex: frame.argIndex,
openTokenIndex: frame.openTokenIndex,
argIndex: frame.argIndex,
}));
if (token.text === "(") {
const previous = tokens[tokenIndex - 1];
const previous = tokens[tokenIndex - 1];
const hasCallee = previous && previous.kind === "identifier";
stack.push({
callee: hasCallee ? previous.text : null,
callee: hasCallee ? previous.text : null,
calleeTokenIndex: hasCallee ? tokenIndex - 1 : -1,
openTokenIndex: tokenIndex,
argIndex: 0,
openTokenIndex: tokenIndex,
argIndex: 0,
});
continue;
}
+47 -38
View File
@@ -4,8 +4,10 @@ const path = require("node:path");
const { buildCallContexts, lex, nearestCall } = require("./lexer");
const BASE_TYPES = [
"B1", "B2", "B4", "B8", "F4", "F8", "S1", "S2", "S4", "S8",
"U1", "U2", "U4", "U8", "MipsAtom", "MipsCode", "Reg",
"B1", "B2", "B4", "B8",
"F4", "F8", "S1", "S2", "S4", "S8",
"U1", "U2", "U4", "U8",
"MipsAtom", "MipsCode", "Reg",
];
const C_BUILTINS = new Set([
@@ -15,11 +17,13 @@ const C_BUILTINS = new Set([
]);
const BASE_ATTRIBUTES = [
"FI_", "I_", "NI_", "Relative_", "Struct_", "Enum_", "Union_", "Array_",
"Slice_", "TypeR_", "TypeV_", "align_", "internal", "local_persist", "global",
"RO_", "LP_", "gknown", "expect_", "cexpr_",
"FI_", "I_", "NI_",
"Relative_", "Struct_", "Enum_", "Union_", "Array_", "Slice_",
"align_", "internal", "local_persist", "global",
"RO_", "LP_",
"gknown", "expect_", "cexpr_",
"asm", "asm_words", "asm_rpins", "asm_clobber",
"O_", "S_", "C_", "T_", "tmpl", "glue", "r_", "v_", "rt_", "vt_",
"O_", "OA_", "S_", "C_", "T_", "tmpl", "glue", "r_", "v_", "rt_", "vt_",
"rgcc", "r_use", "r_set", "r_mod", "r_imm", "r_mem",
"u1_", "u2_", "u4_", "u8_", "s1_", "s2_", "s4_", "s8_",
"u1_r", "u2_r", "u4_r", "u8_r", "u1_v", "u2_v", "u4_v", "u8_v",
@@ -27,16 +31,16 @@ const BASE_ATTRIBUTES = [
function createIndex() {
return {
atoms: new Set(),
components: new Set(),
atoms: new Set(),
components: new Set(),
componentAliases: new Set(),
macros: new Map(),
registers: new Map(),
bindTypes: new Set(),
types: new Set(BASE_TYPES),
phases: new Set(),
labels: new Set(),
attributes: new Set(BASE_ATTRIBUTES),
macros: new Map(),
registers: new Map(),
bindTypes: new Set(),
types: new Set(BASE_TYPES),
phases: new Set(),
labels: new Set(),
attributes: new Set(BASE_ATTRIBUTES),
componentCallees: new Map(),
};
}
@@ -46,8 +50,8 @@ function cloneIndex(source) {
for (const key of ["atoms", "components", "componentAliases", "bindTypes", "types", "phases", "labels", "attributes"]) {
for (const value of source[key]) result[key].add(value);
}
for (const [name, domain] of source.macros) result.macros.set(name, domain);
for (const [name, domain] of source.registers) result.registers.set(name, domain);
for (const [name, domain] of source.macros) result.macros.set(name, domain);
for (const [name, domain] of source.registers) result.registers.set(name, domain);
for (const [name, callees] of source.componentCallees) result.componentCallees.set(name, callees.slice());
return result;
}
@@ -63,7 +67,7 @@ function mergeIndexes(...sources) {
const existing = result.macros.get(name);
if (!existing || domainRank(domain) >= domainRank(existing)) result.macros.set(name, domain);
}
for (const [name, domain] of source.registers) result.registers.set(name, domain);
for (const [name, domain] of source.registers) result.registers.set(name, domain);
for (const [name, callees] of source.componentCallees) {
const existing = result.componentCallees.get(name) || [];
result.componentCallees.set(name, existing.concat(callees));
@@ -75,21 +79,21 @@ function mergeIndexes(...sources) {
function domainFromPath(filePath) {
const base = path.basename(filePath.replaceAll("\\", "/")).toLowerCase();
if (base === "mips.h") return "cpu";
if (base === "gte.h") return "gte";
if (base === "gp.h") return "gpu";
if (base === "gte.h") return "gte";
if (base === "gp.h") return "gpu";
return null;
}
function prefixDomain(name) {
if (/^(?:branch_|jump_|call_)/.test(name)) return "control";
if (/^gte_(?!cr_)/.test(name) || name.startsWith("mac_gte_") || name.startsWith("ac_gte_")) return "gte";
if (/^gp[01]_/.test(name) || name.startsWith("mac_gp_") || name.startsWith("ac_gp_")) return "gpu";
if (/^gp[01]_/.test(name) || name.startsWith("mac_gp_") || name.startsWith("ac_gp_")) return "gpu";
return null;
}
function collectBraceIdentifiers(tokens, openBraceIndex) {
const names = [];
let depth = 0;
let depth = 0;
for (let tokenIndex = openBraceIndex; tokenIndex < tokens.length; tokenIndex += 1) {
if (tokens[tokenIndex].text === "{") depth += 1;
if (tokens[tokenIndex].text === "}") {
@@ -103,17 +107,17 @@ function collectBraceIdentifiers(tokens, openBraceIndex) {
function resolveComponentDomains(index) {
const hardwareRank = { cpu: 1, gpu: 2, gte: 3, control: 4 };
let changed = true;
let changed = true;
while (changed) {
changed = false;
for (const [alias, callees] of index.componentCallees) {
let best = index.macros.get(alias) || "component";
let best = index.macros.get(alias) || "component";
let bestRank = hardwareRank[best] || 0;
for (const callee of callees) {
const domain = prefixDomain(callee) || index.macros.get(callee);
const rank = hardwareRank[domain] || 0;
const rank = hardwareRank[domain] || 0;
if (rank > bestRank) {
best = domain;
best = domain;
bestRank = rank;
}
}
@@ -134,7 +138,7 @@ function domainRank(domain) {
}
function registerKind(name) {
if (/^R_[A-Za-z0-9_]+$/.test(name)) return "gpr";
if (/^R_[A-Za-z0-9_]+$/.test(name)) return "gpr";
if (/^(?:C2_|gte_cr_)[A-Za-z0-9_]+$/.test(name)) return "cop2";
return null;
}
@@ -161,14 +165,15 @@ function findFunctionNameBefore(tokens, calleeTokenIndex) {
return null;
}
function scanSource(source, filePath) {
const lexical = lex(source);
const balanced = buildCallContexts(lexical.tokens);
const tokens = lexical.tokens;
const contexts = balanced.contexts;
const index = createIndex();
function scanSource(source, filePath)
{
const lexical = lex(source);
const balanced = buildCallContexts(lexical.tokens);
const tokens = lexical.tokens;
const contexts = balanced.contexts;
const index = createIndex();
const declarations = new Map();
const domain = domainFromPath(filePath);
const domain = domainFromPath(filePath);
function mark(token, role, modifiers = ["declaration"]) {
declarations.set(token.start, { role, modifiers });
@@ -191,7 +196,8 @@ function scanSource(source, filePath) {
if (!index.macros.has(alias)) index.macros.set(alias, "component");
}
for (let tokenIndex = 0; tokenIndex < tokens.length; tokenIndex += 1) {
for (let tokenIndex = 0; tokenIndex < tokens.length; tokenIndex += 1)
{
const token = tokens[tokenIndex];
if (token.kind !== "identifier") continue;
@@ -240,12 +246,14 @@ function scanSource(source, filePath) {
mark(token, "gprRegister", ["declaration", "tapeAuto"]);
}
if (token.text === "define" && tokens[tokenIndex - 1] && tokens[tokenIndex - 1].text === "#") {
if (token.text === "define" && tokens[tokenIndex - 1] && tokens[tokenIndex - 1].text === "#")
{
const name = tokens[tokenIndex + 1];
if (name && name.kind === "identifier" && name.line === token.line) {
if (/^(?:RegUse_|Struct_|Enum_|Union_|TypeR_|TypeV_|Relative_|Binds_)/.test(name.text)) {
index.types.add(name.text);
} else if (/^(?:ac_|mac_)/.test(name.text)) {
}
else if (/^(?:ac_|mac_)/.test(name.text)) {
const alias = name.text.startsWith("ac_") ? componentAlias(name.text) : name.text;
const rest = [];
for (let restIndex = tokenIndex + 2; restIndex < tokens.length && tokens[restIndex].line === name.line; restIndex += 1) {
@@ -256,7 +264,8 @@ function scanSource(source, filePath) {
index.macros.set(alias, prefixDomain(alias) || "component");
if (rest.length) index.componentCallees.set(alias, rest);
}
} else {
}
else {
index.macros.set(name.text, domain || "utility");
}
}
+7 -8
View File
@@ -14,8 +14,8 @@
#define glue(A, B) glue_impl(A, B)
#define tmpl(prefix, type) prefix ## _ ## type
#define stringify_impl(S) #S
#define stringify(S) stringify_impl(S)
#define stringify_impl(S) #S
#define stringify(S) stringify_impl(S)
#define VA_Sel_1( _1, ... ) _1 // <-- Of all th args passed pick _1.
#define VA_Sel_2( _1, _2, ... ) _2 // <-- Of all the args passed pick _2.
@@ -121,7 +121,6 @@ typedef unsigned char TSet_(B1);
typedef __UINT16_TYPE__ TSet_(B2);
typedef __UINT32_TYPE__ TSet_(B4);
#define b1_(value) C_(B1, value)
#define b2_(value) C_(B2, value)
#define b4_(value) C_(B4, value)
@@ -196,12 +195,12 @@ def_signed_ops(le, <=)
#undef def_generic_sop
#endif
#define alignas _Alignas
#define alignof _Alignof
#define byte_pad(amount, ...) B1 glue(_PAD_, __VA_ARGS__) [amount]
#define C_ptr(type, data) (C_(type*, & (data)) [0])
#define alignas _Alignas
#define alignof _Alignof
#define byte_pad(amount, ...) B1 glue(_PAD_, __VA_ARGS__) [amount]
#define C_ptr(type, data) (C_(type*, & (data)) [0])
#define dbg_args(...) __VA_ARGS__
#define dbg_args(...) __VA_ARGS__
#pragma region Control Flow & Iteration
#define each_iter(type, iter, end) (type iter = 0; iter < end; ++ iter)
+7
View File
@@ -35,6 +35,7 @@
* These do NOT yield. They are expanded inline inside Tape Atoms.
* ---------------------------------------------------------------------------*/
// The 'Yield' sequence for Tape Atoms (mac_yield).
// In Forth this is considered the "NEXT" mechanism.
#define mac_yield(...) \
load_word(R_AtomJmp, R_TapePtr, 0) \
LdSlot_ \
@@ -55,6 +56,12 @@ WORD_COUNT(mac_yield_load, 1)
, BdSlot_ nop
WORD_COUNT(mac_yield_tail, 3)
/* atom_dbg_skip */
#define mac_yield_to(code_ptr) \
jump_reg(code_ptr) \
, BdSlot_ nop
WORD_COUNT(mac_yield_to, 2)
/* atom_dbg_skip */
#define mac_load_half_v3(tx, ty, tz, base, offset) \
load_half(tx, base, offset + OA_(U2,[0])) \
+1 -1
View File
@@ -271,7 +271,7 @@ typedef Struct_(RegUse_normalize_v3s4) {
union { Reg r1, dst_ptr; };
union { Reg r2, dst_offset, mac1, v_sqr_aligned; };
union { Reg r3, src_offset, btarget, shift_count, sqrtbl_index; };
union { Reg r4, mac3, v_sqr_sum, scale_exp, srav_shift; };
union { Reg r4, mac3, v_sqr_sum, srav_shift; };
union { Reg r5, lzcr, inv_len; };
};
/* ─── Full normalize (all 4 stages inline) ───
+49 -35
View File
@@ -13,50 +13,58 @@
#pragma region Tape Drive
/* -----------------------------------------------------------------------------------------------------------
* TAPE DRIVE ABI
* THREADED ATOMS - TAPE EXECUTION & ABI
* _________
* | ___ |
* | o___o | ,-----<-----.
* |__/___\__| V ^
* \_[Enter]_[A]->[A]->[A]->[A(B)]->[A]->[Exit]
* -----------------------------------------------------------------------------------------------------------
* Note(Ed): One of the main purposes of this codebase is to help me learn this,
* as such the information below may not* be entirely realized or finalized conceptually.
* -----------------------------------------------------------------------------------------------------------
* This ABI and its associated legos were directly inspired by researching the work of
* Timothy Lottes and Onat Türkçüoğlu; along with many others. It's the simplest bootstrap of a
* directly executed chain of assemby arrays (Atoms) that terminate with a yield sequence to the next atom.
* These eventually lead to a terminal atom for the tape which is defined below as "tape_exit".
* This ABI and its associated legos were directly inspired by researching the work of Timothy Lottes and
* Onat Türkçüoğlu; Forth, threaded code system, and various other people or programming techniques.
*
* The setup is simple:
* A tape is a linear stream containing addresses of directly executable native-code fragments ("Atoms").
* Most atoms terminate in a small yield sequence which loads the next atom address from the tape.
* It's a runtime composed of directly executed native machine-code sequences (Atoms) that usually terminate
* in a yield sequence to the next atom. These eventually lead to a terminal atom for the tape
* which is defined below as "tape_exit". Traditionally referred to as Direct Threaded Execution.
*
* It behaves as one of the simplest runtime harnesses ontop of a host-enviornment's execution engine
* to author and compose programs with. From here various conventions can be further applied.
* To make things easier to understand it may be better to focus on what this ABI does not have.
* It does not have have any branching within the tape but relative branches within atoms or between atoms.
* Branching nearly is always downstream. Automatic stack usage is non-existent.
* Push/Pop, FIFO, or Arena/Bump data structures are used by atoms explicitly.
* In it's current form with the C11 macro DSL, the user also has fullfill manual register allocation per atom.
* To make things easier to understand it may be better to focus on what this ABI does not have.
* The tape itself does not have have any branching behavior.
* Branches, loops, skips, or other control-flow policies must be implemented explicitly by atoms.
* Push/Pop, FIFO, or Arena/Bump data structures are utilized by atoms explicitly.
* There is no implicit call-stack, return stack, or per-atom stack-frame.
* The user must also explictly handle register allocation per atom (by default).
* However they could procedurally automate it using metaprogramming functionality.
*
* One of the remarkable things about utilizing this ABI is its essentially interopable with CPUs, GPUs, FPGA,
* or, basically anything from the 5th generation consoles and onward.
* The ABI directly reflects how all computational hardware must be architected in order to execute
* digital logic effectively on current era tech.
* One of the remarkable things about utilizing this ABI's composition model is that its essentially interopable
* with all modern general purpose machines, or, basically anything from the 5th generation consoles and onward.
* This model does not try resolve some optimal runtime for one particular modern machine,
* but adheres the the most bare constraints shared our most common kinds of hardware may all execute.
* On the PS1 we don't have access to a few features like multi-threading, speculative execution, or L3 cache;
* but, we can set the foundation for legoing whats required for eventually expanding this ABI's paradigm
* and core atoms to take those newer hardware features into account. For example, you can easily expand
* this to support wave-based execution model on a PS2 or PS3. Not having a stack or
* automatic register allocation means the user cannot ignore excessive argument shuffle across workload or
* waves and thier phases. Crossing ABI boundaries to other runtimes that do has obviouss penalties.
* this to support multi-threaded execution model on a PS2 or PS3 (or modern machines).
* Not having an implicit-call-frame boundary means register lifetime and data movement remain visible.
* Any poor composition becomes obvious and will convey to the initiated user register shuffling,
* spills, reloads, or any unnecessary traffic they may not have intended (no need to dig through disassembly).
*
* Learning data-oriented code becomes a natural progression. Your not fighting a stack-based procedural
* paradigm that wants to argument shuffle. There is no ambiguity due to the lack of constraints, for example,
* on how the user may "call" a procedure in traditional random dispatch runtimes. The user does have to
* hammer down "rules" or patterns for massaging the compiler to dissolve those call frames; just to get
* hammer down "rules" or Ifpatterns for massaging the compiler to dissolve those call frames; just to get
* the asesmbly into its desired form. The form is obvious, and once the user gets to author these compoonents
* it becomes a game of tetris.
*
* Another feature is this ABI is very compatible with bootstrapping and developing simple toolchains built off
* of bit-packed annotated command streams the user can directly author, maintatain, and immediately execute.
* That being like a color forth, or maybe something more familar like an immediate mode library
* for various systems such as GUIs. This can make the tetris less of a chore with some helpful policy
* (for various systems such as GUIs). This can make the tetris less of a chore with some helpful policy
* generation for allocation of registers, helping to choose resuable components, designing DSL on the fly, etc.
* -----------------------------------------------------------------------------------------------------------
* TODO(Ed): We need pretty ascii diagrams and proper guides, articles, etc.
* -----------------------------------------------------------------------------------------------------------
* For now this ideation has just started functioning. I'm abusing C11 & a lua metaprogram to help establish
* a hybrid toolchain to ideate on a traditional text-based authoring UX for this paradigm.
* If pcsx-redux provides viable hot-reload and persistent data storage beyond save-states
@@ -239,6 +247,9 @@ typedef void Proc_(TapeEntryFn)(MipsAtom* tape_ptr);
FI_ void tape_run(Tape tape) { C_(TapeEntryFn*, tape_enter)(tape.ptr); }
// Procedural authoring of tapes:
typedef Relative_(FArena) Struct_(TapeBuilder) { U4 ptr; U4 capacity; U4 used; };
FI_ void tb_init(TapeBuilder* tb, FArena* arena) { tb->ptr = arena->start; tb->used = 0; }
@@ -272,7 +283,8 @@ FI_ void tb_scope_run_end(TapeBuilder* tb) { tb_emit(tb,tape_exit); tape_run(tb_
* These do NOT yield. They are expanded inline inside Tape Atoms.
* ---------------------------------------------------------------------------*/
// The 'Yield' sequence for Tape Atoms (mac_yield).
// The 'Yield' sequence for Tape Atoms (mac_yield).
// In Forth this is considered the "NEXT" mechanism.
atom_dbg_skip MipsAtomComp_(ac_yield) {
load_word(R_AtomJmp, R_TapePtr, 0), LdSlot_
@@ -360,8 +372,7 @@ typedef Struct_(RegFile) { A2_U2 GPR; };
#define regfile(pin_mask) {.GPR={u4_lo(pin_mask), u4_hi(pin_mask)} }
FI_ void regfile_init(RegFile_R rf) {
/* pack the 32-bit ABI mask into the two U2s */
rf->GPR[0] = u4_lo(regfile_abi_mask);
rf->GPR[1] = u4_hi(regfile_abi_mask);
rf->GPR[0] = u4_lo(regfile_abi_mask); rf->GPR[1] = u4_hi(regfile_abi_mask);
}
FI_ RegFile regfile_make(void) { RegFile rf; regfile_init(& rf); return rf; }
@@ -405,16 +416,19 @@ FI_ void regfile_free_reg(RegFile_R rf, Reg r_id) {
RegFile_RInfo info = regfile_rinfo(rf->GPR, r_id);
info.section[0] &= ~info.mask;
}
FI_ void regfile_reset(RegFile_R rf) {
rf->GPR[0] = u4_lo(regfile_abi_mask);
rf->GPR[1] = u4_hi(regfile_abi_mask);
}
FI_ void regfile_reset_to_mask(RegFile_R rf, U4 mask) {
rf->GPR[0] = u4_lo(mask);
rf->GPR[1] = u4_hi(mask);
}
FI_ void regfile_reset (RegFile_R rf) { rf->GPR[0] = u4_lo(regfile_abi_mask); rf->GPR[1] = u4_hi(regfile_abi_mask); }
FI_ void regfile_reset_to_mask(RegFile_R rf, U4 mask) { rf->GPR[0] = u4_lo(mask); rf->GPR[1] = u4_hi(mask); }
#pragma endregion RegFileArena (Register File Allocator)
#pragma region Mips Atom Components (Procedures)
// For doing direct-chaining of "atoms or fragments".
FI_ Slice_MipsCode ac_yield_to(AtomBuilder_R ab, Reg code_ptr) atom_dbg_skip MipsAtomComp_Proc_(ab, {
jump_reg(code_ptr), BdSlot_ nop,
})
#pragma endregion Mips Atom Components (Procedures)
#pragma region Mips Atom Procs
/* RegUse structs are a convention to organize register allocations for a mips atom procedure.
Unlike the usual enum-based declarations, they provide a namespaced scope and have view types via union declarations. */
+5 -8
View File
@@ -292,6 +292,8 @@ void update(PrimitiveArena* pa, U4* ordering_buf)
gte_matrix_set_rotation (& smem.tform_view);
gte_matrix_set_translation(& smem.tform_view);
// TODO(Ed): We should do a bounds check beforehand to confirm pa can hold all tris?
// The tape atoms in-flight should not need to care.
U1* prim_base = u1_r(pa->buf[smem.active_buf_id]);
U1* prim_cursor = prim_base + pa->used;
tb.used = 0; tb_scope_run(& tb) {
@@ -317,21 +319,16 @@ void update(PrimitiveArena* pa, U4* ordering_buf)
mt3s2s4_rotation (& smem.floor.rot, & smem.tform_world);
mt3s2s4_translation(& smem.tform_world, & smem.floor.pos);
mt3s2s4_scale (& smem.tform_world, & smem.floor.scale);
// Combine world and look_at matrix.
gte_comp_coord_m3s2(& smem.cam.look_at, & smem.tform_world, & smem.tform_view);
gte_matrix_set_rotation (& smem.tform_view);
gte_matrix_set_translation(& smem.tform_view);
U1_R prim_base = u1_r(pa->buf[smem.active_buf_id]);
U1_R prim_cursor = prim_base + pa->used;
// TODO(Ed): We should do a bounds check beforehand to confirm pa can hold all tris?
// The tape atoms in-flight should not need to care.
// Prepare the tape. (Push protocol to tape)
tb.used = 0; tb_scope_run(& tb) {
U1_R prim_base = u1_r(pa->buf[smem.active_buf_id]);
U1_R prim_cursor = prim_base + pa->used;
tb.used = 0; tb_scope_run(& tb) { // Prepare the tape. (Push protocol to tape)
tb_emit(& tb, rbind_floor_f3_face); tb_bind_(& tb, Binds_FloorTri,
.prim_cursor = prim_cursor,
.face_cursor = smem.floor.faces,