initial plugin setup for syntax highlighting in vscode...

This commit is contained in:
ed
2026-08-14 18:57:37 -04:00
parent 67a84d34f3
commit 764ded4557
11 changed files with 1210 additions and 0 deletions
+195
View File
@@ -0,0 +1,195 @@
"use strict";
const { nearestCall } = require("./lexer");
const { mergeIndexes, scanSource } = require("./source-index");
const TOKEN_TYPES = [
"tapeAtomKeyword",
"tapeAtomName",
"tapeComponentKeyword",
"tapeComponentName",
"tapeAnnotation",
"tapeBindType",
"tapePhase",
"tapeLabel",
"tapeCpuInstruction",
"tapeControlFlow",
"tapeGteInstruction",
"tapeGpuInstruction",
"tapeComponentInstruction",
"tapeDelaySlot",
"tapeGprRegister",
"tapeCop2Register",
"tapeDuffleType",
"tapeAttribute",
];
const TOKEN_MODIFIERS = ["declaration", "tapeRead", "tapeWrite", "tapeAuto"];
const TOKEN_TYPE_INDEX = new Map(TOKEN_TYPES.map((name, index) => [name, index]));
const TOKEN_MODIFIER_INDEX = new Map(TOKEN_MODIFIERS.map((name, index) => [name, index]));
const ATOM_KEYWORDS = new Set(["MipsAtom_", "MipsAtom_Proc_"]);
const COMPONENT_KEYWORDS = new Set(["MipsAtomComp_", "MipsAtomComp_Proc_"]);
const ANNOTATIONS = new Set([
"atom_info", "atom_bind", "atom_reads", "atom_writes", "atom_label",
"atom_offset", "atom_reg", "atom_type", "atom_ctx", "atom_phase",
"atom_auto_reg", "phase_auto_reg", "atom_dbg_skip",
]);
const DELAY_SLOT_KEYWORDS = new Set(["LdSlot_", "BdSlot_"]);
const CONTROL_FLOW_PREFIXES = /^(?:branch_|jump_|call_)/;
const ROLE_TO_TYPE = {
atomName: "tapeAtomName",
componentName: "tapeComponentName",
bindType: "tapeBindType",
duffleType: "tapeDuffleType",
gprRegister: "tapeGprRegister",
cop2Register: "tapeCop2Register",
};
function registerType(name, index) {
const kind = index.registers.get(name);
if (kind === "gpr" || /^R_[A-Za-z0-9_]+$/.test(name)) return "tapeGprRegister";
if (kind === "cop2" || /^(?:C2_|gte_cr_)[A-Za-z0-9_]+$/.test(name)) return "tapeCop2Register";
return null;
}
function instructionType(name, index) {
const domain = index.macros.get(name);
if (domain === "cpu") return "tapeCpuInstruction";
if (domain === "gte") return "tapeGteInstruction";
if (domain === "gpu") return "tapeGpuInstruction";
if (domain === "component") return "tapeComponentInstruction";
if (/^gte_(?!cr_)/.test(name)) return "tapeGteInstruction";
if (/^gp[01]_/.test(name)) return "tapeGpuInstruction";
if (/^mac_/.test(name)) return "tapeComponentInstruction";
return null;
}
function modifierMask(modifiers) {
let mask = 0;
for (const modifier of modifiers) {
const index = TOKEN_MODIFIER_INDEX.get(modifier);
if (index !== undefined) mask |= (1 << index);
}
return mask;
}
function isRegUseAccess(tokens, tokenIndex) {
const prev = tokens[tokenIndex - 1];
if (!prev || prev.text !== ".") return false;
const prevPrev = tokens[tokenIndex - 2];
if (!prevPrev || prevPrev.kind !== "identifier") return false;
if (prevPrev.text === "r") return true;
const prev3 = tokens[tokenIndex - 3];
const prev4 = tokens[tokenIndex - 4];
if (prev3 && prev3.text === "." && prev4 && prev4.kind === "identifier" && prev4.text === "r") return true;
return false;
}
function classifyDocument(source, filePath, workspaceIndex, shouldCancel = () => false) {
const scanned = scanSource(source, filePath);
const index = mergeIndexes(workspaceIndex, scanned.index);
const spans = [];
for (let tokenIndex = 0; tokenIndex < scanned.tokens.length; tokenIndex += 1) {
if (shouldCancel()) break;
const token = scanned.tokens[tokenIndex];
if (token.kind !== "identifier") continue;
let type = null;
let modifiers = [];
const declaration = scanned.declarations.get(token.start);
const context = nearestCall(scanned.contexts, tokenIndex);
if (declaration) {
type = ROLE_TO_TYPE[declaration.role] || null;
modifiers = declaration.modifiers.slice();
} else if (ATOM_KEYWORDS.has(token.text)) {
type = "tapeAtomKeyword";
} else if (COMPONENT_KEYWORDS.has(token.text)) {
type = "tapeComponentKeyword";
} else if (ANNOTATIONS.has(token.text)) {
type = "tapeAnnotation";
} else if (context && context.callee === "atom_bind" && context.argIndex === 0) {
type = "tapeBindType";
} else if (context && context.callee === "atom_phase" && context.argIndex === 0) {
type = "tapePhase";
modifiers = ["declaration"];
} else if (context && context.callee === "atom_ctx" && context.argIndex === 0) {
type = "tapeAtomName";
} else if (context && context.callee === "atom_label" && context.argIndex === 0) {
type = "tapeLabel";
modifiers = ["declaration"];
} else if (context && context.callee === "atom_offset" && context.argIndex <= 1) {
type = "tapeLabel";
} else if (context && context.callee === "atom_reads") {
type = registerType(token.text, index);
if (type) modifiers = ["tapeRead"];
} else if (context && context.callee === "atom_writes") {
type = registerType(token.text, index);
if (type) modifiers = ["tapeWrite"];
} else if (context && context.callee === "atom_auto_reg") {
if (context.argIndex === 0) type = "tapeAtomName";
if (context.argIndex === 1) {
type = "tapeGprRegister";
modifiers = ["declaration", "tapeAuto"];
}
} else if (context && context.callee === "phase_auto_reg") {
if (context.argIndex === 0) type = "tapePhase";
if (context.argIndex === 1) {
type = "tapeGprRegister";
modifiers = ["declaration", "tapeAuto"];
}
}
if (!type && index.bindTypes.has(token.text)) type = "tapeBindType";
if (!type && index.types.has(token.text)) type = "tapeDuffleType";
if (!type && index.attributes.has(token.text)) type = "tapeAttribute";
if (!type) type = registerType(token.text, index);
if (!type && DELAY_SLOT_KEYWORDS.has(token.text)) type = "tapeDelaySlot";
if (!type) {
const domain = index.macros.get(token.text);
if (domain && CONTROL_FLOW_PREFIXES.test(token.text)) {
type = "tapeControlFlow";
}
}
if (!type && isRegUseAccess(scanned.tokens, tokenIndex)) type = "tapeGprRegister";
if (!type) type = instructionType(token.text, index);
if (!type && index.atoms.has(token.text)) type = "tapeAtomName";
if (!type && index.components.has(token.text)) type = "tapeComponentName";
if (!type && index.phases.has(token.text)) type = "tapePhase";
if (!type && index.labels.has(token.text)) type = "tapeLabel";
if (!type) continue;
spans.push({
text: token.text,
type,
typeIndex: TOKEN_TYPE_INDEX.get(type),
modifiers,
modifierMask: modifierMask(modifiers),
start: token.start,
length: token.end - token.start,
line: token.line,
character: token.character,
});
}
spans.sort((left, right) => left.start - right.start || left.length - right.length);
const nonOverlapping = [];
for (const span of spans) {
const previous = nonOverlapping[nonOverlapping.length - 1];
if (!previous || previous.start + previous.length <= span.start) nonOverlapping.push(span);
}
return { spans: nonOverlapping, errors: scanned.errors };
}
module.exports = {
TOKEN_MODIFIERS,
TOKEN_TYPES,
classifyDocument,
modifierMask,
};
+111
View File
@@ -0,0 +1,111 @@
"use strict";
const vscode = require("vscode");
const { TOKEN_MODIFIERS, TOKEN_TYPES, classifyDocument } = require("./classifier");
const { createIndex, mergeIndexes, scanSource } = require("./source-index");
const SOURCE_GLOB = "**/*.{c,h,cc,cpp,cxx,hh,hpp,hxx}";
const EXCLUDE_GLOB = "**/{gen,build,.slop_cache,toolchain,node_modules}/**";
const EXCLUDED_SEGMENTS = new Set(["gen", "build", ".slop_cache", "toolchain", "node_modules"]);
function isExcluded(uri) {
const segments = uri.fsPath.replaceAll("\\", "/").split("/");
return segments.some((segment) => EXCLUDED_SEGMENTS.has(segment));
}
function formatError(filePath, error) {
return `${filePath}:${error.offset}: ${error.kind}`;
}
async function activate(context) {
const output = vscode.window.createOutputChannel("Tape Atom DSL");
const emitter = new vscode.EventEmitter();
const legend = new vscode.SemanticTokensLegend(TOKEN_TYPES, TOKEN_MODIFIERS);
let workspaceIndex = createIndex();
let rebuildGeneration = 0;
let debounceHandle = null;
async function rebuildIndex() {
const generation = ++rebuildGeneration;
const files = await vscode.workspace.findFiles(SOURCE_GLOB, EXCLUDE_GLOB);
let nextIndex = createIndex();
for (const uri of files) {
if (generation !== rebuildGeneration) return;
if (isExcluded(uri)) continue;
try {
const bytes = await vscode.workspace.fs.readFile(uri);
const source = Buffer.from(bytes).toString("utf8");
const result = scanSource(source, uri.fsPath);
nextIndex = mergeIndexes(nextIndex, result.index);
for (const error of result.errors) output.appendLine(formatError(uri.fsPath, error));
} catch (error) {
output.appendLine(`${uri.fsPath}: ${error.stack || error.message || error}`);
}
}
if (generation !== rebuildGeneration) return;
workspaceIndex = nextIndex;
emitter.fire();
}
function scheduleRebuild(uri) {
if (uri && isExcluded(uri)) return;
if (debounceHandle !== null) clearTimeout(debounceHandle);
debounceHandle = setTimeout(() => {
debounceHandle = null;
rebuildIndex().catch((error) => output.appendLine(error.stack || String(error)));
}, 100);
}
const provider = {
onDidChangeSemanticTokens: emitter.event,
provideDocumentSemanticTokens(document, cancellationToken) {
try {
const result = classifyDocument(
document.getText(),
document.uri.fsPath,
workspaceIndex,
() => cancellationToken.isCancellationRequested
);
const builder = new vscode.SemanticTokensBuilder(legend);
for (const span of result.spans) {
if (cancellationToken.isCancellationRequested) break;
builder.push(span.line, span.character, span.length, span.typeIndex, span.modifierMask);
}
for (const error of result.errors) {
output.appendLine(formatError(document.uri.fsPath || document.uri.toString(), error));
}
return builder.build();
} catch (error) {
output.appendLine(`${document.uri}: ${error.stack || error.message || error}`);
return new vscode.SemanticTokensBuilder(legend).build();
}
},
};
const selector = [
{ language: "c", scheme: "file" },
{ language: "c", scheme: "untitled" },
{ language: "cpp", scheme: "file" },
{ language: "cpp", scheme: "untitled" },
];
const watcher = vscode.workspace.createFileSystemWatcher(SOURCE_GLOB);
context.subscriptions.push(
output,
emitter,
watcher,
watcher.onDidCreate(scheduleRebuild),
watcher.onDidChange(scheduleRebuild),
watcher.onDidDelete(scheduleRebuild),
vscode.languages.registerDocumentSemanticTokensProvider(selector, provider, legend),
{ dispose() { if (debounceHandle !== null) clearTimeout(debounceHandle); } }
);
await rebuildIndex();
}
function deactivate() {}
module.exports = { activate, deactivate };
+186
View File
@@ -0,0 +1,186 @@
"use strict";
function isIdentifierStart(code) {
return code === 95 ||
(code >= 65 && code <= 90) ||
(code >= 97 && code <= 122);
}
function isIdentifierContinue(code) {
return isIdentifierStart(code) || (code >= 48 && code <= 57);
}
function lex(source) {
if (typeof source !== "string") throw new TypeError("source must be a string");
const tokens = [];
const errors = [];
let offset = 0;
let line = 0;
let character = 0;
function advance() {
if (source[offset] === "\r" && source[offset + 1] === "\n") {
offset += 2;
line += 1;
character = 0;
return;
}
if (source[offset] === "\n") {
offset += 1;
line += 1;
character = 0;
return;
}
offset += 1;
character += 1;
}
function pushToken(kind, start, startLine, startCharacter) {
tokens.push({
kind,
text: source.slice(start, offset),
start,
end: offset,
line: startLine,
character: startCharacter,
});
}
while (offset < source.length) {
const ch = source[offset];
if (/\s/.test(ch)) {
advance();
continue;
}
if (ch === "/" && source[offset + 1] === "/") {
while (offset < source.length && source[offset] !== "\r" && source[offset] !== "\n") advance();
continue;
}
if (ch === "/" && source[offset + 1] === "*") {
const start = offset;
advance();
advance();
let closed = false;
while (offset < source.length) {
if (source[offset] === "*" && source[offset + 1] === "/") {
advance();
advance();
closed = true;
break;
}
advance();
}
if (!closed) errors.push({ kind: "unterminated-block-comment", offset: start });
continue;
}
if (ch === "\"" || ch === "'") {
const quote = ch;
const start = offset;
advance();
let closed = false;
while (offset < source.length) {
if (source[offset] === "\\") {
advance();
if (offset < source.length) advance();
continue;
}
if (source[offset] === quote) {
advance();
closed = true;
break;
}
if (source[offset] === "\n" || source[offset] === "\r") break;
advance();
}
if (!closed) errors.push({ kind: "unterminated-literal", offset: start });
continue;
}
const code = source.charCodeAt(offset);
if (isIdentifierStart(code)) {
const start = offset;
const startLine = line;
const startCharacter = character;
advance();
while (offset < source.length && isIdentifierContinue(source.charCodeAt(offset))) advance();
pushToken("identifier", start, startLine, startCharacter);
continue;
}
const start = offset;
const startLine = line;
const startCharacter = character;
advance();
pushToken("punctuation", start, startLine, startCharacter);
}
return { tokens, errors };
}
function buildCallContexts(tokens) {
const contexts = Array.from({ length: tokens.length }, () => []);
const calls = [];
const errors = [];
const stack = [];
for (let tokenIndex = 0; tokenIndex < tokens.length; tokenIndex += 1) {
const token = tokens[tokenIndex];
if (token.text === ")") {
if (stack.length === 0) {
errors.push({ kind: "unmatched-close-paren", offset: token.start });
} else {
const frame = stack.pop();
if (frame.callee !== null) calls.push({ ...frame, closeTokenIndex: tokenIndex });
}
}
contexts[tokenIndex] = stack
.filter((frame) => frame.callee !== null)
.map((frame) => ({
callee: frame.callee,
calleeTokenIndex: frame.calleeTokenIndex,
openTokenIndex: frame.openTokenIndex,
argIndex: frame.argIndex,
}));
if (token.text === "(") {
const previous = tokens[tokenIndex - 1];
const hasCallee = previous && previous.kind === "identifier";
stack.push({
callee: hasCallee ? previous.text : null,
calleeTokenIndex: hasCallee ? tokenIndex - 1 : -1,
openTokenIndex: tokenIndex,
argIndex: 0,
});
continue;
}
if (token.text === "," && stack.length > 0) {
const frame = stack[stack.length - 1];
if (frame.callee !== null) frame.argIndex += 1;
}
}
for (const frame of stack) {
errors.push({ kind: "unmatched-open-paren", offset: tokens[frame.openTokenIndex].start });
}
return { contexts, calls, errors };
}
function nearestCall(contexts, tokenIndex, callee) {
const entries = contexts[tokenIndex] || [];
for (let contextIndex = entries.length - 1; contextIndex >= 0; contextIndex -= 1) {
const entry = entries[contextIndex];
if (callee === undefined || entry.callee === callee) return entry;
}
return null;
}
module.exports = { buildCallContexts, lex, nearestCall };
+81
View File
@@ -0,0 +1,81 @@
{
"name": "tape-atom-syntax",
"displayName": "Tape Atom DSL",
"description": "Source-derived semantic highlighting for the Tape/Atom MIPS macro DSL",
"publisher": "local",
"version": "0.2.0",
"engines": { "vscode": "^1.80.0" },
"categories": ["Programming Languages"],
"activationEvents": ["onLanguage:c", "onLanguage:cpp"],
"main": "./extension.js",
"files": [
"classifier.js",
"extension.js",
"lexer.js",
"source-index.js",
"syntaxes/tape_atom.tmLanguage.json"
],
"scripts": {
"test": "node --test test/*.test.js",
"package": "npx --yes @vscode/vsce@3.6.1 package --allow-missing-repository --skip-license --out tape-atom-syntax-0.2.0.vsix"
},
"contributes": {
"semanticTokenTypes": [
{ "id": "tapeAtomKeyword", "description": "Tape atom declaration keyword" },
{ "id": "tapeAtomName", "description": "Tape atom name" },
{ "id": "tapeComponentKeyword", "description": "Tape atom component declaration keyword" },
{ "id": "tapeComponentName", "description": "Tape atom component name" },
{ "id": "tapeAnnotation", "description": "Tape atom annotation" },
{ "id": "tapeBindType", "description": "Tape bind structure type" },
{ "id": "tapePhase", "description": "Tape atom phase" },
{ "id": "tapeLabel", "description": "Tape atom branch label" },
{ "id": "tapeCpuInstruction", "description": "MIPS CPU instruction emitter" },
{ "id": "tapeControlFlow", "description": "MIPS branch or jump instruction" },
{ "id": "tapeGteInstruction", "description": "GTE instruction emitter" },
{ "id": "tapeGpuInstruction", "description": "GPU command emitter" },
{ "id": "tapeComponentInstruction", "description": "Tape atom component invocation" },
{ "id": "tapeDelaySlot", "description": "Load or branch delay slot annotation" },
{ "id": "tapeGprRegister", "description": "MIPS GPR alias" },
{ "id": "tapeCop2Register", "description": "COP2 data or control register alias" },
{ "id": "tapeDuffleType", "description": "Duffle type or type constructor" },
{ "id": "tapeAttribute", "description": "Duffle linkage or storage attribute" }
],
"semanticTokenModifiers": [
{ "id": "tapeRead", "description": "Register declared in atom_reads" },
{ "id": "tapeWrite", "description": "Register declared in atom_writes" },
{ "id": "tapeAuto", "description": "Auto-allocated register" }
],
"semanticTokenScopes": [
{
"language": "c",
"scopes": {
"tapeAtomKeyword": ["keyword.control.duffle.atom"],
"tapeAtomName": ["entity.name.function.duffle.atom"],
"tapeComponentKeyword": ["keyword.control.duffle.component"],
"tapeComponentName": ["entity.name.function.duffle.component"],
"tapeAnnotation": ["support.function.duffle.annotation"],
"tapeBindType": ["entity.name.type.duffle.bind"],
"tapePhase": ["entity.name.tag.duffle.phase"],
"tapeLabel": ["entity.name.label.duffle.atom"],
"tapeCpuInstruction": ["support.function.duffle.cpu"],
"tapeControlFlow": ["keyword.control.duffle.branch"],
"tapeGteInstruction": ["support.function.duffle.gte"],
"tapeGpuInstruction": ["support.function.duffle.gpu"],
"tapeComponentInstruction": ["support.function.duffle.component"],
"tapeDelaySlot": ["keyword.operator.duffle.delayslot"],
"tapeGprRegister": ["variable.other.constant.duffle.gpr"],
"tapeCop2Register": ["variable.other.constant.duffle.cop2"],
"tapeDuffleType": ["storage.type.duffle.type"],
"tapeAttribute": ["storage.modifier.duffle.attr"]
}
}
],
"grammars": [
{
"scopeName": "tape_atom.injection",
"path": "./syntaxes/tape_atom.tmLanguage.json",
"injectTo": ["source.c", "source.cpp"]
}
]
}
}
+222
View File
@@ -0,0 +1,222 @@
"use strict";
const path = require("node:path");
const { buildCallContexts, lex, nearestCall } = require("./lexer");
const BASE_TYPES = [
"B1", "B2", "B4", "B8", "F4", "F8", "S1", "S2", "S4", "S8",
"U1", "U2", "U4", "U8", "MipsAtom", "MipsCode", "Reg",
];
const BASE_ATTRIBUTES = [
"FI_", "I_", "NI_", "Relative_", "Struct_", "Enum_", "Union_",
"TypeR_", "TypeV_", "align_", "internal", "local_persist", "global",
];
function createIndex() {
return {
atoms: new Set(),
components: new Set(),
componentAliases: new Set(),
macros: new Map(),
registers: new Map(),
bindTypes: new Set(),
types: new Set(BASE_TYPES),
phases: new Set(),
labels: new Set(),
attributes: new Set(BASE_ATTRIBUTES),
};
}
function cloneIndex(source) {
const result = createIndex();
for (const key of ["atoms", "components", "componentAliases", "bindTypes", "types", "phases", "labels", "attributes"]) {
for (const value of source[key]) result[key].add(value);
}
for (const [name, domain] of source.macros) result.macros.set(name, domain);
for (const [name, domain] of source.registers) result.registers.set(name, domain);
return result;
}
function mergeIndexes(...sources) {
const result = createIndex();
for (const source of sources) {
if (!source) continue;
for (const key of ["atoms", "components", "componentAliases", "bindTypes", "types", "phases", "labels", "attributes"]) {
for (const value of source[key]) result[key].add(value);
}
for (const [name, domain] of source.macros) result.macros.set(name, domain);
for (const [name, domain] of source.registers) result.registers.set(name, domain);
}
return result;
}
function domainFromPath(filePath) {
const base = path.basename(filePath.replaceAll("\\", "/")).toLowerCase();
if (base === "mips.h" || base === "mips.atom.c") return "cpu";
if (base === "gte.h" || base === "gte.atom.c") return "gte";
if (base === "gp.h" || base === "gp.atom.c") return "gpu";
return "component";
}
function registerKind(name) {
if (/^R_[A-Za-z0-9_]+$/.test(name)) return "gpr";
if (/^(?:C2_|gte_cr_)[A-Za-z0-9_]+$/.test(name)) return "cop2";
return null;
}
function componentAlias(name) {
return name.startsWith("ac_") ? `mac_${name.slice(3)}` : null;
}
function findFunctionNameBefore(tokens, calleeTokenIndex) {
let closeIndex = calleeTokenIndex - 1;
while (closeIndex >= 0 && tokens[closeIndex].kind === "identifier" && tokens[closeIndex].text === "atom_dbg_skip") {
closeIndex -= 1;
}
if (!tokens[closeIndex] || tokens[closeIndex].text !== ")") return null;
let depth = 1;
for (let tokenIndex = closeIndex - 1; tokenIndex >= 0; tokenIndex -= 1) {
if (tokens[tokenIndex].text === ")") depth += 1;
if (tokens[tokenIndex].text === "(") depth -= 1;
if (depth !== 0) continue;
const name = tokens[tokenIndex - 1];
return name && name.kind === "identifier" ? name : null;
}
return null;
}
function scanSource(source, filePath) {
const lexical = lex(source);
const balanced = buildCallContexts(lexical.tokens);
const tokens = lexical.tokens;
const contexts = balanced.contexts;
const index = createIndex();
const declarations = new Map();
const domain = domainFromPath(filePath);
function mark(token, role, modifiers = ["declaration"]) {
declarations.set(token.start, { role, modifiers });
}
function addComponent(token) {
index.components.add(token.text);
mark(token, "componentName");
const alias = componentAlias(token.text);
if (alias) {
index.componentAliases.add(alias);
index.macros.set(alias, domain);
}
}
for (let tokenIndex = 0; tokenIndex < tokens.length; tokenIndex += 1) {
const token = tokens[tokenIndex];
if (token.kind !== "identifier") continue;
const kind = registerKind(token.text);
if (kind) {
index.registers.set(token.text, kind);
if (tokens[tokenIndex + 1] && tokens[tokenIndex + 1].text === "=") {
mark(token, kind === "gpr" ? "gprRegister" : "cop2Register");
}
}
const context = nearestCall(contexts, tokenIndex);
if (context && context.argIndex === 0) {
if (context.callee === "MipsAtom_") {
index.atoms.add(token.text);
mark(token, "atomName");
}
if (context.callee === "MipsAtomComp_") addComponent(token);
if (context.callee === "atom_bind") index.bindTypes.add(token.text);
if (context.callee === "atom_phase" || context.callee === "phase_auto_reg") index.phases.add(token.text);
if (context.callee === "atom_label" || context.callee === "atom_offset") index.labels.add(token.text);
}
const isWrappedType = context && (
((context.callee === "Struct_" || context.callee === "Union_") && context.argIndex === 0) ||
(context.callee === "Enum_" && context.argIndex === 1)
);
if (isWrappedType) {
index.types.add(token.text);
mark(token, token.text.startsWith("Binds_") ? "bindType" : "duffleType");
if (token.text.startsWith("Binds_")) index.bindTypes.add(token.text);
}
if (context && context.callee === "atom_offset" && context.argIndex === 1) index.labels.add(token.text);
if (context && context.callee === "atom_auto_reg") {
if (context.argIndex === 0) index.atoms.add(token.text);
if (context.argIndex === 1) {
index.registers.set(token.text, "gpr");
mark(token, "gprRegister", ["declaration", "tapeAuto"]);
}
}
if (context && context.callee === "phase_auto_reg" && context.argIndex === 1) {
index.registers.set(token.text, "gpr");
mark(token, "gprRegister", ["declaration", "tapeAuto"]);
}
if (token.text === "define" && tokens[tokenIndex - 1] && tokens[tokenIndex - 1].text === "#") {
const name = tokens[tokenIndex + 1];
if (name && name.kind === "identifier" && name.line === token.line) index.macros.set(name.text, domain);
}
if (token.text === "typedef") {
let endIndex = tokenIndex + 1;
let hasBrace = false;
let lastIdentifier = null;
while (endIndex < tokens.length && tokens[endIndex].text !== ";") {
if (tokens[endIndex].text === "{") hasBrace = true;
if (tokens[endIndex].kind === "identifier") lastIdentifier = tokens[endIndex];
endIndex += 1;
}
if (!hasBrace && lastIdentifier) {
index.types.add(lastIdentifier.text);
mark(lastIdentifier, "duffleType");
}
}
if (token.text === "MipsAtom_Proc_") {
const functionName = findFunctionNameBefore(tokens, tokenIndex);
if (functionName) {
const atomName = functionName.text.endsWith("_proc")
? functionName.text.slice(0, -5)
: functionName.text;
index.atoms.add(atomName);
index.atoms.add(functionName.text);
mark(functionName, "atomName");
}
}
if (token.text === "MipsAtomComp_Proc_") {
const functionName = findFunctionNameBefore(tokens, tokenIndex);
if (functionName) addComponent(functionName);
}
}
for (const call of balanced.calls) {
if (domain === "component") continue;
const name = tokens[call.calleeTokenIndex];
const after = tokens[call.closeTokenIndex + 1];
if (!name || !after || after.text !== "{") continue;
if (/^(?:gp0_|gp1_|gte_|mac_)/.test(name.text)) index.macros.set(name.text, domain);
}
return {
index: cloneIndex(index),
declarations,
tokens,
contexts,
errors: [...lexical.errors, ...balanced.errors],
};
}
module.exports = {
createIndex,
domainFromPath,
mergeIndexes,
scanSource,
};
@@ -0,0 +1,85 @@
{
"scopeName": "tape_atom.injection",
"injectionSelector": "L:source.c -comment -string, L:source.cpp -comment -string",
"patterns": [
{ "include": "#atom-declarations" },
{ "include": "#component-declarations" },
{ "include": "#annotation-arguments" },
{ "include": "#annotations" },
{ "include": "#instructions" },
{ "include": "#registers" },
{ "include": "#types" },
{ "include": "#attributes" }
],
"repository": {
"atom-declarations": {
"patterns": [
{
"match": "\\b(MipsAtom_)\\s*\\(\\s*([A-Za-z_][A-Za-z0-9_]*)",
"captures": {
"1": { "name": "keyword.control.duffle.atom" },
"2": { "name": "entity.name.function.duffle.atom" }
}
},
{ "match": "\\bMipsAtom_Proc_\\b", "name": "keyword.control.duffle.atom" },
{ "match": "\\b[A-Za-z_][A-Za-z0-9_]*_proc\\b", "name": "entity.name.function.duffle.atom" }
]
},
"component-declarations": {
"patterns": [
{
"match": "\\b(MipsAtomComp_)\\s*\\(\\s*(ac_[A-Za-z0-9_]*)",
"captures": {
"1": { "name": "keyword.control.duffle.component" },
"2": { "name": "entity.name.function.duffle.component" }
}
},
{ "match": "\\bMipsAtomComp_Proc_\\b", "name": "keyword.control.duffle.component" },
{ "match": "\\bac_[A-Za-z0-9_]+\\b", "name": "entity.name.function.duffle.component" }
]
},
"annotation-arguments": {
"patterns": [
{ "match": "(?<=\\batom_bind\\()\\s*Binds_[A-Za-z0-9_]+", "name": "entity.name.type.duffle.bind" },
{ "match": "(?<=\\batom_phase\\()\\s*[A-Za-z_][A-Za-z0-9_]*", "name": "entity.name.tag.duffle.phase" },
{ "match": "(?<=\\batom_label\\()\\s*[A-Za-z_][A-Za-z0-9_]*", "name": "entity.name.label.duffle.atom" },
{
"match": "\\b(atom_offset)\\s*\\(\\s*([A-Za-z_][A-Za-z0-9_]*)\\s*,\\s*([A-Za-z_][A-Za-z0-9_]*)",
"captures": {
"1": { "name": "support.function.duffle.annotation" },
"2": { "name": "entity.name.label.duffle.atom" },
"3": { "name": "entity.name.label.duffle.atom" }
}
}
]
},
"annotations": {
"match": "\\b(atom_info|atom_bind|atom_reads|atom_writes|atom_label|atom_offset|atom_reg|atom_type|atom_ctx|atom_phase|atom_auto_reg|phase_auto_reg|atom_dbg_skip)\\b",
"name": "support.function.duffle.annotation"
},
"instructions": {
"patterns": [
{ "match": "\\b(branch_|jump_|jump_rel|call_)[A-Za-z0-9_]*\\b", "name": "keyword.control.duffle.branch" },
{ "match": "\\b(LdSlot_|BdSlot_)\\b", "name": "keyword.operator.duffle.delayslot" },
{ "match": "\\b(load_|store_|add_|sub_|shift_|set_lt_|and_i|or_i|xor_i|nop[0-9]*)[A-Za-z0-9_]*\\b", "name": "support.function.duffle.cpu" },
{ "match": "\\bgte_(?!cr_)[A-Za-z0-9_]+\\b", "name": "support.function.duffle.gte" },
{ "match": "\\bgp[01]_[A-Za-z0-9_]+\\b", "name": "support.function.duffle.gpu" },
{ "match": "\\bmac_[A-Za-z0-9_]+\\b", "name": "support.function.duffle.component" }
]
},
"registers": {
"patterns": [
{ "match": "\\bR_[A-Za-z0-9_]+\\b", "name": "variable.other.constant.duffle.gpr" },
{ "match": "\\b(?:C2_|gte_cr_)[A-Za-z0-9_]+\\b", "name": "variable.other.constant.duffle.cop2" }
]
},
"types": {
"match": "\\b(?:[USFB][1248]|MipsAtom|MipsCode|Reg|Binds_[A-Za-z0-9_]+|Struct_|Enum_|Union_|TypeR_|TypeV_)\\b",
"name": "storage.type.duffle.type"
},
"attributes": {
"match": "\\b(?:FI_|I_|NI_|Relative_|align_|internal|local_persist|global)\\b",
"name": "storage.modifier.duffle.attr"
}
}
}
Binary file not shown.
+83
View File
@@ -0,0 +1,83 @@
"use strict";
const assert = require("node:assert/strict");
const test = require("node:test");
const { classifyDocument } = require("../classifier");
const { createIndex } = require("../source-index");
function byText(result, text) {
return result.spans.filter((span) => span.text === text);
}
test("classifyDocument distinguishes declaration, annotation, phase, bind, and label roles", () => {
const source = [
"typedef Struct_(Binds_CubeTri) { U4 PrimCursor; };",
"MipsAtom_(cube_g4_face) atom_info(atom_bind(Binds_CubeTri), atom_phase(cube_g4),",
"\tatom_reads(R_PrimCursor), atom_writes(R_FaceCursor)) {",
"\tbranch_le_zero(R_T0, atom_offset(cull, exit)),",
"\tatom_label(exit)",
"};",
].join("\n");
const result = classifyDocument(source, "C:/x/code/hello_camera/hello_camera.atom.c", createIndex());
assert.equal(byText(result, "MipsAtom_")[0].type, "tapeAtomKeyword");
assert.deepEqual(byText(result, "cube_g4_face")[0].modifiers, ["declaration"]);
assert.equal(byText(result, "atom_bind")[0].type, "tapeAnnotation");
assert.equal(byText(result, "Binds_CubeTri").at(-1).type, "tapeBindType");
assert.equal(byText(result, "cube_g4")[0].type, "tapePhase");
assert.deepEqual(byText(result, "cube_g4")[0].modifiers, ["declaration"]);
assert.equal(byText(result, "cull")[0].type, "tapeLabel");
assert.equal(byText(result, "exit").every((span) => span.type === "tapeLabel"), true);
});
test("classifyDocument applies read and write modifiers to GPRs", () => {
const source = "atom_info(atom_reads(R_PrimCursor), atom_writes(R_FaceCursor))";
const result = classifyDocument(source, "C:/x/code/test.atom.c", createIndex());
assert.deepEqual(byText(result, "R_PrimCursor")[0].modifiers, ["tapeRead"]);
assert.deepEqual(byText(result, "R_FaceCursor")[0].modifiers, ["tapeWrite"]);
});
test("classifyDocument separates CPU, GTE, GPU, and component domains", () => {
const workspace = createIndex();
workspace.macros.set("load_word", "cpu");
workspace.macros.set("gte_cmdw_rtpt", "gte");
workspace.macros.set("gp1_word_DisplayOn", "gpu");
workspace.macros.set("mac_yield", "component");
workspace.componentAliases.add("mac_yield");
const source = "load_word(R_T0, R_T1, 0), gte_cmdw_rtpt, gp1_word_DisplayOn(), mac_yield(), C2_MAC0, gte_cr_OFX_Code";
const result = classifyDocument(source, "C:/x/code/test.c", workspace);
assert.equal(byText(result, "load_word")[0].type, "tapeCpuInstruction");
assert.equal(byText(result, "gte_cmdw_rtpt")[0].type, "tapeGteInstruction");
assert.equal(byText(result, "gp1_word_DisplayOn")[0].type, "tapeGpuInstruction");
assert.equal(byText(result, "mac_yield")[0].type, "tapeComponentInstruction");
assert.equal(byText(result, "C2_MAC0")[0].type, "tapeCop2Register");
assert.equal(byText(result, "gte_cr_OFX_Code")[0].type, "tapeCop2Register");
});
test("document-local declarations override an empty workspace index", () => {
const source = [
"MipsAtomComp_(ac_new_component) { nop };",
"mac_new_component(),",
].join("\n");
const result = classifyDocument(source, "C:/x/code/duffle/math.atom.c", createIndex());
assert.equal(byText(result, "ac_new_component")[0].type, "tapeComponentName");
assert.equal(byText(result, "mac_new_component")[0].type, "tapeComponentInstruction");
});
test("classifier returns ordered non-overlapping spans and partial malformed output", () => {
const source = "atom_reads(R_A /* broken";
const result = classifyDocument(source, "C:/x/code/test.atom.c", createIndex());
assert.equal(result.errors.some((error) => error.kind === "unterminated-block-comment"), true);
for (let spanIndex = 1; spanIndex < result.spans.length; spanIndex += 1) {
const previous = result.spans[spanIndex - 1];
const current = result.spans[spanIndex];
assert.equal(previous.start + previous.length <= current.start, true);
}
});
+93
View File
@@ -0,0 +1,93 @@
"use strict";
const assert = require("node:assert/strict");
const fs = require("node:fs");
const path = require("node:path");
const test = require("node:test");
const { TOKEN_MODIFIERS, TOKEN_TYPES } = require("../classifier");
const ROOT = path.resolve(__dirname, "..");
function readJson(filePath) {
return JSON.parse(fs.readFileSync(filePath, "utf8"));
}
function collectScopeNames(value, output = new Set()) {
if (Array.isArray(value)) {
for (const entry of value) collectScopeNames(entry, output);
return output;
}
if (!value || typeof value !== "object") return output;
if (typeof value.name === "string") output.add(value.name);
for (const child of Object.values(value)) collectScopeNames(child, output);
return output;
}
test("package semantic legend matches classifier exports", () => {
const packageJson = readJson(path.join(ROOT, "package.json"));
const contributedTypes = packageJson.contributes.semanticTokenTypes.map((entry) => entry.id);
const contributedModifiers = packageJson.contributes.semanticTokenModifiers.map((entry) => entry.id);
assert.equal(packageJson.version, "0.2.0");
assert.deepEqual(contributedTypes, TOKEN_TYPES);
assert.deepEqual(contributedModifiers, TOKEN_MODIFIERS.filter((name) => name !== "declaration"));
});
test("package includes runtime files only and acknowledges local-only metadata", () => {
const packageJson = readJson(path.join(ROOT, "package.json"));
assert.deepEqual(packageJson.files, [
"classifier.js",
"extension.js",
"lexer.js",
"source-index.js",
"syntaxes/tape_atom.tmLanguage.json",
]);
assert.equal(packageJson.scripts.package.includes("--allow-missing-repository"), true);
assert.equal(packageJson.scripts.package.includes("--skip-license"), true);
});
test("every semantic token has a TextMate fallback scope and grammar scope", () => {
const packageJson = readJson(path.join(ROOT, "package.json"));
const grammar = readJson(path.join(ROOT, "syntaxes", "tape_atom.tmLanguage.json"));
const mappings = packageJson.contributes.semanticTokenScopes[0].scopes;
const grammarScopes = collectScopeNames(grammar);
for (const tokenType of TOKEN_TYPES) {
assert.equal(Array.isArray(mappings[tokenType]), true, `missing scope mapping: ${tokenType}`);
assert.equal(mappings[tokenType].some((scope) => grammarScopes.has(scope)), true, `grammar does not emit: ${tokenType}`);
}
});
test("TextMate offset labels stay scoped to atom_offset calls", () => {
const grammar = readJson(path.join(ROOT, "syntaxes", "tape_atom.tmLanguage.json"));
const serialized = JSON.stringify(grammar);
const offsetRule = grammar.repository["annotation-arguments"].patterns
.find((rule) => rule.match.includes("atom_offset"));
assert.equal(serialized.includes("(?<=,)"), false);
assert.equal(offsetRule.captures[1].name, "support.function.duffle.annotation");
assert.equal(offsetRule.captures[2].name, "entity.name.label.duffle.atom");
assert.equal(offsetRule.captures[3].name, "entity.name.label.duffle.atom");
});
test("workspace enables semantic highlighting and colors every custom token", () => {
const settings = readJson(path.resolve(ROOT, "..", "settings.json"));
const semantic = settings["editor.semanticTokenColorCustomizations"];
assert.equal(settings["editor.semanticHighlighting.enabled"], true);
assert.equal(semantic.enabled, true);
for (const tokenType of TOKEN_TYPES) {
assert.equal(Object.hasOwn(semantic.rules, tokenType), true, `missing color: ${tokenType}`);
}
for (const rule of [
"tapeGprRegister.tapeRead",
"tapeGprRegister.tapeWrite",
"tapeCop2Register.tapeRead",
"tapeCop2Register.tapeWrite",
"*.tapeAuto",
]) {
assert.equal(Object.hasOwn(semantic.rules, rule), true, `missing modifier color: ${rule}`);
}
});
+77
View File
@@ -0,0 +1,77 @@
"use strict";
const assert = require("node:assert/strict");
const test = require("node:test");
const { buildCallContexts, lex, nearestCall } = require("../lexer");
test("lex skips comments, strings, and character literals", () => {
const source = [
"MipsAtom_(visible)",
"// MipsAtom_(line_comment)",
"const char *s = \"atom_reads(R_Hidden)\";",
"char c = '\\''; /* gte_cmdw_hidden */",
"atom_reads(R_Visible)",
].join("\n");
const result = lex(source);
const identifiers = result.tokens
.filter((token) => token.kind === "identifier")
.map((token) => token.text);
assert.deepEqual(result.errors, []);
assert.equal(identifiers.includes("visible"), true);
assert.equal(identifiers.includes("R_Visible"), true);
assert.equal(identifiers.includes("line_comment"), false);
assert.equal(identifiers.includes("R_Hidden"), false);
assert.equal(identifiers.includes("gte_cmdw_hidden"), false);
});
test("lex reports unterminated block comments without returning comment tokens", () => {
const result = lex("R_Visible /* atom_reads(R_Hidden)");
assert.equal(result.tokens.some((token) => token.text === "R_Visible"), true);
assert.equal(result.tokens.some((token) => token.text === "R_Hidden"), false);
assert.deepEqual(result.errors.map((error) => error.kind), ["unterminated-block-comment"]);
});
test("line comments stop at CRLF boundaries", () => {
const result = lex("// atom_reads(R_Hidden)\r\natom_reads(R_Visible)\r\n");
const identifiers = result.tokens
.filter((token) => token.kind === "identifier")
.map((token) => token.text);
assert.equal(identifiers.includes("R_Hidden"), false);
assert.equal(identifiers.includes("R_Visible"), true);
});
test("balanced contexts retain multiline nesting and argument indexes", () => {
const source = [
"atom_info(",
"\tatom_phase(cube_g4),",
"\tatom_reads(R_A, nested(R_B, R_C)),",
"\tatom_writes(R_D)",
")",
].join("\n");
const lexical = lex(source);
const balanced = buildCallContexts(lexical.tokens);
const byText = new Map();
lexical.tokens.forEach((token, index) => {
if (token.kind === "identifier") byText.set(token.text, index);
});
assert.equal(nearestCall(balanced.contexts, byText.get("cube_g4")).callee, "atom_phase");
assert.equal(nearestCall(balanced.contexts, byText.get("R_A")).callee, "atom_reads");
assert.equal(nearestCall(balanced.contexts, byText.get("R_A")).argIndex, 0);
assert.equal(nearestCall(balanced.contexts, byText.get("R_C")).callee, "nested");
assert.equal(nearestCall(balanced.contexts, byText.get("R_D")).callee, "atom_writes");
assert.deepEqual(balanced.errors, []);
});
test("balanced contexts report unmatched parentheses", () => {
const lexical = lex("atom_reads(R_A");
const balanced = buildCallContexts(lexical.tokens);
assert.deepEqual(balanced.errors.map((error) => error.kind), ["unmatched-open-paren"]);
});
+77
View File
@@ -0,0 +1,77 @@
"use strict";
const assert = require("node:assert/strict");
const test = require("node:test");
const {
createIndex,
domainFromPath,
mergeIndexes,
scanSource,
} = require("../source-index");
test("scanSource discovers current atom and component forms", () => {
const source = [
"MipsAtom_(cube_g4_face) atom_info(atom_phase(cube_g4), atom_reads(R_PrimCursor)) { mac_yield() };",
"MipsAtomComp_(ac_load_pair) { load_word(R_T0, R_T1, 0) };",
"internal MipsAtom* normalize_proc(AtomArena_R aa) MipsAtom_Proc_(aa, { mac_yield() })",
"FI_ void ac_store_pair(MipsAtomBuilder_R ab) atom_dbg_skip MipsAtomComp_Proc_(ab, { store_word(R_T0, R_T1, 0) })",
].join("\n");
const result = scanSource(source, "C:/projects/Pikuma/ps1/code/duffle/mips.atom.c");
assert.equal(result.index.atoms.has("cube_g4_face"), true);
assert.equal(result.index.atoms.has("normalize"), true);
assert.equal(result.index.components.has("ac_load_pair"), true);
assert.equal(result.index.components.has("ac_store_pair"), true);
assert.equal(result.index.componentAliases.has("mac_load_pair"), true);
assert.equal(result.index.componentAliases.has("mac_store_pair"), true);
assert.equal(result.index.macros.get("mac_store_pair"), "cpu");
assert.equal(result.index.phases.has("cube_g4"), true);
assert.equal(result.index.registers.get("R_PrimCursor"), "gpr");
assert.deepEqual(result.errors, []);
});
test("scanSource discovers binds, labels, registers, typedefs, and macro domains", () => {
const source = [
"typedef Struct_(Binds_CubeTri) { U4 PrimCursor; };",
"typedef Enum_(U4, PadStatus) { PadStatus_Ok };",
"typedef U4 const MipsCode;",
"enum { R_PrimCursor = R_T7 atom_reg, C2_Custom = 12, gte_cr_Custom = 13 };",
"#define load_word(rt, base, off) enc_i(rt, base, off)",
"atom_bind(Binds_CubeTri)",
"atom_label(exit)",
"atom_offset(entry, exit)",
].join("\n");
const result = scanSource(source, "C:/projects/Pikuma/ps1/code/duffle/mips.h");
assert.equal(result.index.bindTypes.has("Binds_CubeTri"), true);
assert.equal(result.index.types.has("PadStatus"), true);
assert.equal(result.index.types.has("MipsCode"), true);
assert.equal(result.index.registers.get("R_PrimCursor"), "gpr");
assert.equal(result.index.registers.get("C2_Custom"), "cop2");
assert.equal(result.index.registers.get("gte_cr_Custom"), "cop2");
assert.equal(result.index.macros.get("load_word"), "cpu");
assert.equal(result.index.labels.has("entry"), true);
assert.equal(result.index.labels.has("exit"), true);
});
test("domainFromPath uses the declaration file rather than parent directory names", () => {
assert.equal(domainFromPath("C:/x/code/hello_gte/hello_gte.atom.c"), "component");
assert.equal(domainFromPath("C:/x/code/duffle/mips.h"), "cpu");
assert.equal(domainFromPath("C:/x/code/duffle/gte.atom.c"), "gte");
assert.equal(domainFromPath("C:/x/code/duffle/gp.h"), "gpu");
});
test("mergeIndexes preserves domain-specific aliases", () => {
const left = createIndex();
left.macros.set("load_word", "cpu");
const right = createIndex();
right.componentAliases.add("mac_gte_store");
right.macros.set("mac_gte_store", "gte");
const merged = mergeIndexes(left, right);
assert.equal(merged.macros.get("load_word"), "cpu");
assert.equal(merged.macros.get("mac_gte_store"), "gte");
});