initial plugin setup for syntax highlighting in vscode...

.vscode/tape-atom-syntax/test/classifier.test.js

@@ -0,0 +1,83 @@
+"use strict";
+
+const assert = require("node:assert/strict");
+const test = require("node:test");
+
+const { classifyDocument } = require("../classifier");
+const { createIndex } = require("../source-index");
+
+function byText(result, text) {
+	return result.spans.filter((span) => span.text === text);
+}
+
+test("classifyDocument distinguishes declaration, annotation, phase, bind, and label roles", () => {
+	const source = [
+		"typedef Struct_(Binds_CubeTri) { U4 PrimCursor; };",
+		"MipsAtom_(cube_g4_face) atom_info(atom_bind(Binds_CubeTri), atom_phase(cube_g4),",
+		"\tatom_reads(R_PrimCursor), atom_writes(R_FaceCursor)) {",
+		"\tbranch_le_zero(R_T0, atom_offset(cull, exit)),",
+		"\tatom_label(exit)",
+		"};",
+	].join("\n");
+
+	const result = classifyDocument(source, "C:/x/code/hello_camera/hello_camera.atom.c", createIndex());
+
+	assert.equal(byText(result, "MipsAtom_")[0].type, "tapeAtomKeyword");
+	assert.deepEqual(byText(result, "cube_g4_face")[0].modifiers, ["declaration"]);
+	assert.equal(byText(result, "atom_bind")[0].type, "tapeAnnotation");
+	assert.equal(byText(result, "Binds_CubeTri").at(-1).type, "tapeBindType");
+	assert.equal(byText(result, "cube_g4")[0].type, "tapePhase");
+	assert.deepEqual(byText(result, "cube_g4")[0].modifiers, ["declaration"]);
+	assert.equal(byText(result, "cull")[0].type, "tapeLabel");
+	assert.equal(byText(result, "exit").every((span) => span.type === "tapeLabel"), true);
+});
+
+test("classifyDocument applies read and write modifiers to GPRs", () => {
+	const source = "atom_info(atom_reads(R_PrimCursor), atom_writes(R_FaceCursor))";
+	const result = classifyDocument(source, "C:/x/code/test.atom.c", createIndex());
+
+	assert.deepEqual(byText(result, "R_PrimCursor")[0].modifiers, ["tapeRead"]);
+	assert.deepEqual(byText(result, "R_FaceCursor")[0].modifiers, ["tapeWrite"]);
+});
+
+test("classifyDocument separates CPU, GTE, GPU, and component domains", () => {
+	const workspace = createIndex();
+	workspace.macros.set("load_word", "cpu");
+	workspace.macros.set("gte_cmdw_rtpt", "gte");
+	workspace.macros.set("gp1_word_DisplayOn", "gpu");
+	workspace.macros.set("mac_yield", "component");
+	workspace.componentAliases.add("mac_yield");
+
+	const source = "load_word(R_T0, R_T1, 0), gte_cmdw_rtpt, gp1_word_DisplayOn(), mac_yield(), C2_MAC0, gte_cr_OFX_Code";
+	const result = classifyDocument(source, "C:/x/code/test.c", workspace);
+
+	assert.equal(byText(result, "load_word")[0].type, "tapeCpuInstruction");
+	assert.equal(byText(result, "gte_cmdw_rtpt")[0].type, "tapeGteInstruction");
+	assert.equal(byText(result, "gp1_word_DisplayOn")[0].type, "tapeGpuInstruction");
+	assert.equal(byText(result, "mac_yield")[0].type, "tapeComponentInstruction");
+	assert.equal(byText(result, "C2_MAC0")[0].type, "tapeCop2Register");
+	assert.equal(byText(result, "gte_cr_OFX_Code")[0].type, "tapeCop2Register");
+});
+
+test("document-local declarations override an empty workspace index", () => {
+	const source = [
+		"MipsAtomComp_(ac_new_component) { nop };",
+		"mac_new_component(),",
+	].join("\n");
+	const result = classifyDocument(source, "C:/x/code/duffle/math.atom.c", createIndex());
+
+	assert.equal(byText(result, "ac_new_component")[0].type, "tapeComponentName");
+	assert.equal(byText(result, "mac_new_component")[0].type, "tapeComponentInstruction");
+});
+
+test("classifier returns ordered non-overlapping spans and partial malformed output", () => {
+	const source = "atom_reads(R_A /* broken";
+	const result = classifyDocument(source, "C:/x/code/test.atom.c", createIndex());
+
+	assert.equal(result.errors.some((error) => error.kind === "unterminated-block-comment"), true);
+	for (let spanIndex = 1; spanIndex < result.spans.length; spanIndex += 1) {
+		const previous = result.spans[spanIndex - 1];
+		const current = result.spans[spanIndex];
+		assert.equal(previous.start + previous.length <= current.start, true);
+	}
+});

.vscode/tape-atom-syntax/test/contract.test.js

@@ -0,0 +1,93 @@
+"use strict";
+
+const assert = require("node:assert/strict");
+const fs = require("node:fs");
+const path = require("node:path");
+const test = require("node:test");
+
+const { TOKEN_MODIFIERS, TOKEN_TYPES } = require("../classifier");
+
+const ROOT = path.resolve(__dirname, "..");
+
+function readJson(filePath) {
+	return JSON.parse(fs.readFileSync(filePath, "utf8"));
+}
+
+function collectScopeNames(value, output = new Set()) {
+	if (Array.isArray(value)) {
+		for (const entry of value) collectScopeNames(entry, output);
+		return output;
+	}
+	if (!value || typeof value !== "object") return output;
+	if (typeof value.name === "string") output.add(value.name);
+	for (const child of Object.values(value)) collectScopeNames(child, output);
+	return output;
+}
+
+test("package semantic legend matches classifier exports", () => {
+	const packageJson = readJson(path.join(ROOT, "package.json"));
+	const contributedTypes = packageJson.contributes.semanticTokenTypes.map((entry) => entry.id);
+	const contributedModifiers = packageJson.contributes.semanticTokenModifiers.map((entry) => entry.id);
+
+	assert.equal(packageJson.version, "0.2.0");
+	assert.deepEqual(contributedTypes, TOKEN_TYPES);
+	assert.deepEqual(contributedModifiers, TOKEN_MODIFIERS.filter((name) => name !== "declaration"));
+});
+
+test("package includes runtime files only and acknowledges local-only metadata", () => {
+	const packageJson = readJson(path.join(ROOT, "package.json"));
+
+	assert.deepEqual(packageJson.files, [
+		"classifier.js",
+		"extension.js",
+		"lexer.js",
+		"source-index.js",
+		"syntaxes/tape_atom.tmLanguage.json",
+	]);
+	assert.equal(packageJson.scripts.package.includes("--allow-missing-repository"), true);
+	assert.equal(packageJson.scripts.package.includes("--skip-license"), true);
+});
+
+test("every semantic token has a TextMate fallback scope and grammar scope", () => {
+	const packageJson = readJson(path.join(ROOT, "package.json"));
+	const grammar = readJson(path.join(ROOT, "syntaxes", "tape_atom.tmLanguage.json"));
+	const mappings = packageJson.contributes.semanticTokenScopes[0].scopes;
+	const grammarScopes = collectScopeNames(grammar);
+
+	for (const tokenType of TOKEN_TYPES) {
+		assert.equal(Array.isArray(mappings[tokenType]), true, `missing scope mapping: ${tokenType}`);
+		assert.equal(mappings[tokenType].some((scope) => grammarScopes.has(scope)), true, `grammar does not emit: ${tokenType}`);
+	}
+});
+
+test("TextMate offset labels stay scoped to atom_offset calls", () => {
+	const grammar = readJson(path.join(ROOT, "syntaxes", "tape_atom.tmLanguage.json"));
+	const serialized = JSON.stringify(grammar);
+	const offsetRule = grammar.repository["annotation-arguments"].patterns
+		.find((rule) => rule.match.includes("atom_offset"));
+
+	assert.equal(serialized.includes("(?<=,)"), false);
+	assert.equal(offsetRule.captures[1].name, "support.function.duffle.annotation");
+	assert.equal(offsetRule.captures[2].name, "entity.name.label.duffle.atom");
+	assert.equal(offsetRule.captures[3].name, "entity.name.label.duffle.atom");
+});
+
+test("workspace enables semantic highlighting and colors every custom token", () => {
+	const settings = readJson(path.resolve(ROOT, "..", "settings.json"));
+	const semantic = settings["editor.semanticTokenColorCustomizations"];
+
+	assert.equal(settings["editor.semanticHighlighting.enabled"], true);
+	assert.equal(semantic.enabled, true);
+	for (const tokenType of TOKEN_TYPES) {
+		assert.equal(Object.hasOwn(semantic.rules, tokenType), true, `missing color: ${tokenType}`);
+	}
+	for (const rule of [
+		"tapeGprRegister.tapeRead",
+		"tapeGprRegister.tapeWrite",
+		"tapeCop2Register.tapeRead",
+		"tapeCop2Register.tapeWrite",
+		"*.tapeAuto",
+	]) {
+		assert.equal(Object.hasOwn(semantic.rules, rule), true, `missing modifier color: ${rule}`);
+	}
+});

.vscode/tape-atom-syntax/test/lexer.test.js

@@ -0,0 +1,77 @@
+"use strict";
+
+const assert = require("node:assert/strict");
+const test = require("node:test");
+
+const { buildCallContexts, lex, nearestCall } = require("../lexer");
+
+test("lex skips comments, strings, and character literals", () => {
+	const source = [
+		"MipsAtom_(visible)",
+		"// MipsAtom_(line_comment)",
+		"const char *s = \"atom_reads(R_Hidden)\";",
+		"char c = '\\''; /* gte_cmdw_hidden */",
+		"atom_reads(R_Visible)",
+	].join("\n");
+
+	const result = lex(source);
+	const identifiers = result.tokens
+		.filter((token) => token.kind === "identifier")
+		.map((token) => token.text);
+
+	assert.deepEqual(result.errors, []);
+	assert.equal(identifiers.includes("visible"), true);
+	assert.equal(identifiers.includes("R_Visible"), true);
+	assert.equal(identifiers.includes("line_comment"), false);
+	assert.equal(identifiers.includes("R_Hidden"), false);
+	assert.equal(identifiers.includes("gte_cmdw_hidden"), false);
+});
+
+test("lex reports unterminated block comments without returning comment tokens", () => {
+	const result = lex("R_Visible /* atom_reads(R_Hidden)");
+
+	assert.equal(result.tokens.some((token) => token.text === "R_Visible"), true);
+	assert.equal(result.tokens.some((token) => token.text === "R_Hidden"), false);
+	assert.deepEqual(result.errors.map((error) => error.kind), ["unterminated-block-comment"]);
+});
+
+test("line comments stop at CRLF boundaries", () => {
+	const result = lex("// atom_reads(R_Hidden)\r\natom_reads(R_Visible)\r\n");
+	const identifiers = result.tokens
+		.filter((token) => token.kind === "identifier")
+		.map((token) => token.text);
+
+	assert.equal(identifiers.includes("R_Hidden"), false);
+	assert.equal(identifiers.includes("R_Visible"), true);
+});
+
+test("balanced contexts retain multiline nesting and argument indexes", () => {
+	const source = [
+		"atom_info(",
+		"\tatom_phase(cube_g4),",
+		"\tatom_reads(R_A, nested(R_B, R_C)),",
+		"\tatom_writes(R_D)",
+		")",
+	].join("\n");
+	const lexical = lex(source);
+	const balanced = buildCallContexts(lexical.tokens);
+
+	const byText = new Map();
+	lexical.tokens.forEach((token, index) => {
+		if (token.kind === "identifier") byText.set(token.text, index);
+	});
+
+	assert.equal(nearestCall(balanced.contexts, byText.get("cube_g4")).callee, "atom_phase");
+	assert.equal(nearestCall(balanced.contexts, byText.get("R_A")).callee, "atom_reads");
+	assert.equal(nearestCall(balanced.contexts, byText.get("R_A")).argIndex, 0);
+	assert.equal(nearestCall(balanced.contexts, byText.get("R_C")).callee, "nested");
+	assert.equal(nearestCall(balanced.contexts, byText.get("R_D")).callee, "atom_writes");
+	assert.deepEqual(balanced.errors, []);
+});
+
+test("balanced contexts report unmatched parentheses", () => {
+	const lexical = lex("atom_reads(R_A");
+	const balanced = buildCallContexts(lexical.tokens);
+
+	assert.deepEqual(balanced.errors.map((error) => error.kind), ["unmatched-open-paren"]);
+});

.vscode/tape-atom-syntax/test/source-index.test.js

@@ -0,0 +1,77 @@
+"use strict";
+
+const assert = require("node:assert/strict");
+const test = require("node:test");
+
+const {
+	createIndex,
+	domainFromPath,
+	mergeIndexes,
+	scanSource,
+} = require("../source-index");
+
+test("scanSource discovers current atom and component forms", () => {
+	const source = [
+		"MipsAtom_(cube_g4_face) atom_info(atom_phase(cube_g4), atom_reads(R_PrimCursor)) { mac_yield() };",
+		"MipsAtomComp_(ac_load_pair) { load_word(R_T0, R_T1, 0) };",
+		"internal MipsAtom* normalize_proc(AtomArena_R aa) MipsAtom_Proc_(aa, { mac_yield() })",
+		"FI_ void ac_store_pair(MipsAtomBuilder_R ab) atom_dbg_skip MipsAtomComp_Proc_(ab, { store_word(R_T0, R_T1, 0) })",
+	].join("\n");
+
+	const result = scanSource(source, "C:/projects/Pikuma/ps1/code/duffle/mips.atom.c");
+
+	assert.equal(result.index.atoms.has("cube_g4_face"), true);
+	assert.equal(result.index.atoms.has("normalize"), true);
+	assert.equal(result.index.components.has("ac_load_pair"), true);
+	assert.equal(result.index.components.has("ac_store_pair"), true);
+	assert.equal(result.index.componentAliases.has("mac_load_pair"), true);
+	assert.equal(result.index.componentAliases.has("mac_store_pair"), true);
+	assert.equal(result.index.macros.get("mac_store_pair"), "cpu");
+	assert.equal(result.index.phases.has("cube_g4"), true);
+	assert.equal(result.index.registers.get("R_PrimCursor"), "gpr");
+	assert.deepEqual(result.errors, []);
+});
+
+test("scanSource discovers binds, labels, registers, typedefs, and macro domains", () => {
+	const source = [
+		"typedef Struct_(Binds_CubeTri) { U4 PrimCursor; };",
+		"typedef Enum_(U4, PadStatus) { PadStatus_Ok };",
+		"typedef U4 const MipsCode;",
+		"enum { R_PrimCursor = R_T7 atom_reg, C2_Custom = 12, gte_cr_Custom = 13 };",
+		"#define load_word(rt, base, off) enc_i(rt, base, off)",
+		"atom_bind(Binds_CubeTri)",
+		"atom_label(exit)",
+		"atom_offset(entry, exit)",
+	].join("\n");
+
+	const result = scanSource(source, "C:/projects/Pikuma/ps1/code/duffle/mips.h");
+
+	assert.equal(result.index.bindTypes.has("Binds_CubeTri"), true);
+	assert.equal(result.index.types.has("PadStatus"), true);
+	assert.equal(result.index.types.has("MipsCode"), true);
+	assert.equal(result.index.registers.get("R_PrimCursor"), "gpr");
+	assert.equal(result.index.registers.get("C2_Custom"), "cop2");
+	assert.equal(result.index.registers.get("gte_cr_Custom"), "cop2");
+	assert.equal(result.index.macros.get("load_word"), "cpu");
+	assert.equal(result.index.labels.has("entry"), true);
+	assert.equal(result.index.labels.has("exit"), true);
+});
+
+test("domainFromPath uses the declaration file rather than parent directory names", () => {
+	assert.equal(domainFromPath("C:/x/code/hello_gte/hello_gte.atom.c"), "component");
+	assert.equal(domainFromPath("C:/x/code/duffle/mips.h"), "cpu");
+	assert.equal(domainFromPath("C:/x/code/duffle/gte.atom.c"), "gte");
+	assert.equal(domainFromPath("C:/x/code/duffle/gp.h"), "gpu");
+});
+
+test("mergeIndexes preserves domain-specific aliases", () => {
+	const left = createIndex();
+	left.macros.set("load_word", "cpu");
+	const right = createIndex();
+	right.componentAliases.add("mac_gte_store");
+	right.macros.set("mac_gte_store", "gte");
+
+	const merged = mergeIndexes(left, right);
+	assert.equal(merged.macros.get("load_word"), "cpu");
+	assert.equal(merged.macros.get("mac_gte_store"), "gte");
+});