diff --git a/editors/vscode/README.md b/editors/vscode/README.md index 15f2d13..0e7c876 100644 --- a/editors/vscode/README.md +++ b/editors/vscode/README.md @@ -12,7 +12,9 @@ compiles to native code via LLVM. Files use the `.ql` extension. - Operators: `|>` (pipe), `:=` (mutable bind) vs `=` (immutable bind), `::` (type annotation), `=>` (function body / match arm), `->` (return type), `<-` (loop iterate), `?` / `|` (pattern matching), arithmetic `+ - * / %`, comparison `== != < <= > >=`, - logical `&& || !`. + logical `&& || !`. Each **multi-character** operator (`=>`, `->`, `:=`, `|>`, + `<-`, `::`, `==`, `!=`, `<=`, `>=`, `&&`, `||`) is highlighted as a **single** + token — never split into its first character colored separately from the rest. - Built-in types `Num` / `Text` / `Bool`, and the unit type/value `$` (`$` is both the type, as in `-> $`, and its sole value — highlighted like the other built-in types). @@ -77,9 +79,21 @@ touches `editors/vscode/**` (see [Publishing](#publishing)). ### Tests & manual verification -Unit tests cover the diagnostic-output parser (`src/diagnostics.ts`), which is -kept free of any `vscode` import so it runs under plain Node. To verify the -**inline diagnostics** end-to-end manually: +Unit tests (`pnpm test`) cover three things, all kept free of any `vscode` +import so they run under plain Node: + +- the diagnostic-output parser (`src/diagnostics.ts` ↔ `src/diagnostics.test.ts`); +- the entry-point detector behind the CodeLens (`src/entryPoints.ts` ↔ + `src/entryPoints.test.ts`); +- **grammar tokenization** (`src/grammar.test.ts`) — it loads the real + `syntaxes/quilon.tmLanguage.json` and asserts each multi-character operator + (`=>`, `->`, `:=`, `|>`, `<-`, `::`, `==`, `!=`, `<=`, `>=`, `&&`, `||`) + tokenizes to a **single** scope, plus regression guards for `<` / `>`, `=`, + `$`, comments, strings, and numbers. `src/grammar.ts` is a tiny dependency-free + re-implementation of TextMate's ordered first-match-wins rule (the behaviour + the operator ordering relies on), so no native engine is needed. + +To verify the **inline diagnostics** end-to-end manually: 1. Set `quilon.command` to a working compiler (e.g. `"cargo run --"` from a checkout, or `"quilon"` if it's on your `PATH`). diff --git a/editors/vscode/src/grammar.test.ts b/editors/vscode/src/grammar.test.ts new file mode 100644 index 0000000..64e2716 --- /dev/null +++ b/editors/vscode/src/grammar.test.ts @@ -0,0 +1,174 @@ +// Tokenization tests for the Quilon TextMate grammar +// (`syntaxes/quilon.tmLanguage.json`). +// +// These guard the bug where a multi-character operator (`=>`, `->`, …) was at +// risk of being highlighted as TWO tokens (its first char in one scope, the +// rest in another). TextMate applies, at each position, the FIRST pattern in +// the list that matches — ties at the same start are decided by list order, not +// length — so the fix is purely about ordering the multi-char operator rules +// before the single-char ones. We assert each operator yields exactly one token +// with one scope. +// +// We tokenize with a small faithful re-implementation of that algorithm +// (`./grammar`) rather than the native `vscode-textmate` engine, to keep the +// tests dependency-free and runnable under plain `node --test`. + +import assert from "node:assert/strict"; +import { join } from "node:path"; +import { test } from "node:test"; +import { Grammar, type Token } from "./grammar"; + +const grammar = Grammar.fromFile(join(__dirname, "..", "syntaxes", "quilon.tmLanguage.json")); + +/** All scoped (non-plain) tokens of a line, in order. */ +function scopedTokens(line: string): Token[] { + return grammar.tokenizeLine(line).filter((t) => t.scope !== undefined); +} + +/** Find the single token whose text is exactly `op`; fail if 0 or >1. */ +function uniqueToken(line: string, op: string): Token { + const matches = grammar.tokenizeLine(line).filter((t) => t.text === op); + assert.equal( + matches.length, + 1, + `expected exactly one token with text ${JSON.stringify(op)} in ${JSON.stringify(line)}, ` + + `got ${JSON.stringify(grammar.tokenizeLine(line))}`, + ); + return matches[0]; +} + +// Every multi-character operator and the single scope it must carry. Each is +// exercised in both a spaced (`a OP b`) and a tight (`aOPb`) line, generated +// from the operator — the tight form is where a naive grammar is most likely to +// split the operator into its first character + the rest, and in `a OP b` the +// operator's first character has no legitimate standalone twin, so a standalone +// first-char token would be the split-operator regression. +const MULTI_CHAR_OPERATORS: ReadonlyArray = [ + ["=>", "keyword.operator.arrow.body.quilon"], + ["->", "keyword.operator.arrow.return.quilon"], + [":=", "keyword.operator.assignment.mutable.quilon"], + ["|>", "keyword.operator.pipeline.quilon"], + ["<-", "keyword.operator.arrow.iterate.quilon"], + ["::", "keyword.operator.type-annotation.quilon"], + ["==", "keyword.operator.comparison.quilon"], + ["!=", "keyword.operator.comparison.quilon"], + ["<=", "keyword.operator.comparison.quilon"], + [">=", "keyword.operator.comparison.quilon"], + ["&&", "keyword.operator.logical.quilon"], + ["||", "keyword.operator.logical.quilon"], +]; + +for (const [op, scope] of MULTI_CHAR_OPERATORS) { + const spaced = `a ${op} b`; + const tight = `a${op}b`; + + test(`multi-char operator ${op} is one token with scope ${scope}`, () => { + for (const line of [spaced, tight]) { + const token = uniqueToken(line, op); + assert.equal(token.text, op); + assert.equal(token.scope, scope); + } + }); + + test(`multi-char operator ${op} is not split into its first character`, () => { + // The regression: the first char would appear as its own scoped token. + const firstChar = op[0]; + const standalone = grammar + .tokenizeLine(spaced) + .filter((t) => t.scope !== undefined && t.text === firstChar); + assert.equal( + standalone.length, + 0, + `${op} leaked a standalone ${JSON.stringify(firstChar)} token: ` + + JSON.stringify(grammar.tokenizeLine(spaced)), + ); + }); +} + +// `<<` / `>>` are module markers (import / export), handled before the operator +// rules — assert they are still single tokens too. +test("<< import marker is a single token", () => { + const token = uniqueToken("<< core.io", "<<"); + assert.equal(token.scope, "keyword.control.import.quilon"); +}); + +test(">> export marker is a single token", () => { + const token = uniqueToken(">> add = (a, b) => a + b", ">>"); + assert.equal(token.scope, "keyword.control.export.quilon"); +}); + +test("adjacent operators on one line each stay a single token", () => { + // `a==b!=c<=d>=e` — four two-char comparisons back to back, no spaces. + const tokens = scopedTokens("a==b!=c<=d>=e"); + const ops = tokens.filter((t) => t.scope === "keyword.operator.comparison.quilon"); + assert.deepEqual( + ops.map((t) => t.text), + ["==", "!=", "<=", ">="], + ); +}); + +test("a representative lambda + arrow-type line highlights each operator once", () => { + // `double = (x :: Num) -> Num => x * 2` + const line = "double = (x :: Num) -> Num => x * 2"; + const tokens = scopedTokens(line); + const find = (text: string) => tokens.filter((t) => t.text === text); + + assert.equal(find("::").length, 1, ":: should be one token"); + assert.equal(find("->").length, 1, "-> should be one token"); + assert.equal(find("=>").length, 1, "=> should be one token"); + assert.equal(find("=").length, 1, "the single = should be one token"); + // No stray single-char fragments from the multi-char operators. + assert.equal(find(">").length, 0, "no bare > fragment"); + assert.equal(find(":").length, 0, "no bare : fragment"); +}); + +// --- Regression guards for things the fix must NOT disturb ------------------- + +test("single < and > stay comparison operators (and block delimiters)", () => { + const lt = uniqueToken("a < b", "<"); + assert.equal(lt.scope, "keyword.operator.comparison.quilon"); + const gt = uniqueToken("a > b", ">"); + assert.equal(gt.scope, "keyword.operator.comparison.quilon"); +}); + +test("single = stays an immutable-binding operator", () => { + const token = uniqueToken("x = 42", "="); + assert.equal(token.scope, "keyword.operator.assignment.quilon"); +}); + +test("$ (unit) keeps its builtin-type scope in both type and value position", () => { + // `f = () -> $ => $`: the `$` return type and the `$` value are both scoped, + // and the surrounding `->` / `=>` are each still a single operator token. + const tokens = grammar.tokenizeLine("f = () -> $ => $"); + const units = tokens.filter((t) => t.text === "$"); + assert.equal(units.length, 2); + for (const u of units) { + assert.equal(u.scope, "support.type.builtin.unit.quilon"); + } + assert.equal(tokens.filter((t) => t.text === "->").length, 1); + assert.equal(tokens.filter((t) => t.text === "=>").length, 1); +}); + +test("~ comment swallows operators to end of line", () => { + const tokens = grammar.tokenizeLine("~ a note => not an operator"); + // The whole comment is one scoped token; no operator scope leaks out of it. + const operatorTokens = tokens.filter((t) => t.scope?.startsWith("keyword.operator")); + assert.equal(operatorTokens.length, 0); + assert.ok( + tokens.some((t) => t.scope === "comment.line.tilde.quilon"), + "expected a comment token", + ); +}); + +test("string contents are not tokenized as operators", () => { + const tokens = grammar.tokenizeLine('s = "a => b"'); + const operatorInString = tokens.filter( + (t) => t.text === "=>" && t.scope?.startsWith("keyword.operator"), + ); + assert.equal(operatorInString.length, 0, "=> inside a string must not be an operator"); +}); + +test("numbers still tokenize", () => { + const token = uniqueToken("x = 3.14", "3.14"); + assert.equal(token.scope, "constant.numeric.quilon"); +}); diff --git a/editors/vscode/src/grammar.ts b/editors/vscode/src/grammar.ts new file mode 100644 index 0000000..a2029cc --- /dev/null +++ b/editors/vscode/src/grammar.ts @@ -0,0 +1,302 @@ +// A small, faithful re-implementation of how a TextMate grammar tokenizes a +// single line, used to test `syntaxes/quilon.tmLanguage.json` without pulling in +// the (native) `vscode-textmate` + `vscode-oniguruma` engine as a dependency. +// +// It reproduces the one behaviour this grammar's correctness hinges on: at each +// position TextMate scans the *ordered* list of patterns and applies the FIRST +// one that matches there — ties at the same start position are decided by list +// order, NOT by match length. That is exactly why every multi-character operator +// (`=>`, `->`, `:=`, `|>`, `<-`, `==`, `!=`, `<=`, `>=`, `&&`, `||`, `::`) must +// be listed before the single-character operator rules: otherwise a rule for the +// first character would win and split the operator into two tokens. +// +// Supported subset (all this grammar uses): `match` rules, `begin`/`end` rules +// (single line — enough for `~` comments and `"…"` strings), `#include` +// references into `repository`, and a single top-level `name`. This module is +// deliberately free of any `vscode` import so it runs under plain Node +// (`node:test`), like `diagnostics.ts`. +// +// Fidelity caveat: the grammar's regexes are run as JavaScript regexes here, not +// Oniguruma (which the real engine uses). The grammar's patterns stay within the +// common subset, so keep new patterns there too — an Oniguruma-only construct +// (e.g. `\G`, possessive quantifiers) would pass these tests yet differ in the +// editor. + +import { readFileSync } from "node:fs"; + +/** A `name`+`match` (or `begin`/`end`) leaf rule, or an `include` reference. */ +interface RawRule { + readonly name?: string; + readonly match?: string; + readonly begin?: string; + readonly end?: string; + readonly include?: string; + readonly patterns?: readonly RawRule[]; + /** Per-capture-group scope names (keyed by group index as a string). */ + readonly captures?: Readonly>; +} + +interface RawGrammar { + readonly patterns: readonly RawRule[]; + readonly repository: Readonly>; +} + +/** One tokenized slice of a line: its text and the scope name applied to it. */ +export interface Token { + readonly text: string; + /** The grammar `name` scope, or `undefined` for unscoped (plain) text. */ + readonly scope: string | undefined; +} + +/** Capture-group index → scope name (for a `match` rule's `captures`). */ +type Captures = ReadonlyMap; + +/** A leaf rule that actually matches text (an `include` has been resolved away). */ +type Rule = + | { + readonly kind: "match"; + readonly name?: string; + readonly re: RegExp; + readonly captures: Captures; + } + | { + readonly kind: "beginEnd"; + readonly name?: string; + readonly begin: RegExp; + /** Global regex (compiled once) for locating where the span closes. */ + readonly end: RegExp; + }; + +export class Grammar { + private readonly rootRules: readonly Rule[]; + + private constructor(grammar: RawGrammar) { + this.rootRules = resolve(grammar.patterns, grammar.repository); + } + + /** Load and compile a tmLanguage JSON grammar from disk. */ + static fromFile(path: string): Grammar { + return new Grammar(JSON.parse(readFileSync(path, "utf8")) as RawGrammar); + } + + /** + * Tokenize a single line. Returns the slices in order; their concatenated + * `text` reproduces the input exactly. Plain (unmatched) runs get an + * `undefined` scope. + */ + tokenizeLine(line: string): Token[] { + const tokens: Token[] = []; + let pos = 0; + let plainStart = 0; + + const flushPlain = (upTo: number): void => { + if (upTo > plainStart) { + tokens.push({ text: line.slice(plainStart, upTo), scope: undefined }); + } + }; + + while (pos < line.length) { + const hit = firstMatch(this.rootRules, line, pos); + if (!hit) { + break; + } + flushPlain(hit.start); + + if (hit.rule.kind === "match") { + tokens.push(...matchTokens(hit.rule, line, hit.start)); + pos = hit.end; + } else { + // begin/end: consume from `begin` through the first `end` on this line + // (or to end-of-line if `end` is `$`/absent), as a single scoped token. + const innerEnd = findEnd(hit.rule, line, hit.end); + tokens.push({ text: line.slice(hit.start, innerEnd), scope: hit.rule.name }); + pos = innerEnd; + } + plainStart = pos; + } + + flushPlain(line.length); + return tokens; + } +} + +/** Flatten a pattern list into leaf rules, resolving `#include` against the repo. */ +function resolve( + patterns: readonly RawRule[], + repo: Readonly>, + seen: ReadonlySet = new Set(), +): Rule[] { + const out: Rule[] = []; + for (const p of patterns) { + if (p.include) { + const key = p.include.replace(/^#/, ""); + if (seen.has(key)) { + continue; // guard against include cycles + } + const target = repo[key]; + if (!target) { + continue; + } + // `compile` already dispatches on match / begin / bare-patterns, so the + // include target goes through the same path as an inline rule. + out.push(...compile(target, repo, new Set(seen).add(key))); + } else { + out.push(...compile(p, repo, seen)); + } + } + return out; +} + +/** Turn a single leaf rule into its compiled form(s). */ +function compile( + rule: RawRule, + repo: Readonly>, + seen: ReadonlySet, +): Rule[] { + if (typeof rule.match === "string") { + return [ + { kind: "match", name: rule.name, re: sticky(rule.match), captures: buildCaptures(rule) }, + ]; + } + if (typeof rule.begin === "string") { + // The whole begin…end span is emitted as one scoped token (enough for `~` + // comments and `"…"` strings), so any inner `patterns` are intentionally not + // sub-scoped here — they don't affect the operator-tokenization this tests. + return [ + { + kind: "beginEnd", + name: rule.name, + begin: sticky(rule.begin), + end: new RegExp(rule.end ?? "$", "g"), + }, + ]; + } + // A bare `{ patterns: [...] }` group (no match/begin): inline its children. + if (rule.patterns) { + return resolve(rule.patterns, repo, seen); + } + return []; +} + +/** + * Compile a TextMate regex as a JS *sticky* regex with capture indices: `y` + * anchors a match to `lastIndex` (so probing position-by-position finds the + * earliest start cleanly and never silently skips ahead), and `d` exposes each + * group's span so a `match` rule's `captures` can be applied as sub-tokens. + */ +function sticky(source: string): RegExp { + return new RegExp(source, "yd"); +} + +/** Read a rule's `captures` into an index→scope map. */ +function buildCaptures(rule: RawRule): Captures { + const map = new Map(); + if (rule.captures) { + for (const [index, value] of Object.entries(rule.captures)) { + map.set(Number(index), value.name); + } + } + return map; +} + +interface Hit { + readonly rule: Rule; + readonly start: number; + readonly end: number; +} + +/** + * Find the winning rule at-or-after `from`: the leftmost match across all rules, + * ties at the same start broken by list order (the first rule wins) — TextMate's + * exact rule. This list-order tiebreak is what makes operator ordering matter. + */ +function firstMatch(rules: readonly Rule[], line: string, from: number): Hit | undefined { + let best: Hit | undefined; + for (const rule of rules) { + const re = rule.kind === "match" ? rule.re : rule.begin; + const start = earliestMatchFrom(re, line, from); + if (!start) { + continue; + } + // Strictly-earlier start wins; an equal start keeps the earlier-listed rule. + if (!best || start.index < best.start) { + best = { rule, start: start.index, end: start.index + start.length }; + } + } + return best; +} + +/** Earliest match of a sticky regex at or after `from`, or undefined. */ +function earliestMatchFrom( + re: RegExp, + line: string, + from: number, +): { index: number; length: number } | undefined { + for (let at = from; at <= line.length; at++) { + re.lastIndex = at; + const m = re.exec(line); + if (m && m[0].length > 0) { + return { index: m.index, length: m[0].length }; + } + } + return undefined; +} + +/** + * Emit the token(s) for a `match` rule at `start`. With no `captures` it is a + * single token scoped to the rule `name`; with `captures` the whole match takes + * `name` and each capture group layers its scope onto its sub-span (a later/ + * inner group overrides an outer one for the characters it covers), matching how + * a theme colors a captured match. + */ +function matchTokens(rule: Extract, line: string, start: number): Token[] { + rule.re.lastIndex = start; + const m = rule.re.exec(line); + if (!m) { + return []; + } + const whole = m[0]; + if (rule.captures.size === 0) { + return [{ text: whole, scope: rule.name }]; + } + + // Per-character scope: start everyone at the rule name, then stamp each + // capture group's span. Iterating groups by ascending index means a + // higher-indexed (inner) capture overrides a lower one where they overlap. + const scopes: (string | undefined)[] = Array.from({ length: whole.length }, () => rule.name); + const indices = m.indices; + if (indices) { + for (let g = 1; g < indices.length; g++) { + const span = indices[g]; + const scope = rule.captures.get(g); + if (!span || scope === undefined) { + continue; + } + for (let i = span[0] - start; i < span[1] - start; i++) { + scopes[i] = scope; + } + } + } + + // Coalesce runs of identical scope into tokens. + const tokens: Token[] = []; + let runStart = 0; + for (let i = 1; i <= whole.length; i++) { + if (i === whole.length || scopes[i] !== scopes[runStart]) { + tokens.push({ text: whole.slice(runStart, i), scope: scopes[runStart] }); + runStart = i; + } + } + return tokens; +} + +/** For a begin/end rule, find where its `end` closes on this line. */ +function findEnd(rule: Extract, line: string, from: number): number { + rule.end.lastIndex = from; + const m = rule.end.exec(line); + if (!m) { + return line.length; + } + // `$` matches with zero width at end-of-line: the comment runs to the line end. + return m.index + (m[0].length > 0 ? m[0].length : line.length - m.index); +} diff --git a/editors/vscode/syntaxes/quilon.tmLanguage.json b/editors/vscode/syntaxes/quilon.tmLanguage.json index 288f157..ddcb9e7 100644 --- a/editors/vscode/syntaxes/quilon.tmLanguage.json +++ b/editors/vscode/syntaxes/quilon.tmLanguage.json @@ -104,6 +104,7 @@ } }, "operators": { + "comment": "TextMate applies, at each position, the FIRST pattern in this list that matches there — ties at the same start position are NOT broken by length. So every multi-character operator MUST be listed before the single-character rules, and before any single-char operator that is a prefix of it (e.g. `|>` and `||` before `|`, `<=` before `<`, `=>`/`==` before `=`). That ordering is what keeps a two-char operator from being split into its first character + the rest. Each multi-char operator is its own scope.", "patterns": [ { "name": "keyword.operator.pipeline.quilon", @@ -130,12 +131,18 @@ "match": "<-" }, { + "comment": "== != <= >= — listed before `=`, `!`, `<`, `>` so a two-char comparison is never split into its first char + the rest.", "name": "keyword.operator.comparison.quilon", "match": "==|!=|<=|>=" }, { + "comment": "&& || — the two-char logical operators, listed before the single-char `!` and `|` below.", "name": "keyword.operator.logical.quilon", - "match": "&&|\\|\\||!" + "match": "&&|\\|\\|" + }, + { + "name": "keyword.operator.logical.quilon", + "match": "!" }, { "name": "keyword.operator.match.quilon",