From 8b77a40ffcb487170becf1e4eb948538ba0cb287 Mon Sep 17 00:00:00 2001 From: Jie Date: Sun, 6 Sep 2026 19:40:11 +0800 Subject: [PATCH 1/9] Carry the Core sync rule into the Chinese doc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Merged in #22 in English only. The rule an integration is meant to follow — sync Core when you release, not when Core ships — reached half the readers, and the half it missed are the ones the Chinese doc exists for. --- docs/projects.zh-CN.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/projects.zh-CN.md b/docs/projects.zh-CN.md index 24fbfb0..7b7fc6e 100644 --- a/docs/projects.zh-CN.md +++ b/docs/projects.zh-CN.md @@ -59,6 +59,15 @@ renderer.languages -> { [language]: label } Core 的变更通过拷贝或打包 `dist/` 资产流向插件仓库。插件的变更不应要求 Core 变更版本，除非它改动了 Core 的 API 或资产。 +**集成在自己发布时同步 Core，而不是 Core 一发布就同步。** 捆绑的副本追不上实时的 +Core：每个产物都冻结在它构建时的那份快照上 —— `.vsix`、插件 zip，以及 GitHub 为 +tag 附带的源码包，一概如此 —— 所以一个在 Core 发布当下就重新同步的仓库，对齐的只是 +自己的源码，用户装得到的东西一点没变。对齐是"发布"这个动作完成的事。 + +因此 `tools/check-core-freshness.mjs` 日常只作提示、在打包关口才严格：两次发布之间 +落后会被报告出来，而陈旧的引擎打不出包（`--strict`，接在各集成的构建或打包脚本 +里）。唯一不该等下一个功能版的，是安全级别的 Core 发布 —— 为它单独切一次集成发布。 + ## 仓库拆分 | 仓库 | 交付形态 | 许可 | 状态 | From d86341b23a3ad1f6f1105a06131a61544771b6a8 Mon Sep 17 00:00:00 2001 From: Jie Date: Sun, 6 Sep 2026 19:40:48 +0800 Subject: [PATCH 2/9] Answer the same question in both repository tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Chinese table's last column was headed 状态 and said 已公开 for Core — a status, where the English column says where a user gets the thing today. The sentence directly underneath already explained the English column, so the Chinese page contradicted itself before it contradicted the translation. --- docs/projects.zh-CN.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/projects.zh-CN.md b/docs/projects.zh-CN.md index 7b7fc6e..8bf5897 100644 --- a/docs/projects.zh-CN.md +++ b/docs/projects.zh-CN.md @@ -70,9 +70,9 @@ tag 附带的源码包，一概如此 —— 所以一个在 Core 发布当下 ## 仓库拆分 -| 仓库 | 交付形态 | 许可 | 状态 | +| 仓库 | 交付形态 | 许可 | 今天从哪里拿到 | |---|---|---|---| -| `jsray` | npm `@jsray/core` | MIT | 已公开 | +| `jsray` | npm `@jsray/core` | MIT | npm —— `@jsray/core@0.0.2-beta.3` | | `jsray-wp` | WordPress.org 插件 | GPLv2 or later | GitHub Release 的 zip | | `jsray-terminal` | npm CLI | MIT | GitHub —— `npm i -g github:jsrayorg/jsray-terminal` | | `jsray-vscode` | VS Code Marketplace | MIT | GitHub Release 的 `.vsix` | From bb4a2e2b1477c3630cfb76f10598e8fd9f463a6c Mon Sep 17 00:00:00 2001 From: Jie Date: Sun, 6 Sep 2026 19:41:03 +0800 Subject: [PATCH 3/9] Name the npm account the package publishes from The English versioning doc says @jsray/core is published from the jsray npm account; the Chinese one dropped the account and kept only the package name. Anyone reading it to find out who can publish came away without the answer. --- docs/versioning.zh-CN.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/versioning.zh-CN.md b/docs/versioning.zh-CN.md index f62b2fb..6e9d22e 100644 --- a/docs/versioning.zh-CN.md +++ b/docs/versioning.zh-CN.md @@ -73,7 +73,8 @@ JSRay Core 是独立的 JavaScript 原生代码渲染内核。平台插件(包 ## npm 发布 -包以 [`@jsray/core`](https://www.npmjs.com/package/@jsray/core) 发布。无作用域的 +包以 [`@jsray/core`](https://www.npmjs.com/package/@jsray/core) 之名、从 `jsray` +这个 npm 账号发布。无作用域的 `jsray` 拿不到 —— npm 判定它与已有的 `js-ray` 过于相似而拒绝 —— 而 `@jsray` 作用域的好处是一次性把整个家族(`@jsray/wp`、`@jsray/vscode`、`@jsray/terminal`) 都占住。 From 641889192cc764ba35f873a5496d29ba6cdb13ae Mon Sep 17 00:00:00 2001 From: Jie Date: Sun, 6 Sep 2026 19:41:03 +0800 Subject: [PATCH 4/9] Correct what the roadmap claims about the two integrations Two things had gone stale in both languages at once, which is the failure a translation check cannot see. jsray-terminal is described as having a release carrying an installable build. It has no releases at all; the only way in is the GitHub install line, and a tag is already sitting there to cut one from. "Syncing them to 0.0.2-beta.2 is the next step" was written when Core was on beta.2 and now names neither the current release nor the rule: an integration syncs Core as part of its own release, so being behind between releases is the expected state, not a queued chore. --- docs/development.md | 9 +++++++-- docs/development.zh-CN.md | 11 +++++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/docs/development.md b/docs/development.md index f8e15d3..b8391e9 100644 --- a/docs/development.md +++ b/docs/development.md @@ -403,7 +403,10 @@ deliberately deferred). warning when a security-grade Core update ships — security fixes are never silently pinnable. - **jsray-vscode / jsray-terminal**: both public since 2026-09-03 and - 2026-09-05, each with a release carrying an installable build. They now + 2026-09-05. jsray-vscode has a release carrying an installable `.vsix`; + jsray-terminal has no release yet, so the only way in is + `npm i -g github:jsrayorg/jsray-terminal` — a tag exists to cut one from. + They now carry what `jsray-wp` gained on 2026-08-26/27, because all three drift the same way: `tools/sync-core-version.mjs` deriving the README Core badge instead of @@ -412,7 +415,9 @@ deliberately deferred). the right one is what let "Internal test build · no public beta yet" survive the whole public beta; and the plugin version ladder (`0.0.1-beta → 0.0.2-beta`, no counter) rather than Core's. Both bundle Core - `0.0.2-beta.1`; syncing them to `0.0.2-beta.2` is the next step. + `0.0.2-beta.1`, and stay there until each cuts its own release: an + integration syncs Core as part of releasing, not when Core ships. The rule + and its one exception are in `docs/projects.md`. - **Core**: minification is deliberately absent (zero-build); revisit at public beta. diff --git a/docs/development.zh-CN.md b/docs/development.zh-CN.md index 9a27795..0003cc5 100644 --- a/docs/development.zh-CN.md +++ b/docs/development.zh-CN.md @@ -252,14 +252,17 @@ CI:每个仓库都有 GitHub Actions(Node 18/20/22 矩阵;Core 另验 `dist/` 管线)跑 32 条断言。 - **终端**:`--bg`(整底色绘制)、分页器集成与 `--core ` 覆盖在路线图上。 - **用户侧 Core 锁定**(路线图):平台原生的更新控制已允许用户拒绝某班列车(WP 手动更新、VS Code 按扩展关自动更新 + 装历史版本、npm 版本锁定)。插件内的"Core 更新策略"(跟随/锁定/自定义文件)对 WP 与终端可行,对 VS Code 预览不可行(贡献点路径静态)。硬规则:锁定状态下遇到安全级 Core 更新必须显式警告——安全修复不允许被静默锁死。 -- **jsray-vscode / jsray-terminal**:分别于 2026-09-03 和 2026-09-05 公开,各自 - 的 Release 里都挂着可直接安装的构建。两者现已补齐 `jsray-wp` 在 2026-08-26/27 +- **jsray-vscode / jsray-terminal**:分别于 2026-09-03 和 2026-09-05 公开。 + jsray-vscode 的 Release 里挂着可直接安装的 `.vsix`;jsray-terminal 还没有任何 + Release,唯一的安装途径是 `npm i -g github:jsrayorg/jsray-terminal` —— tag 已 + 经在,随时可以据此切一个。两者现已补齐 `jsray-wp` 在 2026-08-26/27 得到的那套机制 —— 三个集成的失效方式是同一种:`tools/sync-core-version.mjs` 自己推导 README 的 Core 徽章,而不是留给"跑同步的那个人";`check:versions` **双向**校验徽章与阶段 措辞 —— 正是"该出现的词在、不该出现的词也在"这一点,让「内部测试版 · 尚未发布 公开测试版」在整个公开 beta 期间活了下来;以及插件的版本阶梯(`0.0.1-beta → - 0.0.2-beta`,无计数器)而非 Core 的记法。两者均内置 Core `0.0.2-beta.1`, - 同步到 `0.0.2-beta.2` 是下一步。 + 0.0.2-beta`,无计数器)而非 Core 的记法。两者均内置 Core `0.0.2-beta.1`,并且会 + 一直停在这里,直到各自切自己的发布:集成在发布时同步 Core,而不是 Core 一发布就 + 同步。规则本身与它唯一的例外写在 `docs/projects.md`。 - **Core**:minify 刻意缺席(零构建);公开 beta 时再议。 - **完全没有规则的字面量形式**(beta.5 审查时发现,因属"缺功能"而非"抢错范围"而推迟): From 0b14818cd52525bcaf0103d4d5d8e92375d5984c Mon Sep 17 00:00:00 2001 From: Jie Date: Sun, 6 Sep 2026 19:41:20 +0800 Subject: [PATCH 5/9] Fail the build when one translation says more than the other MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every doc here is written twice, and nothing checked that the two agreed. A change landing in one language breaks no build and reads perfectly well — in the language you happen to be reading. #22 sat English-only for a day, and the Chinese repository table had been answering a different question than the English one for longer than that. The check compares what cannot legitimately differ between translations: versions, paths, package specifiers, links, section structure and table row counts. Prose, sentence counts and translated placeholders are left alone — against <版本> is not drift. Presence is what it asserts, not counts, because a translation may merge or split sentences freely. Run against the two files as they stood before this branch, it reports both of the omissions above. --- .github/workflows/ci.yml | 3 + CONTRIBUTING.md | 6 ++ package.json | 1 + tools/check-docs-parity.mjs | 108 ++++++++++++++++++++++++++++++++++++ 4 files changed, 118 insertions(+) create mode 100644 tools/check-docs-parity.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 98ea4a7..6d334e2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,6 +118,9 @@ jobs: - name: Validate version metadata run: npm run check:versions + - name: Check translated docs agree + run: npm run check:docs-parity + - name: Run tests run: npm test diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4a6d745..8af4c1c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -155,6 +155,7 @@ Every change lands the same way: git checkout -b my-change # ... work, then: npm test && node tools/check-versions.mjs && node tools/integrity.mjs --check +npm run check:docs-parity # if the change touched a document git push origin my-change gh pr create --fill ``` @@ -164,6 +165,11 @@ merge; the branch also has to be up to date with `main`. - One PR per concern, to keep reviews easy. - Engine or grammar changes must come with added / updated tests. +- Every document here is written twice, as `X.md` and `X.zh-CN.md`. Both + translations belong in the same pull request: a rule that exists in one + language is a rule half the readers never see, and it reads perfectly well + in the language you happen to be checking. `check:docs-parity` fails when a + version, path, package specifier, or link appears in only one of a pair. - Palette changes should include demo screenshots (both dark and light). ## Releasing diff --git a/package.json b/package.json index 9bd097f..bf33bce 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ "scripts": { "build": "sh build.sh", "check:versions": "node tools/check-versions.mjs", + "check:docs-parity": "node tools/check-docs-parity.mjs", "test": "node --test tests/*.mjs", "demo": "echo 'Open demo/index.html in a browser, or serve the project root and visit /demo/'", "sync:integrations": "sh tools/sync-integrations.sh", diff --git a/tools/check-docs-parity.mjs b/tools/check-docs-parity.mjs new file mode 100644 index 0000000..255b89d --- /dev/null +++ b/tools/check-docs-parity.mjs @@ -0,0 +1,108 @@ +#!/usr/bin/env node +// Every doc in this repository is written twice. A change that lands in one +// language and not the other does not break a build, does not fail a test, and +// reads perfectly well — in the language you happen to be reading. The Core +// sync rule sat in `docs/projects.md` for a day with no Chinese counterpart; +// the Chinese repository table answered a different question than the English +// one ("status" against "where you get it today") while the sentence under +// both explained the English one. +// +// So this compares the parts of a document that cannot legitimately differ +// between translations: version numbers, file paths, package specifiers, and +// links. Prose differs, sentence counts differ, and a placeholder is expected +// to be translated (`` / `<版本>`) — none of that is checked. What is +// checked is presence: something named in one language and nowhere in the +// other. +import { readFileSync, readdirSync, statSync } from 'node:fs'; +import { join } from 'node:path'; + +const SKIP = new Set(['node_modules', '_site', '.git', '.vscode-test', 'dist']); +const fail = []; + +function walk(dir, out = []) { + for (const entry of readdirSync(dir)) { + if (SKIP.has(entry)) continue; + const full = join(dir, entry); + if (statSync(full).isDirectory()) walk(full, out); + else if (entry.endsWith('.md')) out.push(full); + } + return out; +} + +// A translated placeholder is still the same placeholder. +const normalise = (s) => s.replace(/<[^>]*>/g, '<>').trim(); + +// Anything that names a file, a directory, or a package — the things a reader +// is meant to go and find. A backticked fragment of prose or a translated +// example (`# Heading` / `# 标题`) is not one of these and is left alone. +const PATHISH = /\/|\.(mjs|cjs|js|json|css|sh|php|md|vsix|zip|html|ts|yml|yaml)\b/; + +function invariants(path) { + const text = readFileSync(path, 'utf8'); + const code = new Set(); + for (const [, span] of text.matchAll(/`([^`\n]+)`/g)) { + const value = normalise(span); + if (PATHISH.test(value)) code.add(value); + } + const versions = new Set( + [...text.matchAll(/\b\d+\.\d+\.\d+(?:-[A-Za-z0-9.]+)?\b/g)].map((m) => m[0]) + ); + const links = new Set( + [...text.matchAll(/https?:\/\/[^\s)\]`"'>]+/g)] + .map((m) => normalise(m[0].replace(/[.,;:，。、]+$/, ''))) + // Badge URLs carry their own translated label, by design. + .filter((url) => !url.includes('img.shields.io')) + ); + // Headings are counted outside fenced blocks only: a shell comment reading + // `# Install` is not a section, and it is translated. + const prose = text.replace(/^```[\s\S]*?^```/gm, ''); + const headings = [...prose.matchAll(/^(#{1,6})\s+\S/gm)].map((m) => m[1].length); + const tableRows = (text.match(/^\s*\|.*\|\s*$/gm) || []).length; + return { code, versions, links, headings, tableRows }; +} + +function compare(label, enPath, zhPath, enSet, zhSet) { + for (const [side, missing, from] of [ + ['中文', [...enSet].filter((v) => !zhSet.has(v)), enPath], + ['English', [...zhSet].filter((v) => !enSet.has(v)), zhPath], + ]) { + for (const value of missing) { + fail.push(`${from} names ${label} ${JSON.stringify(value)}, its ${side} counterpart does not`); + } + } +} + +const docs = walk('.'); +const pairs = docs + .filter((p) => p.endsWith('.zh-CN.md')) + .map((zh) => [`${zh.slice(0, -'.zh-CN.md'.length)}.md`, zh]) + .filter(([en]) => docs.includes(en)); + +if (!pairs.length) { + console.error('No translated document pairs found — this check would pass vacuously.'); + process.exit(1); +} + +for (const [en, zh] of pairs) { + const a = invariants(en); + const b = invariants(zh); + compare('the version', en, zh, a.versions, b.versions); + compare('the path', en, zh, a.code, b.code); + compare('the link', en, zh, a.links, b.links); + // A section present in one language and not the other, or a table that grew + // a row on one side only. + if (a.headings.join() !== b.headings.join()) { + fail.push(`${en} and ${zh} do not have the same section structure: heading levels ${a.headings.join('-')} against ${b.headings.join('-')}`); + } + if (a.tableRows !== b.tableRows) { + fail.push(`${en} has ${a.tableRows} table rows, ${zh} has ${b.tableRows}`); + } +} + +if (fail.length) { + console.error('Translated documents have drifted:'); + for (const message of fail) console.error(`- ${message}`); + process.exit(1); +} + +console.log(`docs parity ok: ${pairs.length} translated pairs agree`); From b05d885c77819d6d078db53f705bae1bfe2ed2ca Mon Sep 17 00:00:00 2001 From: Jie Date: Sun, 6 Sep 2026 19:41:20 +0800 Subject: [PATCH 6/9] Hold the repository table's specifier to the current release The table's last column is the only place in the docs quoting a full npm specifier, and it is what a reader copies. A parity check catches it going missing from one language; nothing catches both languages naming the same stale release, which is the shape every other entry in this file was added for. --- tools/check-versions.mjs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/check-versions.mjs b/tools/check-versions.mjs index e045482..8cf7b70 100644 --- a/tools/check-versions.mjs +++ b/tools/check-versions.mjs @@ -70,6 +70,14 @@ includes('docs/versioning.zh-CN.md', `当前版本:\`${version}\``); includes('docs/projects.md', 'JSRay Core'); includes('docs/projects.zh-CN.md', 'JSRay Core'); +// The repository table's last column tells a reader what to install today, and +// it is the one place in the docs that quotes a full npm specifier. A parity +// check catches it going missing from one language; nothing catches both +// languages naming the same stale release. +for (const path of ['docs/projects.md', 'docs/projects.zh-CN.md']) { + includes(path, `@jsray/core@${version}`, `the installable specifier @jsray/core@${version}`); +} + // The supported-versions table is a promise to anyone deciding whether to // report privately. It sat on beta.1 through the whole beta.2 cycle. includes('SECURITY.md', `| ${version} | ✅`, `${version} in the supported-versions table`); From d831ee9d4cde8276df3a6bd2b36c2b8ffaa647a9 Mon Sep 17 00:00:00 2001 From: Jie Date: Mon, 7 Sep 2026 18:48:50 +0800 Subject: [PATCH 7/9] Replace two expiring deadlines with conditions that hold MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "Revisit at public beta" named 2026-07-17. That date passed, nothing happened, and the entry became something every planning round rediscovers and re-argues from scratch. The minify question now carries its answer — a readable, auditable dist is worth more than the 9 KB brotli saves, measured — and a condition for reopening it that cannot silently expire: a real size complaint, or Core growing substantially. Detection tuning said it "belongs with the 0.0.2 engine work" while we are already inside 0.0.2. What it actually needs is a beta round of its own, because retuning the scores reorders all 83 grammars at once and nothing else in the same round would be separable from it. The third of these — string rules waiting on "the API pass in 0.0.2" — is left alone here because the work itself is what rewrites that entry. --- docs/development.md | 16 +++++++++++++--- docs/development.zh-CN.md | 13 ++++++++++--- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/docs/development.md b/docs/development.md index b8391e9..3dc799a 100644 --- a/docs/development.md +++ b/docs/development.md @@ -419,8 +419,18 @@ deliberately deferred). integration syncs Core as part of releasing, not when Core ships. The rule and its one exception are in `docs/projects.md`. -- **Core**: minification is deliberately absent (zero-build); revisit at - public beta. +- **Core**: minification is deliberately absent, and that is a decision now + rather than a pending one. Stripping comments and indentation takes the + brotli transfer from 23.9 KB to 14.8 KB — 9 KB is a real saving, and the + comments alone are 31% of the file, written for maintainers and downloaded + by every visitor. What it buys against that is a second artifact travelling + the whole chain — `integrity.json`, the three bundled snapshots, the + `/v//` paths, the SRI examples in both READMEs — and a new way for + a release to ship corrupted. A `dist/` a user can read and audit against its + digest is worth more than 9 KB today. Reopen this on a real size complaint + or a substantial growth in Core, not at a version number: "revisit at public + beta" named 2026-07-17, a date that passed, after which the entry sat here + being rediscovered as an open question every planning round. - **Literal forms with no rule** (found while auditing for beta.5, deferred because they are absent features rather than wrong spans): heredocs — `<</` 路径、两份 README 里的 + SRI 示例 —— 外加一种新的发布损坏方式。今天,一份用户能读、能对着摘要自行审计的 + `dist/` 比那 9 KB 更值钱。重新评估的条件是出现真实的体积诉求,或 Core 显著变大, + 而不是某个版本号:"公开 beta 时再议"指的是 2026-07-17,那个日子已经过去,此后这一 + 条就杵在这里,每做一轮计划就被当成未决问题重新发现一次。 - **完全没有规则的字面量形式**(beta.5 审查时发现,因属"缺功能"而非"抢错范围"而推迟): heredoc —— PHP 的 `<< Date: Mon, 7 Sep 2026 21:24:57 +0800 Subject: [PATCH 8/9] Record what this beta has taken on so far Six commits landed without a changelog entry between them: the parity check, the translations it was built to catch, and the two roadmap deadlines that were rewritten as conditions. The audits that keep finding unrecorded batches would have found this one. --- CHANGELOG.md | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d73cf40..e800efc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,41 @@ versioning follows [SemVer](https://semver.org/). ## [Unreleased] +### Added + +- **`check:docs-parity`.** Every document here is written twice, and nothing + checked that the two agreed. The check compares what cannot legitimately + differ between translations — versions, paths, package specifiers, links, + section structure, table row counts — and leaves prose, sentence counts and + translated placeholders alone. It is wired into CI beside `check:versions`. + +### Fixed + +- **The Chinese docs said less than the English ones.** The Core sync rule + merged in #22 existed only in English; the Chinese repository table was + headed 状态 and answered a different question than the English column, while + the sentence under it explained the English one; and the versioning doc + dropped the npm account the package publishes from. + +- **The roadmap described jsray-terminal as having a release.** It has none — + the only way in is the GitHub install line. It also carried "syncing them to + 0.0.2-beta.2 is the next step", written when Core was on beta.2 and wrong in + two ways since: it named a superseded release, and an integration syncs Core + as part of its own release, so being behind between releases is the expected + state rather than a queued chore. + +### Changed + +- **Two deadlines became conditions.** "Revisit at public beta" named + 2026-07-17; that date passed and the entry became something every planning + round rediscovered and re-argued. Minification now carries its answer — a + readable, auditable `dist/` is worth more than the 9 KB brotli saves, and + the comments are 31% of the file — together with a trigger that cannot + expire: a real size complaint, or Core growing substantially. Detection + tuning no longer "belongs with the 0.0.2 engine work" while we are inside + 0.0.2; it needs a beta round of its own, because retuning the scores + reorders all 83 grammars at once. + ## [0.0.2-beta.3] — 2026-09-06 Documentation and the guards around it. No runtime change: `dist/` differs from From 3b6fd0f5c4b60b263e8047221e72b9290b6eeb9c Mon Sep 17 00:00:00 2001 From: Jie Date: Tue, 8 Sep 2026 06:09:37 +0800 Subject: [PATCH 9/9] Give a rule a terminator it can only know at match time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Heredocs, %w[…], q{…} and sigils rendered as ordinary code, and not for want of a pattern: their end is genuinely not knowable when the rule is written. `<< out.txt` stays shell instead of being dragged into the literal. Eleven cases in tests/constructs.test.mjs, including the openings that must be declined and four unterminated-opening storms in the backtracking guard — these scan forward, so a file of nothing but openings is the shape that would expose a quadratic one. --- CHANGELOG.md | 24 +++++++ dist/jsray.js | 129 +++++++++++++++++++++++++++++++++++++- docs/development.md | 28 +++++---- docs/development.zh-CN.md | 18 +++--- integrity.json | 2 +- src/jsray.js | 129 +++++++++++++++++++++++++++++++++++++- tests/constructs.test.mjs | 100 +++++++++++++++++++++++++++++ types/jsray.d.ts | 8 +++ 8 files changed, 413 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e800efc..205cbcb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,30 @@ versioning follows [SemVer](https://semver.org/). ### Added +- **Literals whose end is decided at runtime.** A grammar rule may now carry + `close`, a function that receives the opening match and reports where the + form ends. Two builders cover everything the grammars needed: a heredoc ends + at the word its own opening line named — `<< out.txt`) left as shell + rather than pulled into the literal; + - heredoc in **Ruby** (`<<~`, `<<-`, plain `<<`), on uppercase words only, + so `items << thing` stays the append operator; + - `%w[]`, `%i()`, `%q{}`, `%Q<>` and `%r{}` in **Ruby**, `q{}`, `qq{}`, + `qw()` and `qr{}` in **Perl**, and `~s{}`, `~w[]`, `~r//` sigils in + **Elixir** — each nesting correctly through inner brackets, and each + holding a `#` without it becoming a comment. + + An opening whose terminator never arrives reports no match, leaving the text + to the rules behind it. A false opening is likelier than a genuinely + unterminated literal, and the alternative is a stray `<<` swallowing the + rest of the file. + - **`check:docs-parity`.** Every document here is written twice, and nothing checked that the two agreed. The check compares what cannot legitimately differ between translations — versions, paths, package specifiers, links, diff --git a/dist/jsray.js b/dist/jsray.js index 985f414..51d6cff 100644 --- a/dist/jsray.js +++ b/dist/jsray.js @@ -23,7 +23,17 @@ * { cls: 'tk-xxx', * pattern: /re/, // must be globalizable; the 'g' flag is forced internally * inside?: rules, // nested grammar (recursively tokenize captured text) - * lookbehind?: true } // capture group 1 is consumed as prefix but not colored + * lookbehind?: true, // capture group 1 is consumed as prefix but not colored + * close?: fn } // pattern matches the opening only; fn finds the end + * + * `close(match, text, from) -> index | -1` exists for the forms whose end is + * not knowable when the rule is written. A heredoc ends at the word its own + * opening line named — `<< last) next.push(piece.slice(last, start)); next.push({ @@ -51,6 +66,9 @@ content: rule.inside ? tokenize(text, rule.inside) : text, }); last = start + text.length; + // The body of a close-delimited form has already been consumed; + // resuming inside it would re-match its own contents. + if (rule.close) re.lastIndex = last; } if (last < piece.length) next.push(piece.slice(last)); } @@ -74,6 +92,60 @@ const G = {}; // grammars + // ---------- runtime terminators ---------- + // Two `close` builders cover every delimited form the grammars below need. + // Both take the opening match and report where the form ends. + + /** + * A heredoc ends at the word its opening line named. `nameGroup` is the + * capture holding that word; `indentGroup` is the capture holding the `-` or + * `~` that permits an indented terminator (pass `true` where the language + * always permits one, as PHP does since 7.3). `trailing` overrides what may + * follow the word on its closing line. + * + * The name is interpolated into a RegExp, which is only safe because every + * opening pattern here restricts it to `[A-Za-z_]\w*` — no metacharacters + * can reach this. + */ + function heredocEnd(nameGroup, indentGroup, trailing) { + return (m, text, from) => { + const name = m[nameGroup]; + if (!name) return -1; + const indented = indentGroup === true ? true : !!m[indentGroup]; + const re = new RegExp( + '^' + (indented ? '[ \\t]*' : '') + name + (trailing || '[ \\t]*$'), + 'm' + ); + const hit = re.exec(text.slice(from)); + return hit ? from + hit.index + hit[0].length : -1; + }; + } + + const CLOSERS = { '(': ')', '[': ']', '{': '}', '<': '>' }; + + /** + * A delimiter-chosen literal — Ruby's `%w[…]`, Perl's `q{…}`, an Elixir + * sigil — ends at whatever closes the character it opened with. Bracket + * pairs nest; a symmetric delimiter such as `%w!…!` cannot, and counting + * depth on one would end the literal at its own opening character. + */ + function pairedEnd(openGroup) { + return (m, text, from) => { + const open = m[openGroup]; + if (!open) return -1; + const close = CLOSERS[open] || open; + const nests = close !== open; + let depth = 1; + for (let i = from; i < text.length; i++) { + const c = text[i]; + if (c === '\\') { i++; continue; } + if (nests && c === open) depth++; + else if (c === close && --depth === 0) return i + 1; + } + return -1; + }; + } + // ---------- shared fragments ---------- const RX = { string1: /"(?:\\.|[^"\\\n])*"/, @@ -386,6 +458,15 @@ 'docker kubectl python python3 pip pip3 ruby go cargo make brew apt yum'; G.shell = [ + // Heredocs first: their body may hold quotes and `#`, and every rule + // after this one would claim those. The opening line is group 1 and is + // consumed as an uncolored prefix, so `cat < out.txt` keeps its + // redirect as shell rather than dragging it into the literal. + { cls: 'tk-string', + pattern: /(<<(-?)[ \t]*(['"]?)([A-Za-z_]\w*)\3[^\n]*\n)/, + lookbehind: true, + close: heredocEnd(4, 2) }, + // Strings before comments, else # inside "..." is eaten as a comment. // Each `$` form is matched exactly, never by a greedy run that could also // swallow the ones after it — the old `\$[\w{][^"\n]*` overlapped itself @@ -431,6 +512,15 @@ ).split(' '); G.php = [ + // Heredoc and nowdoc ahead of the comment rules: `#` and `//` are + // ordinary text inside one. The closing word may be indented and may be + // followed by `;` or `,`, which is why the terminator ends at a word + // boundary rather than at end of line. + { cls: 'tk-string', + pattern: /(<<<[ \t]*(['"]?)([A-Za-z_]\w*)\2\r?\n)/, + lookbehind: true, + close: heredocEnd(3, true, '\\b') }, + // Block comments before strings; line comments (// and #) after strings // so "https://..." and "#anchor" inside strings never become comments. { cls: 'tk-comment', pattern: /\/\*[\s\S]*?\*\// }, @@ -732,12 +822,32 @@ const RB_BUILTINS = 'puts print p gets raise lambda proc loop each map select reject reduce new'.split(' '); G.ruby = [ + // Heredocs, and only with an uppercase word: `<<` is also the append + // operator, and `items << thing` must not open one. An uppercase name is + // the convention, and where a constant does follow `<<` the terminator + // line will not exist, so the form declines itself rather than eating the + // file. `<<~` and `<<-` permit an indented terminator; plain `<<` does not. + { cls: 'tk-string', + pattern: /(<<([-~]?)(['"]?)([A-Z_]\w*)\3[^\n]*\n)/, + lookbehind: true, + close: heredocEnd(4, 2) }, + // `=begin` / `=end` blocks come before the strings that come before // everything else. The markers are only special at column zero, so the // anchors here are load-bearing rather than decorative. Without this rule // a documentation block was read as ordinary code — the body's words came // out coloured as function calls and keywords. { cls: 'tk-comment', pattern: /^=begin\b[\s\S]*?^=end.*$/m }, + + // %w[…] %i(…) %q{…} %Q<…>: the delimiter is picked at the call site, so + // the closer is only knowable once the opener has been read, and bracket + // pairs nest. %r is a regex, not a string. The bare `%(…)` form is left + // out on purpose — it cannot be told from the modulo operator without + // parsing, and it is rare enough not to be worth mistaking `a %(b)` for a + // literal. + { cls: 'tk-regex', pattern: /%r([([{<|!\/])/, close: pairedEnd(1) }, + { cls: 'tk-string', pattern: /%[wWiIqQsx]([([{<|!\/])/, close: pairedEnd(1) }, + // Strings must come before comments, else # inside "..." (incl. #{} interpolation) // is eaten as a comment. String bodies stay single-line so an unpaired quote // in a comment can't swallow following lines. @@ -921,6 +1031,13 @@ { cls: 'tk-var', pattern: /[$@][A-Za-z_]\w*/ }, ]}, { cls: 'tk-string', pattern: /'(?:\\.|[^'\\\n])*'/ }, + + // q{…} qq{…} qw{…} qr{…} — ahead of the comment rule, because `#` is + // ordinary text inside one. `/` is excluded as a delimiter for the + // quoting forms: after a bare word it is far more often division. + { cls: 'tk-regex', pattern: /\bqr[ \t]*([([{<|!\/])/, close: pairedEnd(1) }, + { cls: 'tk-string', pattern: /\b(?:qq|qw|q)[ \t]*([([{<|!])/, close: pairedEnd(1) }, + { cls: 'tk-comment', pattern: /#.*/ }, { cls: 'tk-regex', pattern: /((?:=~|!~)\s*)(?:m|s|tr|y)?\/(?:\\.|[^/\n])*\/[a-z]*/, lookbehind: true }, { cls: 'tk-var-builtin', pattern: /\$[_0-9&`'+^!]|\$\^\w|\@ARGV\b|\%ENV\b|\$0\b/ }, @@ -983,6 +1100,14 @@ { cls: 'tk-operator', pattern: /#\{[^}\n]*\}/ }, ]}, { cls: 'tk-string', pattern: /'(?:\\.|[^'\\\n])*'/ }, + + // Sigils ~s{…} ~w[…] ~r/…/, ahead of the comment rule for the same reason + // the strings are. A `"` delimiter is not accepted here: it would end + // `~s"""…"""` at the second quote, and the triple-quote rule above + // already renders that form correctly. + { cls: 'tk-regex', pattern: /~[rR]([([{<|\/'])/, close: pairedEnd(1) }, + { cls: 'tk-string', pattern: /~[a-zA-Z]([([{<|\/'])/, close: pairedEnd(1) }, + { cls: 'tk-comment', pattern: /#.*/ }, { cls: 'tk-decorator', pattern: /@[a-z_]\w*/ }, { cls: 'tk-var-const', pattern: /:[a-z_]\w*[?!]?/ }, diff --git a/docs/development.md b/docs/development.md index 3dc799a..01a4aba 100644 --- a/docs/development.md +++ b/docs/development.md @@ -431,13 +431,13 @@ deliberately deferred). or a substantial growth in Core, not at a version number: "revisit at public beta" named 2026-07-17, a date that passed, after which the entry sat here being rediscovered as an open question every planning round. -- **Literal forms with no rule** (found while auditing for beta.5, deferred - because they are absent features rather than wrong spans): heredocs — - `<< last) next.push(piece.slice(last, start)); next.push({ @@ -51,6 +66,9 @@ content: rule.inside ? tokenize(text, rule.inside) : text, }); last = start + text.length; + // The body of a close-delimited form has already been consumed; + // resuming inside it would re-match its own contents. + if (rule.close) re.lastIndex = last; } if (last < piece.length) next.push(piece.slice(last)); } @@ -74,6 +92,60 @@ const G = {}; // grammars + // ---------- runtime terminators ---------- + // Two `close` builders cover every delimited form the grammars below need. + // Both take the opening match and report where the form ends. + + /** + * A heredoc ends at the word its opening line named. `nameGroup` is the + * capture holding that word; `indentGroup` is the capture holding the `-` or + * `~` that permits an indented terminator (pass `true` where the language + * always permits one, as PHP does since 7.3). `trailing` overrides what may + * follow the word on its closing line. + * + * The name is interpolated into a RegExp, which is only safe because every + * opening pattern here restricts it to `[A-Za-z_]\w*` — no metacharacters + * can reach this. + */ + function heredocEnd(nameGroup, indentGroup, trailing) { + return (m, text, from) => { + const name = m[nameGroup]; + if (!name) return -1; + const indented = indentGroup === true ? true : !!m[indentGroup]; + const re = new RegExp( + '^' + (indented ? '[ \\t]*' : '') + name + (trailing || '[ \\t]*$'), + 'm' + ); + const hit = re.exec(text.slice(from)); + return hit ? from + hit.index + hit[0].length : -1; + }; + } + + const CLOSERS = { '(': ')', '[': ']', '{': '}', '<': '>' }; + + /** + * A delimiter-chosen literal — Ruby's `%w[…]`, Perl's `q{…}`, an Elixir + * sigil — ends at whatever closes the character it opened with. Bracket + * pairs nest; a symmetric delimiter such as `%w!…!` cannot, and counting + * depth on one would end the literal at its own opening character. + */ + function pairedEnd(openGroup) { + return (m, text, from) => { + const open = m[openGroup]; + if (!open) return -1; + const close = CLOSERS[open] || open; + const nests = close !== open; + let depth = 1; + for (let i = from; i < text.length; i++) { + const c = text[i]; + if (c === '\\') { i++; continue; } + if (nests && c === open) depth++; + else if (c === close && --depth === 0) return i + 1; + } + return -1; + }; + } + // ---------- shared fragments ---------- const RX = { string1: /"(?:\\.|[^"\\\n])*"/, @@ -386,6 +458,15 @@ 'docker kubectl python python3 pip pip3 ruby go cargo make brew apt yum'; G.shell = [ + // Heredocs first: their body may hold quotes and `#`, and every rule + // after this one would claim those. The opening line is group 1 and is + // consumed as an uncolored prefix, so `cat < out.txt` keeps its + // redirect as shell rather than dragging it into the literal. + { cls: 'tk-string', + pattern: /(<<(-?)[ \t]*(['"]?)([A-Za-z_]\w*)\3[^\n]*\n)/, + lookbehind: true, + close: heredocEnd(4, 2) }, + // Strings before comments, else # inside "..." is eaten as a comment. // Each `$` form is matched exactly, never by a greedy run that could also // swallow the ones after it — the old `\$[\w{][^"\n]*` overlapped itself @@ -431,6 +512,15 @@ ).split(' '); G.php = [ + // Heredoc and nowdoc ahead of the comment rules: `#` and `//` are + // ordinary text inside one. The closing word may be indented and may be + // followed by `;` or `,`, which is why the terminator ends at a word + // boundary rather than at end of line. + { cls: 'tk-string', + pattern: /(<<<[ \t]*(['"]?)([A-Za-z_]\w*)\2\r?\n)/, + lookbehind: true, + close: heredocEnd(3, true, '\\b') }, + // Block comments before strings; line comments (// and #) after strings // so "https://..." and "#anchor" inside strings never become comments. { cls: 'tk-comment', pattern: /\/\*[\s\S]*?\*\// }, @@ -732,12 +822,32 @@ const RB_BUILTINS = 'puts print p gets raise lambda proc loop each map select reject reduce new'.split(' '); G.ruby = [ + // Heredocs, and only with an uppercase word: `<<` is also the append + // operator, and `items << thing` must not open one. An uppercase name is + // the convention, and where a constant does follow `<<` the terminator + // line will not exist, so the form declines itself rather than eating the + // file. `<<~` and `<<-` permit an indented terminator; plain `<<` does not. + { cls: 'tk-string', + pattern: /(<<([-~]?)(['"]?)([A-Z_]\w*)\3[^\n]*\n)/, + lookbehind: true, + close: heredocEnd(4, 2) }, + // `=begin` / `=end` blocks come before the strings that come before // everything else. The markers are only special at column zero, so the // anchors here are load-bearing rather than decorative. Without this rule // a documentation block was read as ordinary code — the body's words came // out coloured as function calls and keywords. { cls: 'tk-comment', pattern: /^=begin\b[\s\S]*?^=end.*$/m }, + + // %w[…] %i(…) %q{…} %Q<…>: the delimiter is picked at the call site, so + // the closer is only knowable once the opener has been read, and bracket + // pairs nest. %r is a regex, not a string. The bare `%(…)` form is left + // out on purpose — it cannot be told from the modulo operator without + // parsing, and it is rare enough not to be worth mistaking `a %(b)` for a + // literal. + { cls: 'tk-regex', pattern: /%r([([{<|!\/])/, close: pairedEnd(1) }, + { cls: 'tk-string', pattern: /%[wWiIqQsx]([([{<|!\/])/, close: pairedEnd(1) }, + // Strings must come before comments, else # inside "..." (incl. #{} interpolation) // is eaten as a comment. String bodies stay single-line so an unpaired quote // in a comment can't swallow following lines. @@ -921,6 +1031,13 @@ { cls: 'tk-var', pattern: /[$@][A-Za-z_]\w*/ }, ]}, { cls: 'tk-string', pattern: /'(?:\\.|[^'\\\n])*'/ }, + + // q{…} qq{…} qw{…} qr{…} — ahead of the comment rule, because `#` is + // ordinary text inside one. `/` is excluded as a delimiter for the + // quoting forms: after a bare word it is far more often division. + { cls: 'tk-regex', pattern: /\bqr[ \t]*([([{<|!\/])/, close: pairedEnd(1) }, + { cls: 'tk-string', pattern: /\b(?:qq|qw|q)[ \t]*([([{<|!])/, close: pairedEnd(1) }, + { cls: 'tk-comment', pattern: /#.*/ }, { cls: 'tk-regex', pattern: /((?:=~|!~)\s*)(?:m|s|tr|y)?\/(?:\\.|[^/\n])*\/[a-z]*/, lookbehind: true }, { cls: 'tk-var-builtin', pattern: /\$[_0-9&`'+^!]|\$\^\w|\@ARGV\b|\%ENV\b|\$0\b/ }, @@ -983,6 +1100,14 @@ { cls: 'tk-operator', pattern: /#\{[^}\n]*\}/ }, ]}, { cls: 'tk-string', pattern: /'(?:\\.|[^'\\\n])*'/ }, + + // Sigils ~s{…} ~w[…] ~r/…/, ahead of the comment rule for the same reason + // the strings are. A `"` delimiter is not accepted here: it would end + // `~s"""…"""` at the second quote, and the triple-quote rule above + // already renders that form correctly. + { cls: 'tk-regex', pattern: /~[rR]([([{<|\/'])/, close: pairedEnd(1) }, + { cls: 'tk-string', pattern: /~[a-zA-Z]([([{<|\/'])/, close: pairedEnd(1) }, + { cls: 'tk-comment', pattern: /#.*/ }, { cls: 'tk-decorator', pattern: /@[a-z_]\w*/ }, { cls: 'tk-var-const', pattern: /:[a-z_]\w*[?!]?/ }, diff --git a/tests/constructs.test.mjs b/tests/constructs.test.mjs index 8cab513..32d83c8 100644 --- a/tests/constructs.test.mjs +++ b/tests/constructs.test.mjs @@ -336,6 +336,13 @@ test('every new string form stays linear on pathological input', () => { ['cpp', '0x1p'.repeat(10000)], // hex exponent storm ['go', '1' + '_'.repeat(20000)], // separator storm ['ruby', '=begin\n'.repeat(5000)], // unterminated block comments + // Runtime terminators scan forward from each opening. An opening that + // never closes makes that scan reach the end of the input, so a file of + // nothing but openings is the shape that would expose a quadratic one. + ['shell', 'cat < { ); } }); + +// ── Forms whose end is decided at runtime ────────────────────────────────── +// A heredoc ends at the word its own opening named; a %w[…] ends at the +// bracket matching its opener. Neither is expressible as a RegExp, so both +// rendered as ordinary code until `close` existed. The cases that matter are +// the ones where the body holds characters another rule wants: `#`, `//`, a +// quote, a brace. + +test('shell: a heredoc body is one string, and its # is not a comment', () => { + const code = 'cat < { + const code = 'cat < out.txt\nbody\nEOF'; + + // The opening line is consumed as an uncolored prefix precisely so that + // `> out.txt` is not dragged into the literal. + notSwallowed(code, 'shell', 'string', '> out.txt'); + token(code, 'shell', 'operator', '>'); +}); + +test('shell: <<- permits an indented terminator, << does not', () => { + token('cat <<-END\n\tbody\n\tEND\n', 'shell', 'string', '\tbody\n\tEND'); + + // Plain `<<` requires the word at column zero; an indented one is not a + // terminator, and with none present the form declines rather than running on. + const indented = 'cat < t.type === 'tk-string'); + assert.equal(strings.length, 0, `expected no string, got ${JSON.stringify(strings)}`); +}); + +test('shell: an arithmetic shift does not open a heredoc', () => { + const code = 'echo $(( a << 2 ))'; + const strings = leaves(JSRay.tokenize(code, 'shell')).filter((t) => t.type === 'tk-string'); + + assert.equal(strings.length, 0, `<< opened a literal: ${JSON.stringify(strings)}`); +}); + +test('PHP: a heredoc holds // and closes on an indented word', () => { + const code = ' { + const code = " { + token('sql = <<~SQL\n SELECT 1\nSQL\n', 'ruby', 'string', ' SELECT 1\nSQL'); + + // `items << thing` is the append operator and must stay one. + const append = 'items << thing'; + const strings = leaves(JSRay.tokenize(append, 'ruby')).filter((t) => t.type === 'tk-string'); + assert.equal(strings.length, 0, `append became a literal: ${JSON.stringify(strings)}`); +}); + +test('Ruby: %w and %q close on the matching bracket, and nest', () => { + token('a = %w[one two three]', 'ruby', 'string', '%w[one two three]'); + token('c = %q{outer {inner} still}', 'ruby', 'string', '%q{outer {inner} still}'); + token('d = %r{^\\d+$}', 'ruby', 'regex', '%r{^\\d+$}'); +}); + +test('Ruby: modulo is not a percent literal', () => { + const code = 'e = x % y'; + const strings = leaves(JSRay.tokenize(code, 'ruby')).filter((t) => t.type === 'tk-string'); + + assert.equal(strings.length, 0, `modulo became a literal: ${JSON.stringify(strings)}`); +}); + +test('Perl: q{} holds a # without becoming a comment', () => { + const code = 'my $s = q{hello # not comment};'; + + token(code, 'perl', 'string', 'q{hello # not comment}'); + notSwallowed(code, 'perl', 'comment', 'not comment'); + token('my @w = qw(a b c);', 'perl', 'string', 'qw(a b c)'); + token('my $r = qr{^\\d+};', 'perl', 'regex', 'qr{^\\d+}'); +}); + +test('Elixir: a sigil carries its own delimiters', () => { + token('a = ~w[one two]', 'elixir', 'string', '~w[one two]'); + token('b = ~r/^\\d+/', 'elixir', 'regex', '~r/^\\d+/'); + + const code = 'c = ~s{hi # not comment}'; + token(code, 'elixir', 'string', '~s{hi # not comment}'); + notSwallowed(code, 'elixir', 'comment', 'not comment'); +}); diff --git a/types/jsray.d.ts b/types/jsray.d.ts index efdddfb..3dce653 100644 --- a/types/jsray.d.ts +++ b/types/jsray.d.ts @@ -11,6 +11,14 @@ declare namespace JSRay { pattern: RegExp; inside?: GrammarRule[]; lookbehind?: boolean; + /** + * For forms whose end is not knowable when the rule is written — a + * heredoc ends at the word its own opening line named, `%w[…]` at the + * bracket matching its opener. `pattern` matches the opening; this + * returns the index just past the end of the whole form, or -1 when no + * terminator is present, which leaves the opening to the rules behind it. + */ + close?: (match: RegExpExecArray, text: string, from: number) => number; } type Grammar = GrammarRule[];