diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 98ea4a7..6d334e2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,6 +118,9 @@ jobs: - name: Validate version metadata run: npm run check:versions + - name: Check translated docs agree + run: npm run check:docs-parity + - name: Run tests run: npm test diff --git a/CHANGELOG.md b/CHANGELOG.md index d73cf40..205cbcb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,65 @@ versioning follows [SemVer](https://semver.org/). ## [Unreleased] +### Added + +- **Literals whose end is decided at runtime.** A grammar rule may now carry + `close`, a function that receives the opening match and reports where the + form ends. Two builders cover everything the grammars needed: a heredoc ends + at the word its own opening line named — `<< out.txt`) left as shell + rather than pulled into the literal; + - heredoc in **Ruby** (`<<~`, `<<-`, plain `<<`), on uppercase words only, + so `items << thing` stays the append operator; + - `%w[]`, `%i()`, `%q{}`, `%Q<>` and `%r{}` in **Ruby**, `q{}`, `qq{}`, + `qw()` and `qr{}` in **Perl**, and `~s{}`, `~w[]`, `~r//` sigils in + **Elixir** — each nesting correctly through inner brackets, and each + holding a `#` without it becoming a comment. + + An opening whose terminator never arrives reports no match, leaving the text + to the rules behind it. A false opening is likelier than a genuinely + unterminated literal, and the alternative is a stray `<<` swallowing the + rest of the file. + +- **`check:docs-parity`.** Every document here is written twice, and nothing + checked that the two agreed. The check compares what cannot legitimately + differ between translations — versions, paths, package specifiers, links, + section structure, table row counts — and leaves prose, sentence counts and + translated placeholders alone. It is wired into CI beside `check:versions`. + +### Fixed + +- **The Chinese docs said less than the English ones.** The Core sync rule + merged in #22 existed only in English; the Chinese repository table was + headed 状态 and answered a different question than the English column, while + the sentence under it explained the English one; and the versioning doc + dropped the npm account the package publishes from. + +- **The roadmap described jsray-terminal as having a release.** It has none — + the only way in is the GitHub install line. It also carried "syncing them to + 0.0.2-beta.2 is the next step", written when Core was on beta.2 and wrong in + two ways since: it named a superseded release, and an integration syncs Core + as part of its own release, so being behind between releases is the expected + state rather than a queued chore. + +### Changed + +- **Two deadlines became conditions.** "Revisit at public beta" named + 2026-07-17; that date passed and the entry became something every planning + round rediscovered and re-argued. Minification now carries its answer — a + readable, auditable `dist/` is worth more than the 9 KB brotli saves, and + the comments are 31% of the file — together with a trigger that cannot + expire: a real size complaint, or Core growing substantially. Detection + tuning no longer "belongs with the 0.0.2 engine work" while we are inside + 0.0.2; it needs a beta round of its own, because retuning the scores + reorders all 83 grammars at once. + ## [0.0.2-beta.3] — 2026-09-06 Documentation and the guards around it. No runtime change: `dist/` differs from diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4a6d745..8af4c1c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -155,6 +155,7 @@ Every change lands the same way: git checkout -b my-change # ... work, then: npm test && node tools/check-versions.mjs && node tools/integrity.mjs --check +npm run check:docs-parity # if the change touched a document git push origin my-change gh pr create --fill ``` @@ -164,6 +165,11 @@ merge; the branch also has to be up to date with `main`. - One PR per concern, to keep reviews easy. - Engine or grammar changes must come with added / updated tests. +- Every document here is written twice, as `X.md` and `X.zh-CN.md`. Both + translations belong in the same pull request: a rule that exists in one + language is a rule half the readers never see, and it reads perfectly well + in the language you happen to be checking. `check:docs-parity` fails when a + version, path, package specifier, or link appears in only one of a pair. - Palette changes should include demo screenshots (both dark and light). ## Releasing diff --git a/dist/jsray.js b/dist/jsray.js index 985f414..51d6cff 100644 --- a/dist/jsray.js +++ b/dist/jsray.js @@ -23,7 +23,17 @@ * { cls: 'tk-xxx', * pattern: /re/, // must be globalizable; the 'g' flag is forced internally * inside?: rules, // nested grammar (recursively tokenize captured text) - * lookbehind?: true } // capture group 1 is consumed as prefix but not colored + * lookbehind?: true, // capture group 1 is consumed as prefix but not colored + * close?: fn } // pattern matches the opening only; fn finds the end + * + * `close(match, text, from) -> index | -1` exists for the forms whose end is + * not knowable when the rule is written. A heredoc ends at the word its own + * opening line named — `<< last) next.push(piece.slice(last, start)); next.push({ @@ -51,6 +66,9 @@ content: rule.inside ? tokenize(text, rule.inside) : text, }); last = start + text.length; + // The body of a close-delimited form has already been consumed; + // resuming inside it would re-match its own contents. + if (rule.close) re.lastIndex = last; } if (last < piece.length) next.push(piece.slice(last)); } @@ -74,6 +92,60 @@ const G = {}; // grammars + // ---------- runtime terminators ---------- + // Two `close` builders cover every delimited form the grammars below need. + // Both take the opening match and report where the form ends. + + /** + * A heredoc ends at the word its opening line named. `nameGroup` is the + * capture holding that word; `indentGroup` is the capture holding the `-` or + * `~` that permits an indented terminator (pass `true` where the language + * always permits one, as PHP does since 7.3). `trailing` overrides what may + * follow the word on its closing line. + * + * The name is interpolated into a RegExp, which is only safe because every + * opening pattern here restricts it to `[A-Za-z_]\w*` — no metacharacters + * can reach this. + */ + function heredocEnd(nameGroup, indentGroup, trailing) { + return (m, text, from) => { + const name = m[nameGroup]; + if (!name) return -1; + const indented = indentGroup === true ? true : !!m[indentGroup]; + const re = new RegExp( + '^' + (indented ? '[ \\t]*' : '') + name + (trailing || '[ \\t]*$'), + 'm' + ); + const hit = re.exec(text.slice(from)); + return hit ? from + hit.index + hit[0].length : -1; + }; + } + + const CLOSERS = { '(': ')', '[': ']', '{': '}', '<': '>' }; + + /** + * A delimiter-chosen literal — Ruby's `%w[…]`, Perl's `q{…}`, an Elixir + * sigil — ends at whatever closes the character it opened with. Bracket + * pairs nest; a symmetric delimiter such as `%w!…!` cannot, and counting + * depth on one would end the literal at its own opening character. + */ + function pairedEnd(openGroup) { + return (m, text, from) => { + const open = m[openGroup]; + if (!open) return -1; + const close = CLOSERS[open] || open; + const nests = close !== open; + let depth = 1; + for (let i = from; i < text.length; i++) { + const c = text[i]; + if (c === '\\') { i++; continue; } + if (nests && c === open) depth++; + else if (c === close && --depth === 0) return i + 1; + } + return -1; + }; + } + // ---------- shared fragments ---------- const RX = { string1: /"(?:\\.|[^"\\\n])*"/, @@ -386,6 +458,15 @@ 'docker kubectl python python3 pip pip3 ruby go cargo make brew apt yum'; G.shell = [ + // Heredocs first: their body may hold quotes and `#`, and every rule + // after this one would claim those. The opening line is group 1 and is + // consumed as an uncolored prefix, so `cat < out.txt` keeps its + // redirect as shell rather than dragging it into the literal. + { cls: 'tk-string', + pattern: /(<<(-?)[ \t]*(['"]?)([A-Za-z_]\w*)\3[^\n]*\n)/, + lookbehind: true, + close: heredocEnd(4, 2) }, + // Strings before comments, else # inside "..." is eaten as a comment. // Each `$` form is matched exactly, never by a greedy run that could also // swallow the ones after it — the old `\$[\w{][^"\n]*` overlapped itself @@ -431,6 +512,15 @@ ).split(' '); G.php = [ + // Heredoc and nowdoc ahead of the comment rules: `#` and `//` are + // ordinary text inside one. The closing word may be indented and may be + // followed by `;` or `,`, which is why the terminator ends at a word + // boundary rather than at end of line. + { cls: 'tk-string', + pattern: /(<<<[ \t]*(['"]?)([A-Za-z_]\w*)\2\r?\n)/, + lookbehind: true, + close: heredocEnd(3, true, '\\b') }, + // Block comments before strings; line comments (// and #) after strings // so "https://..." and "#anchor" inside strings never become comments. { cls: 'tk-comment', pattern: /\/\*[\s\S]*?\*\// }, @@ -732,12 +822,32 @@ const RB_BUILTINS = 'puts print p gets raise lambda proc loop each map select reject reduce new'.split(' '); G.ruby = [ + // Heredocs, and only with an uppercase word: `<<` is also the append + // operator, and `items << thing` must not open one. An uppercase name is + // the convention, and where a constant does follow `<<` the terminator + // line will not exist, so the form declines itself rather than eating the + // file. `<<~` and `<<-` permit an indented terminator; plain `<<` does not. + { cls: 'tk-string', + pattern: /(<<([-~]?)(['"]?)([A-Z_]\w*)\3[^\n]*\n)/, + lookbehind: true, + close: heredocEnd(4, 2) }, + // `=begin` / `=end` blocks come before the strings that come before // everything else. The markers are only special at column zero, so the // anchors here are load-bearing rather than decorative. Without this rule // a documentation block was read as ordinary code — the body's words came // out coloured as function calls and keywords. { cls: 'tk-comment', pattern: /^=begin\b[\s\S]*?^=end.*$/m }, + + // %w[…] %i(…) %q{…} %Q<…>: the delimiter is picked at the call site, so + // the closer is only knowable once the opener has been read, and bracket + // pairs nest. %r is a regex, not a string. The bare `%(…)` form is left + // out on purpose — it cannot be told from the modulo operator without + // parsing, and it is rare enough not to be worth mistaking `a %(b)` for a + // literal. + { cls: 'tk-regex', pattern: /%r([([{<|!\/])/, close: pairedEnd(1) }, + { cls: 'tk-string', pattern: /%[wWiIqQsx]([([{<|!\/])/, close: pairedEnd(1) }, + // Strings must come before comments, else # inside "..." (incl. #{} interpolation) // is eaten as a comment. String bodies stay single-line so an unpaired quote // in a comment can't swallow following lines. @@ -921,6 +1031,13 @@ { cls: 'tk-var', pattern: /[$@][A-Za-z_]\w*/ }, ]}, { cls: 'tk-string', pattern: /'(?:\\.|[^'\\\n])*'/ }, + + // q{…} qq{…} qw{…} qr{…} — ahead of the comment rule, because `#` is + // ordinary text inside one. `/` is excluded as a delimiter for the + // quoting forms: after a bare word it is far more often division. + { cls: 'tk-regex', pattern: /\bqr[ \t]*([([{<|!\/])/, close: pairedEnd(1) }, + { cls: 'tk-string', pattern: /\b(?:qq|qw|q)[ \t]*([([{<|!])/, close: pairedEnd(1) }, + { cls: 'tk-comment', pattern: /#.*/ }, { cls: 'tk-regex', pattern: /((?:=~|!~)\s*)(?:m|s|tr|y)?\/(?:\\.|[^/\n])*\/[a-z]*/, lookbehind: true }, { cls: 'tk-var-builtin', pattern: /\$[_0-9&`'+^!]|\$\^\w|\@ARGV\b|\%ENV\b|\$0\b/ }, @@ -983,6 +1100,14 @@ { cls: 'tk-operator', pattern: /#\{[^}\n]*\}/ }, ]}, { cls: 'tk-string', pattern: /'(?:\\.|[^'\\\n])*'/ }, + + // Sigils ~s{…} ~w[…] ~r/…/, ahead of the comment rule for the same reason + // the strings are. A `"` delimiter is not accepted here: it would end + // `~s"""…"""` at the second quote, and the triple-quote rule above + // already renders that form correctly. + { cls: 'tk-regex', pattern: /~[rR]([([{<|\/'])/, close: pairedEnd(1) }, + { cls: 'tk-string', pattern: /~[a-zA-Z]([([{<|\/'])/, close: pairedEnd(1) }, + { cls: 'tk-comment', pattern: /#.*/ }, { cls: 'tk-decorator', pattern: /@[a-z_]\w*/ }, { cls: 'tk-var-const', pattern: /:[a-z_]\w*[?!]?/ }, diff --git a/docs/development.md b/docs/development.md index f8e15d3..01a4aba 100644 --- a/docs/development.md +++ b/docs/development.md @@ -403,7 +403,10 @@ deliberately deferred). warning when a security-grade Core update ships — security fixes are never silently pinnable. - **jsray-vscode / jsray-terminal**: both public since 2026-09-03 and - 2026-09-05, each with a release carrying an installable build. They now + 2026-09-05. jsray-vscode has a release carrying an installable `.vsix`; + jsray-terminal has no release yet, so the only way in is + `npm i -g github:jsrayorg/jsray-terminal` — a tag exists to cut one from. + They now carry what `jsray-wp` gained on 2026-08-26/27, because all three drift the same way: `tools/sync-core-version.mjs` deriving the README Core badge instead of @@ -412,32 +415,48 @@ deliberately deferred). the right one is what let "Internal test build · no public beta yet" survive the whole public beta; and the plugin version ladder (`0.0.1-beta → 0.0.2-beta`, no counter) rather than Core's. Both bundle Core - `0.0.2-beta.1`; syncing them to `0.0.2-beta.2` is the next step. - -- **Core**: minification is deliberately absent (zero-build); revisit at - public beta. -- **Literal forms with no rule** (found while auditing for beta.5, deferred - because they are absent features rather than wrong spans): heredocs — - `<</` paths, the SRI examples in both READMEs — and a new way for + a release to ship corrupted. A `dist/` a user can read and audit against its + digest is worth more than 9 KB today. Reopen this on a real size complaint + or a substantial growth in Core, not at a version number: "revisit at public + beta" named 2026-07-17, a date that passed, after which the entry sat here + being rediscovered as an open question every planning round. +- **Nested block comments** are what remains of the literal forms audited for + beta.5. Haskell's `{- {- -} -}` closes at the first inner terminator: the + outer comment ends early and the rest of the line is read as code. A flat + pattern cannot count depth, and `close` does not help here either — the + opener carries no information about its own end, which is exactly what a + heredoc's does. It needs the nesting support embedded languages will need + anyway, so the two belong in one round. - **Language detection tuning** (audited for beta.5, deferred): detection is correct on all 22 realistic multi-line samples, and correctly returns empty for prose, digits and single words rather than guessing. On one-line snippets it misreads Go's `func f(x int) int` as Swift and a shell `x=1; echo "$x"` as PHP, and returns empty for short C#, Kotlin and TOML. Retuning the scores changes the relative ranking of all 83 grammars at - once, so it needs its own corpus and belongs with the 0.0.2 engine work — + once, so it needs its own corpus and a beta round of its own — the failure is mild (usually plain text) and only reachable when the caller supplies no language, which the integrations normally do. - **Cosmetic, deliberately left**: a literal prefix that sits outside its string — `@"…"` and `$"…"` in C#, `r"…"` in Rust, `#"…"#` in Swift, `s"…"` in Scala — and the sign in a CSS `-1.5em` or the leading dot in JavaScript's `.5`. The literal is coloured; one character in front of it is not. -- **The string rules themselves** are hand-written once per grammar family - without encoding a terminator model, which is what produced every fix in - beta.5. Rewriting them onto one builder changes the shape of the objects in - `languages`, which is a declared public type, so it waits for the API pass in - 0.0.2. `tests/constructs.test.mjs` is what guards the behaviour until then. +- **The string rules themselves** are still hand-written once per grammar + family without encoding a terminator model, which is what produced every fix + in beta.5. What changed in beta.4 is that a terminator can now be computed + at match time (`close`), which is what the delimited forms needed; the rules + that were already working were left where they are. Migrating them onto one + builder is the remaining half, and it does change the shape of the objects + in `languages` — a declared public type, so it belongs in a beta round of + its own inside the 0.0.x window rather than riding along with unrelated + work. `tests/constructs.test.mjs` is what guards the behaviour until then. diff --git a/docs/development.zh-CN.md b/docs/development.zh-CN.md index 9a27795..4b7dde0 100644 --- a/docs/development.zh-CN.md +++ b/docs/development.zh-CN.md @@ -252,30 +252,42 @@ CI:每个仓库都有 GitHub Actions(Node 18/20/22 矩阵;Core 另验 `dist/` 管线)跑 32 条断言。 - **终端**:`--bg`(整底色绘制)、分页器集成与 `--core ` 覆盖在路线图上。 - **用户侧 Core 锁定**(路线图):平台原生的更新控制已允许用户拒绝某班列车(WP 手动更新、VS Code 按扩展关自动更新 + 装历史版本、npm 版本锁定)。插件内的"Core 更新策略"(跟随/锁定/自定义文件)对 WP 与终端可行,对 VS Code 预览不可行(贡献点路径静态)。硬规则:锁定状态下遇到安全级 Core 更新必须显式警告——安全修复不允许被静默锁死。 -- **jsray-vscode / jsray-terminal**:分别于 2026-09-03 和 2026-09-05 公开,各自 - 的 Release 里都挂着可直接安装的构建。两者现已补齐 `jsray-wp` 在 2026-08-26/27 +- **jsray-vscode / jsray-terminal**:分别于 2026-09-03 和 2026-09-05 公开。 + jsray-vscode 的 Release 里挂着可直接安装的 `.vsix`;jsray-terminal 还没有任何 + Release,唯一的安装途径是 `npm i -g github:jsrayorg/jsray-terminal` —— tag 已 + 经在,随时可以据此切一个。两者现已补齐 `jsray-wp` 在 2026-08-26/27 得到的那套机制 —— 三个集成的失效方式是同一种:`tools/sync-core-version.mjs` 自己推导 README 的 Core 徽章,而不是留给"跑同步的那个人";`check:versions` **双向**校验徽章与阶段 措辞 —— 正是"该出现的词在、不该出现的词也在"这一点,让「内部测试版 · 尚未发布 公开测试版」在整个公开 beta 期间活了下来;以及插件的版本阶梯(`0.0.1-beta → - 0.0.2-beta`,无计数器)而非 Core 的记法。两者均内置 Core `0.0.2-beta.1`, - 同步到 `0.0.2-beta.2` 是下一步。 - -- **Core**:minify 刻意缺席(零构建);公开 beta 时再议。 -- **完全没有规则的字面量形式**(beta.5 审查时发现,因属"缺功能"而非"抢错范围"而推迟): - heredoc —— PHP 的 `<</` 路径、两份 README 里的 + SRI 示例 —— 外加一种新的发布损坏方式。今天,一份用户能读、能对着摘要自行审计的 + `dist/` 比那 9 KB 更值钱。重新评估的条件是出现真实的体积诉求,或 Core 显著变大, + 而不是某个版本号:"公开 beta 时再议"指的是 2026-07-17,那个日子已经过去,此后这一 + 条就杵在这里,每做一轮计划就被当成未决问题重新发现一次。 +- **嵌套块注释**是 beta.5 那批字面量形式里剩下的一条。Haskell 的 `{- {- -} -}` 会在 + 第一个内层终止符处闭合:外层注释提前结束,该行余下部分被当成代码读。扁平模式数不了 + 深度,而 `close` 在这里也帮不上 —— 它的开头不携带任何关于自身结尾的信息,而这恰恰是 + heredoc 的开头所携带的。它需要的嵌套能力与嵌入语言本来就是同一件事,所以两者应当放 + 在同一轮里做。 - **语言检测调优**(beta.5 审查过,推迟):对 22 个真实多行样本全部判对,并且对散文、 纯数字、单个词都正确返回空而不是乱猜。短片段上会把 Go 的 `func f(x int) int` 判成 Swift、把 shell 的 `x=1; echo "$x"` 判成 PHP,对短的 C#、Kotlin、TOML 返回空。 - 重调打分会一次性改变全部 83 个语法的相对排序,需要自己的语料,归入 0.0.2 的引擎 - 工作 —— 它的失败模式温和(多数退化成纯文本),且只在调用方没给语言时才触发,而集成 + 重调打分会一次性改变全部 83 个语法的相对排序,需要自己的语料,也需要自成一轮 + beta —— 它的失败模式温和(多数退化成纯文本),且只在调用方没给语言时才触发,而集成 通常都会给。 - **刻意保留的瑕疵**:落在字符串外面的字面量前缀 —— C# 的 `@"…"` 与 `$"…"`、Rust 的 `r"…"`、Swift 的 `#"…"#`、Scala 的 `s"…"` —— 以及 CSS `-1.5em` 的负号和 JavaScript `.5` 的前导点。字面量本身着色正确,只是前面一个字符没有。 -- **字符串规则本身**按语法族手写、不编码终止符模型,这正是 beta.5 每一条修复的共同 - 成因。把它们收拢到一个构造器上会改变 `languages` 里对象的形状,而那是已声明的公开 - 类型,因此随 0.0.2 的 API 收口一起做。在那之前由 `tests/constructs.test.mjs` 守住行为。 +- **字符串规则本身**仍按语法族手写、不编码终止符模型,这正是 beta.5 每一条修复的共同 + 成因。beta.4 改变的是:终止符现在可以在匹配时算出来(`close`),定界形式需要的正是这个; + 原本就工作正常的规则则原样保留。把它们收拢到一个构造器上是剩下的另一半,那一步确实会 + 改变 `languages` 里对象的形状 —— 那是已声明的公开类型,所以它应当在 0.0.x 窗口内自成 + 一轮 beta,而不是搭在无关的工作上一起走。在那之前由 `tests/constructs.test.mjs` 守住行为。 diff --git a/docs/projects.zh-CN.md b/docs/projects.zh-CN.md index 24fbfb0..8bf5897 100644 --- a/docs/projects.zh-CN.md +++ b/docs/projects.zh-CN.md @@ -59,11 +59,20 @@ renderer.languages -> { [language]: label } Core 的变更通过拷贝或打包 `dist/` 资产流向插件仓库。插件的变更不应要求 Core 变更版本，除非它改动了 Core 的 API 或资产。 +**集成在自己发布时同步 Core，而不是 Core 一发布就同步。** 捆绑的副本追不上实时的 +Core：每个产物都冻结在它构建时的那份快照上 —— `.vsix`、插件 zip，以及 GitHub 为 +tag 附带的源码包，一概如此 —— 所以一个在 Core 发布当下就重新同步的仓库，对齐的只是 +自己的源码，用户装得到的东西一点没变。对齐是"发布"这个动作完成的事。 + +因此 `tools/check-core-freshness.mjs` 日常只作提示、在打包关口才严格：两次发布之间 +落后会被报告出来，而陈旧的引擎打不出包（`--strict`，接在各集成的构建或打包脚本 +里）。唯一不该等下一个功能版的，是安全级别的 Core 发布 —— 为它单独切一次集成发布。 + ## 仓库拆分 -| 仓库 | 交付形态 | 许可 | 状态 | +| 仓库 | 交付形态 | 许可 | 今天从哪里拿到 | |---|---|---|---| -| `jsray` | npm `@jsray/core` | MIT | 已公开 | +| `jsray` | npm `@jsray/core` | MIT | npm —— `@jsray/core@0.0.2-beta.3` | | `jsray-wp` | WordPress.org 插件 | GPLv2 or later | GitHub Release 的 zip | | `jsray-terminal` | npm CLI | MIT | GitHub —— `npm i -g github:jsrayorg/jsray-terminal` | | `jsray-vscode` | VS Code Marketplace | MIT | GitHub Release 的 `.vsix` | diff --git a/docs/versioning.zh-CN.md b/docs/versioning.zh-CN.md index f62b2fb..6e9d22e 100644 --- a/docs/versioning.zh-CN.md +++ b/docs/versioning.zh-CN.md @@ -73,7 +73,8 @@ JSRay Core 是独立的 JavaScript 原生代码渲染内核。平台插件(包 ## npm 发布 -包以 [`@jsray/core`](https://www.npmjs.com/package/@jsray/core) 发布。无作用域的 +包以 [`@jsray/core`](https://www.npmjs.com/package/@jsray/core) 之名、从 `jsray` +这个 npm 账号发布。无作用域的 `jsray` 拿不到 —— npm 判定它与已有的 `js-ray` 过于相似而拒绝 —— 而 `@jsray` 作用域的好处是一次性把整个家族(`@jsray/wp`、`@jsray/vscode`、`@jsray/terminal`) 都占住。 diff --git a/integrity.json b/integrity.json index dd3fe03..cdd5837 100644 --- a/integrity.json +++ b/integrity.json @@ -4,7 +4,7 @@ "algorithm": "sha256", "note": "Base64 SHA-256 digests of the released Core assets. Integrations copy the relevant digest at sync time and verify their bundled snapshot against it.", "files": { - "dist/jsray.js": "sha256-yYHHNdRaH/YE1ohX6y8lwRO3IYpGp4Fn/BMhROBxtGk=", + "dist/jsray.js": "sha256-BIsbeOmZ00Y8FqAaKyrPOUL1b5Q36LC4b52CVYcX2xQ=", "dist/jsray.css": "sha256-6Fuva+aZwCcmltNi2oN+1yWIJoKE+1rLpUxZvD9iutc=", "dist/themes/aurora.css": "sha256-S8X+R8XZNC8WRdHOen839zMkPxVFol6PPTrpZibbeJA=", "dist/themes/default.css": "sha256-vENOigRjwn1hW6wPjdM4tbZ58Ja44FMsa+akavPiMSI=", diff --git a/package.json b/package.json index 9bd097f..bf33bce 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ "scripts": { "build": "sh build.sh", "check:versions": "node tools/check-versions.mjs", + "check:docs-parity": "node tools/check-docs-parity.mjs", "test": "node --test tests/*.mjs", "demo": "echo 'Open demo/index.html in a browser, or serve the project root and visit /demo/'", "sync:integrations": "sh tools/sync-integrations.sh", diff --git a/src/jsray.js b/src/jsray.js index 985f414..51d6cff 100644 --- a/src/jsray.js +++ b/src/jsray.js @@ -23,7 +23,17 @@ * { cls: 'tk-xxx', * pattern: /re/, // must be globalizable; the 'g' flag is forced internally * inside?: rules, // nested grammar (recursively tokenize captured text) - * lookbehind?: true } // capture group 1 is consumed as prefix but not colored + * lookbehind?: true, // capture group 1 is consumed as prefix but not colored + * close?: fn } // pattern matches the opening only; fn finds the end + * + * `close(match, text, from) -> index | -1` exists for the forms whose end is + * not knowable when the rule is written. A heredoc ends at the word its own + * opening line named — `<< last) next.push(piece.slice(last, start)); next.push({ @@ -51,6 +66,9 @@ content: rule.inside ? tokenize(text, rule.inside) : text, }); last = start + text.length; + // The body of a close-delimited form has already been consumed; + // resuming inside it would re-match its own contents. + if (rule.close) re.lastIndex = last; } if (last < piece.length) next.push(piece.slice(last)); } @@ -74,6 +92,60 @@ const G = {}; // grammars + // ---------- runtime terminators ---------- + // Two `close` builders cover every delimited form the grammars below need. + // Both take the opening match and report where the form ends. + + /** + * A heredoc ends at the word its opening line named. `nameGroup` is the + * capture holding that word; `indentGroup` is the capture holding the `-` or + * `~` that permits an indented terminator (pass `true` where the language + * always permits one, as PHP does since 7.3). `trailing` overrides what may + * follow the word on its closing line. + * + * The name is interpolated into a RegExp, which is only safe because every + * opening pattern here restricts it to `[A-Za-z_]\w*` — no metacharacters + * can reach this. + */ + function heredocEnd(nameGroup, indentGroup, trailing) { + return (m, text, from) => { + const name = m[nameGroup]; + if (!name) return -1; + const indented = indentGroup === true ? true : !!m[indentGroup]; + const re = new RegExp( + '^' + (indented ? '[ \\t]*' : '') + name + (trailing || '[ \\t]*$'), + 'm' + ); + const hit = re.exec(text.slice(from)); + return hit ? from + hit.index + hit[0].length : -1; + }; + } + + const CLOSERS = { '(': ')', '[': ']', '{': '}', '<': '>' }; + + /** + * A delimiter-chosen literal — Ruby's `%w[…]`, Perl's `q{…}`, an Elixir + * sigil — ends at whatever closes the character it opened with. Bracket + * pairs nest; a symmetric delimiter such as `%w!…!` cannot, and counting + * depth on one would end the literal at its own opening character. + */ + function pairedEnd(openGroup) { + return (m, text, from) => { + const open = m[openGroup]; + if (!open) return -1; + const close = CLOSERS[open] || open; + const nests = close !== open; + let depth = 1; + for (let i = from; i < text.length; i++) { + const c = text[i]; + if (c === '\\') { i++; continue; } + if (nests && c === open) depth++; + else if (c === close && --depth === 0) return i + 1; + } + return -1; + }; + } + // ---------- shared fragments ---------- const RX = { string1: /"(?:\\.|[^"\\\n])*"/, @@ -386,6 +458,15 @@ 'docker kubectl python python3 pip pip3 ruby go cargo make brew apt yum'; G.shell = [ + // Heredocs first: their body may hold quotes and `#`, and every rule + // after this one would claim those. The opening line is group 1 and is + // consumed as an uncolored prefix, so `cat < out.txt` keeps its + // redirect as shell rather than dragging it into the literal. + { cls: 'tk-string', + pattern: /(<<(-?)[ \t]*(['"]?)([A-Za-z_]\w*)\3[^\n]*\n)/, + lookbehind: true, + close: heredocEnd(4, 2) }, + // Strings before comments, else # inside "..." is eaten as a comment. // Each `$` form is matched exactly, never by a greedy run that could also // swallow the ones after it — the old `\$[\w{][^"\n]*` overlapped itself @@ -431,6 +512,15 @@ ).split(' '); G.php = [ + // Heredoc and nowdoc ahead of the comment rules: `#` and `//` are + // ordinary text inside one. The closing word may be indented and may be + // followed by `;` or `,`, which is why the terminator ends at a word + // boundary rather than at end of line. + { cls: 'tk-string', + pattern: /(<<<[ \t]*(['"]?)([A-Za-z_]\w*)\2\r?\n)/, + lookbehind: true, + close: heredocEnd(3, true, '\\b') }, + // Block comments before strings; line comments (// and #) after strings // so "https://..." and "#anchor" inside strings never become comments. { cls: 'tk-comment', pattern: /\/\*[\s\S]*?\*\// }, @@ -732,12 +822,32 @@ const RB_BUILTINS = 'puts print p gets raise lambda proc loop each map select reject reduce new'.split(' '); G.ruby = [ + // Heredocs, and only with an uppercase word: `<<` is also the append + // operator, and `items << thing` must not open one. An uppercase name is + // the convention, and where a constant does follow `<<` the terminator + // line will not exist, so the form declines itself rather than eating the + // file. `<<~` and `<<-` permit an indented terminator; plain `<<` does not. + { cls: 'tk-string', + pattern: /(<<([-~]?)(['"]?)([A-Z_]\w*)\3[^\n]*\n)/, + lookbehind: true, + close: heredocEnd(4, 2) }, + // `=begin` / `=end` blocks come before the strings that come before // everything else. The markers are only special at column zero, so the // anchors here are load-bearing rather than decorative. Without this rule // a documentation block was read as ordinary code — the body's words came // out coloured as function calls and keywords. { cls: 'tk-comment', pattern: /^=begin\b[\s\S]*?^=end.*$/m }, + + // %w[…] %i(…) %q{…} %Q<…>: the delimiter is picked at the call site, so + // the closer is only knowable once the opener has been read, and bracket + // pairs nest. %r is a regex, not a string. The bare `%(…)` form is left + // out on purpose — it cannot be told from the modulo operator without + // parsing, and it is rare enough not to be worth mistaking `a %(b)` for a + // literal. + { cls: 'tk-regex', pattern: /%r([([{<|!\/])/, close: pairedEnd(1) }, + { cls: 'tk-string', pattern: /%[wWiIqQsx]([([{<|!\/])/, close: pairedEnd(1) }, + // Strings must come before comments, else # inside "..." (incl. #{} interpolation) // is eaten as a comment. String bodies stay single-line so an unpaired quote // in a comment can't swallow following lines. @@ -921,6 +1031,13 @@ { cls: 'tk-var', pattern: /[$@][A-Za-z_]\w*/ }, ]}, { cls: 'tk-string', pattern: /'(?:\\.|[^'\\\n])*'/ }, + + // q{…} qq{…} qw{…} qr{…} — ahead of the comment rule, because `#` is + // ordinary text inside one. `/` is excluded as a delimiter for the + // quoting forms: after a bare word it is far more often division. + { cls: 'tk-regex', pattern: /\bqr[ \t]*([([{<|!\/])/, close: pairedEnd(1) }, + { cls: 'tk-string', pattern: /\b(?:qq|qw|q)[ \t]*([([{<|!])/, close: pairedEnd(1) }, + { cls: 'tk-comment', pattern: /#.*/ }, { cls: 'tk-regex', pattern: /((?:=~|!~)\s*)(?:m|s|tr|y)?\/(?:\\.|[^/\n])*\/[a-z]*/, lookbehind: true }, { cls: 'tk-var-builtin', pattern: /\$[_0-9&`'+^!]|\$\^\w|\@ARGV\b|\%ENV\b|\$0\b/ }, @@ -983,6 +1100,14 @@ { cls: 'tk-operator', pattern: /#\{[^}\n]*\}/ }, ]}, { cls: 'tk-string', pattern: /'(?:\\.|[^'\\\n])*'/ }, + + // Sigils ~s{…} ~w[…] ~r/…/, ahead of the comment rule for the same reason + // the strings are. A `"` delimiter is not accepted here: it would end + // `~s"""…"""` at the second quote, and the triple-quote rule above + // already renders that form correctly. + { cls: 'tk-regex', pattern: /~[rR]([([{<|\/'])/, close: pairedEnd(1) }, + { cls: 'tk-string', pattern: /~[a-zA-Z]([([{<|\/'])/, close: pairedEnd(1) }, + { cls: 'tk-comment', pattern: /#.*/ }, { cls: 'tk-decorator', pattern: /@[a-z_]\w*/ }, { cls: 'tk-var-const', pattern: /:[a-z_]\w*[?!]?/ }, diff --git a/tests/constructs.test.mjs b/tests/constructs.test.mjs index 8cab513..32d83c8 100644 --- a/tests/constructs.test.mjs +++ b/tests/constructs.test.mjs @@ -336,6 +336,13 @@ test('every new string form stays linear on pathological input', () => { ['cpp', '0x1p'.repeat(10000)], // hex exponent storm ['go', '1' + '_'.repeat(20000)], // separator storm ['ruby', '=begin\n'.repeat(5000)], // unterminated block comments + // Runtime terminators scan forward from each opening. An opening that + // never closes makes that scan reach the end of the input, so a file of + // nothing but openings is the shape that would expose a quadratic one. + ['shell', 'cat < { ); } }); + +// ── Forms whose end is decided at runtime ────────────────────────────────── +// A heredoc ends at the word its own opening named; a %w[…] ends at the +// bracket matching its opener. Neither is expressible as a RegExp, so both +// rendered as ordinary code until `close` existed. The cases that matter are +// the ones where the body holds characters another rule wants: `#`, `//`, a +// quote, a brace. + +test('shell: a heredoc body is one string, and its # is not a comment', () => { + const code = 'cat < { + const code = 'cat < out.txt\nbody\nEOF'; + + // The opening line is consumed as an uncolored prefix precisely so that + // `> out.txt` is not dragged into the literal. + notSwallowed(code, 'shell', 'string', '> out.txt'); + token(code, 'shell', 'operator', '>'); +}); + +test('shell: <<- permits an indented terminator, << does not', () => { + token('cat <<-END\n\tbody\n\tEND\n', 'shell', 'string', '\tbody\n\tEND'); + + // Plain `<<` requires the word at column zero; an indented one is not a + // terminator, and with none present the form declines rather than running on. + const indented = 'cat < t.type === 'tk-string'); + assert.equal(strings.length, 0, `expected no string, got ${JSON.stringify(strings)}`); +}); + +test('shell: an arithmetic shift does not open a heredoc', () => { + const code = 'echo $(( a << 2 ))'; + const strings = leaves(JSRay.tokenize(code, 'shell')).filter((t) => t.type === 'tk-string'); + + assert.equal(strings.length, 0, `<< opened a literal: ${JSON.stringify(strings)}`); +}); + +test('PHP: a heredoc holds // and closes on an indented word', () => { + const code = ' { + const code = " { + token('sql = <<~SQL\n SELECT 1\nSQL\n', 'ruby', 'string', ' SELECT 1\nSQL'); + + // `items << thing` is the append operator and must stay one. + const append = 'items << thing'; + const strings = leaves(JSRay.tokenize(append, 'ruby')).filter((t) => t.type === 'tk-string'); + assert.equal(strings.length, 0, `append became a literal: ${JSON.stringify(strings)}`); +}); + +test('Ruby: %w and %q close on the matching bracket, and nest', () => { + token('a = %w[one two three]', 'ruby', 'string', '%w[one two three]'); + token('c = %q{outer {inner} still}', 'ruby', 'string', '%q{outer {inner} still}'); + token('d = %r{^\\d+$}', 'ruby', 'regex', '%r{^\\d+$}'); +}); + +test('Ruby: modulo is not a percent literal', () => { + const code = 'e = x % y'; + const strings = leaves(JSRay.tokenize(code, 'ruby')).filter((t) => t.type === 'tk-string'); + + assert.equal(strings.length, 0, `modulo became a literal: ${JSON.stringify(strings)}`); +}); + +test('Perl: q{} holds a # without becoming a comment', () => { + const code = 'my $s = q{hello # not comment};'; + + token(code, 'perl', 'string', 'q{hello # not comment}'); + notSwallowed(code, 'perl', 'comment', 'not comment'); + token('my @w = qw(a b c);', 'perl', 'string', 'qw(a b c)'); + token('my $r = qr{^\\d+};', 'perl', 'regex', 'qr{^\\d+}'); +}); + +test('Elixir: a sigil carries its own delimiters', () => { + token('a = ~w[one two]', 'elixir', 'string', '~w[one two]'); + token('b = ~r/^\\d+/', 'elixir', 'regex', '~r/^\\d+/'); + + const code = 'c = ~s{hi # not comment}'; + token(code, 'elixir', 'string', '~s{hi # not comment}'); + notSwallowed(code, 'elixir', 'comment', 'not comment'); +}); diff --git a/tools/check-docs-parity.mjs b/tools/check-docs-parity.mjs new file mode 100644 index 0000000..255b89d --- /dev/null +++ b/tools/check-docs-parity.mjs @@ -0,0 +1,108 @@ +#!/usr/bin/env node +// Every doc in this repository is written twice. A change that lands in one +// language and not the other does not break a build, does not fail a test, and +// reads perfectly well — in the language you happen to be reading. The Core +// sync rule sat in `docs/projects.md` for a day with no Chinese counterpart; +// the Chinese repository table answered a different question than the English +// one ("status" against "where you get it today") while the sentence under +// both explained the English one. +// +// So this compares the parts of a document that cannot legitimately differ +// between translations: version numbers, file paths, package specifiers, and +// links. Prose differs, sentence counts differ, and a placeholder is expected +// to be translated (`` / `<版本>`) — none of that is checked. What is +// checked is presence: something named in one language and nowhere in the +// other. +import { readFileSync, readdirSync, statSync } from 'node:fs'; +import { join } from 'node:path'; + +const SKIP = new Set(['node_modules', '_site', '.git', '.vscode-test', 'dist']); +const fail = []; + +function walk(dir, out = []) { + for (const entry of readdirSync(dir)) { + if (SKIP.has(entry)) continue; + const full = join(dir, entry); + if (statSync(full).isDirectory()) walk(full, out); + else if (entry.endsWith('.md')) out.push(full); + } + return out; +} + +// A translated placeholder is still the same placeholder. +const normalise = (s) => s.replace(/<[^>]*>/g, '<>').trim(); + +// Anything that names a file, a directory, or a package — the things a reader +// is meant to go and find. A backticked fragment of prose or a translated +// example (`# Heading` / `# 标题`) is not one of these and is left alone. +const PATHISH = /\/|\.(mjs|cjs|js|json|css|sh|php|md|vsix|zip|html|ts|yml|yaml)\b/; + +function invariants(path) { + const text = readFileSync(path, 'utf8'); + const code = new Set(); + for (const [, span] of text.matchAll(/`([^`\n]+)`/g)) { + const value = normalise(span); + if (PATHISH.test(value)) code.add(value); + } + const versions = new Set( + [...text.matchAll(/\b\d+\.\d+\.\d+(?:-[A-Za-z0-9.]+)?\b/g)].map((m) => m[0]) + ); + const links = new Set( + [...text.matchAll(/https?:\/\/[^\s)\]`"'>]+/g)] + .map((m) => normalise(m[0].replace(/[.,;:，。、]+$/, ''))) + // Badge URLs carry their own translated label, by design. + .filter((url) => !url.includes('img.shields.io')) + ); + // Headings are counted outside fenced blocks only: a shell comment reading + // `# Install` is not a section, and it is translated. + const prose = text.replace(/^```[\s\S]*?^```/gm, ''); + const headings = [...prose.matchAll(/^(#{1,6})\s+\S/gm)].map((m) => m[1].length); + const tableRows = (text.match(/^\s*\|.*\|\s*$/gm) || []).length; + return { code, versions, links, headings, tableRows }; +} + +function compare(label, enPath, zhPath, enSet, zhSet) { + for (const [side, missing, from] of [ + ['中文', [...enSet].filter((v) => !zhSet.has(v)), enPath], + ['English', [...zhSet].filter((v) => !enSet.has(v)), zhPath], + ]) { + for (const value of missing) { + fail.push(`${from} names ${label} ${JSON.stringify(value)}, its ${side} counterpart does not`); + } + } +} + +const docs = walk('.'); +const pairs = docs + .filter((p) => p.endsWith('.zh-CN.md')) + .map((zh) => [`${zh.slice(0, -'.zh-CN.md'.length)}.md`, zh]) + .filter(([en]) => docs.includes(en)); + +if (!pairs.length) { + console.error('No translated document pairs found — this check would pass vacuously.'); + process.exit(1); +} + +for (const [en, zh] of pairs) { + const a = invariants(en); + const b = invariants(zh); + compare('the version', en, zh, a.versions, b.versions); + compare('the path', en, zh, a.code, b.code); + compare('the link', en, zh, a.links, b.links); + // A section present in one language and not the other, or a table that grew + // a row on one side only. + if (a.headings.join() !== b.headings.join()) { + fail.push(`${en} and ${zh} do not have the same section structure: heading levels ${a.headings.join('-')} against ${b.headings.join('-')}`); + } + if (a.tableRows !== b.tableRows) { + fail.push(`${en} has ${a.tableRows} table rows, ${zh} has ${b.tableRows}`); + } +} + +if (fail.length) { + console.error('Translated documents have drifted:'); + for (const message of fail) console.error(`- ${message}`); + process.exit(1); +} + +console.log(`docs parity ok: ${pairs.length} translated pairs agree`); diff --git a/tools/check-versions.mjs b/tools/check-versions.mjs index e045482..8cf7b70 100644 --- a/tools/check-versions.mjs +++ b/tools/check-versions.mjs @@ -70,6 +70,14 @@ includes('docs/versioning.zh-CN.md', `当前版本:\`${version}\``); includes('docs/projects.md', 'JSRay Core'); includes('docs/projects.zh-CN.md', 'JSRay Core'); +// The repository table's last column tells a reader what to install today, and +// it is the one place in the docs that quotes a full npm specifier. A parity +// check catches it going missing from one language; nothing catches both +// languages naming the same stale release. +for (const path of ['docs/projects.md', 'docs/projects.zh-CN.md']) { + includes(path, `@jsray/core@${version}`, `the installable specifier @jsray/core@${version}`); +} + // The supported-versions table is a promise to anyone deciding whether to // report privately. It sat on beta.1 through the whole beta.2 cycle. includes('SECURITY.md', `| ${version} | ✅`, `${version} in the supported-versions table`); diff --git a/types/jsray.d.ts b/types/jsray.d.ts index efdddfb..3dce653 100644 --- a/types/jsray.d.ts +++ b/types/jsray.d.ts @@ -11,6 +11,14 @@ declare namespace JSRay { pattern: RegExp; inside?: GrammarRule[]; lookbehind?: boolean; + /** + * For forms whose end is not knowable when the rule is written — a + * heredoc ends at the word its own opening line named, `%w[…]` at the + * bracket matching its opener. `pattern` matches the opening; this + * returns the index just past the end of the whole form, or -1 when no + * terminator is present, which leaves the opening to the rules behind it. + */ + close?: (match: RegExpExecArray, text: string, from: number) => number; } type Grammar = GrammarRule[];