feat(spec)!: publish the two named refinement patterns the runtime already enforces

.changeset/18670-project-expressible-refinements.md

@@ -0,0 +1,22 @@
+---
+"@objectstack/spec": minor
+---
+
+**BREAKING (published artifact narrows)** — `packages/spec/json-schema/**` now states two of the rules it used to leave entirely to the runtime, so a validator reading the published files stops answering PASS on metadata the platform then refuses (#18670 item 2).
+
+Clause-②: yes (narrowing)
+
+`z.toJSONSchema()` has no arm for a `custom` check: on zod 4.4.3 a plain record, the same record with a `.refine()`, and the same record with an **aborting** `.refine()` all project byte-identically. Every rule written as a refinement was therefore enforced by the runtime and absent from the published file — the direction in which an author's, or an AI's, validator says yes right up to the moment the platform says no.
+
+Two named patterns now project, and only those two:
+
+- **at least one of these keys is present** — emitted as `anyOf` of one `required` per key. `shared/Expression.json` states the source-or-ast rule, so `{ "dialect": "cel" }` is refused by the published file exactly as the runtime already refused it.
+- **a string with at least one non-whitespace character** — emitted as `minLength: 1` plus the pattern `\S`. Every evaluated and typed expression slot states it, so a whitespace-only `source` is refused at the door.
+
+**⛔ Not a behaviour change, and no document the runtime accepts becomes refused.** Both patterns are EXACT rather than approximate: a key absent from a JSON object is the only way for its value to read `undefined`, and `String.prototype.trim` removes exactly the ECMA-262 whitespace set that `\S` is the complement of. Both equalities are pinned over their whole input space in `packages/spec/scripts/refinement-projection.test.ts`, including every ECMA-262 WhiteSpace and LineTerminator code point. No refinement was weakened, removed or added; the runtime accepts and refuses exactly what it did before.
+
+**The list is CLOSED.** `packages/spec/src/shared/refinement-projection.ts` declares the vocabulary and builds each predicate from its own declaration, so the rule the runtime enforces and the keywords the file publishes cannot name different things. A refinement outside that list stays unprojected and keeps its `x-dropped-refinements` annotation. Adding an arm is a public-contract decision with its own measurement, never a refactor — and ⛔ never an open-ended zod-to-JSON-Schema translator over the whole population.
+
+**Proof of work, in the shrink-only ledger.** `packages/spec/dropped-refinements.baseline.json` reads 201 published schemas / 553 dropped sites, from 246 / 750: 45 rows deleted, 75 rows shrunk, 197 sites closed, zero sites added anywhere. The generator now prints the closed population per pattern on every run (137 `required-one-of`, 60 `non-blank-string`), and reports a site that projects with no declared pattern on its own line.
+
+<!-- adr-0087: not-required (no-migration-prescription) Nothing an author can write is removed, renamed or re-spelled: no spec key, no export and no config field changes, and the accepted set of metadata documents is byte-for-byte what it was. What changed is a machine-readable DECLARATION catching up with the runtime it always described, so there is nothing for `objectstack migrate meta` to rewrite and no stored representation to convert. -->

packages/spec/scripts/build-schemas.ts

@@ -45,6 +45,11 @@ import {
   projectByPruningUnionBranches,
   type PrunedBranch,
 } from './lib/union-branch-projection';
+// The closed list of refinements this generator DOES publish (#18670 item 2).
+// The ratchet below measures against this same override, so a rule the list
+// emits leaves the ledger and a rule it does not emit stays in it — see the
+// module header for why the two halves must not be read against each other.
+import { refinementProjectionOverride } from './lib/refinement-projection';
 // The dropped-refinement ratchet (#18670). The mirror image of the branch
 // pruning above, and deliberately its own module for the same reason: the
 // pruner guards a projection NARROWER than the Zod type, this one the direction
@@ -495,6 +500,7 @@ for (const [namespaceName, namespaceExports] of Object.entries(Protocol)) {
           try {
             jsonSchema = z.toJSONSchema(value, {
               target: 'draft-2020-12',
+              override: refinementProjectionOverride,
             }) as Record<string, unknown>;
           } catch (outputError) {
             if (!isKnownUnsupported(outputError)) throw outputError;
@@ -503,6 +509,7 @@ for (const [namespaceName, namespaceExports] of Object.entries(Protocol)) {
               jsonSchema = z.toJSONSchema(value, {
                 target: 'draft-2020-12',
                 io: 'input',
+                override: refinementProjectionOverride,
               }) as Record<string, unknown>;
             } catch (inputError) {
               if (!isKnownUnsupported(inputError)) throw inputError;
@@ -519,7 +526,10 @@ for (const [namespaceName, namespaceExports] of Object.entries(Protocol)) {
               // then re-thrown with the message Zod produced, so this attempt
               // can never change WHY an export is skipped, and so never the
               // `cause` recorded for it in unemitted-schemas.baseline.json.
-              const projected = projectByPruningUnionBranches(value, { target: 'draft-2020-12' });
+              const projected = projectByPruningUnionBranches(value, {
+                target: 'draft-2020-12',
+                override: refinementProjectionOverride,
+              });
               if (!projected) throw inputError;
               jsonSchema = projected.schema;
               io = projected.io;
@@ -550,15 +560,21 @@ for (const [namespaceName, namespaceExports] of Object.entries(Protocol)) {
             branchPrunedProjections.push({ namespace: namespaceName, exportKey: key, pruned: prunedBranches });
           }
 
-          // The refinements this projection DROPPED (#18670), named on the
+          // The refinements this projection STILL drops (#18670), named on the
           // artifact for the same reason `x-unprojectable-branches` is: a reader
           // of this file — an author, a reference page, an AI validating a
           // document against it — can otherwise not tell that the contract
           // underneath carries rules this file does not state. It is an
           // annotation and nothing more: `x-` keywords are ignored by every
           // validator, so the set of documents this schema ACCEPTS is unchanged
-          // by it. Narrowing the published shape to match the Zod type is a
-          // public-contract change and is deliberately NOT done here.
+          // by it.
+          //
+          // What DID narrow (#18670 item 2) is the closed list in
+          // `src/shared/refinement-projection.ts`, applied by the `override`
+          // above: a refinement declared through it is emitted as real
+          // keywords, so it never reaches `census.dropped` and never reaches
+          // this annotation. ⛔ The two are exclusive by construction — a site
+          // cannot be both stated and annotated as unstated.
           const census = collectDroppedRefinements(`${categorySlug}/${schemaName}`, value);
           refinementCensus.push(census);
           if (census.dropped.length > 0) {
@@ -3421,9 +3437,16 @@ if (unemittedSkips.length > 0) {
 // no. Measured on this tree at the change that added this block: 682 refinement
 // sites across 237 published schemas, zero of which projected anything.
 //
-// ⛔ It does NOT narrow any published shape and does not touch the refinements
-// themselves — the runtime rule is correct. It makes the population declared,
-// so the next one arrives as a line in a diff instead of as nothing at all.
+// ⛔ This ratchet still narrows nothing by itself and touches no refinement —
+// the runtime rule is correct. It makes the remaining population declared, so
+// the next gap arrives as a line in a diff instead of as nothing at all.
+//
+// The narrowing is the CLOSED list in `src/shared/refinement-projection.ts`
+// (#18670 item 2), emitted by the `override` this generator passes to every
+// projection. It and this ratchet compose in one direction: a site the list
+// emits is `projected` and its ledger row is deleted in the same PR; every
+// other site is `dropped` and stays declared. So the ledger is shrink-only in
+// the strong sense — a repair is the only thing that shortens it.
 const droppedRefinementsBaseline = readDroppedRefinementsBaseline(PKG_DIR);
 if (!droppedRefinementsBaseline) {
   console.error(`\n❌ ${DROPPED_REFINEMENTS_BASELINE_FILE} is missing — it is a committed, hand-edited ledger (#18670).`);
@@ -3544,17 +3567,42 @@ if (droppedSiteTotal > 0) {
       `RUNTIME and not the published JSON Schema — all declared in ${DROPPED_REFINEMENTS_BASELINE_FILE} (#18670).`,
   );
   console.log(
-    `     The published files are therefore WIDER than the Zod types they are generated from:\n` +
-      `     a document one of them accepts can still be refused at parse time. Each affected file\n` +
-      `     names its own sites as \`x-dropped-refinements\`. Narrowing the published shape to match\n` +
-      `     is a public-contract change and is NOT what this ratchet does.`,
+    `     Those files are therefore still WIDER than the Zod types they are generated from:\n` +
+      `     a document one of them accepts can be refused at parse time. Each affected file names\n` +
+      `     its own remaining sites as \`x-dropped-refinements\`. Closing one means teaching the\n` +
+      `     CLOSED list in src/shared/refinement-projection.ts a NAMED pattern — ⛔ never deleting\n` +
+      `     the refinement, and ⛔ never an open-ended translator over the whole population.`,
   );
   console.log(
     `     Also measured this run: ${projectedSiteTotal} refinement site(s) DID reach the file, ` +
       `${undecidableSiteTotal} had no JSON form on either side to compare.`,
   );
 }
 
+// Which projected sites got there through which arm of the closed list (#18670
+// item 2). Printed per pattern rather than as one total, for the reason the
+// ledger records sites rather than a count: a total cannot tell "one arm stopped
+// emitting" from "somebody deleted a refinement", and the two have opposite
+// remedies. A site that projects with NO declared pattern is reported on its own
+// line — it means zod started emitting something by itself, which is news.
+if (projectedSiteTotal > 0) {
+  const byPattern = new Map<string, number>();
+  for (const entry of refinementCensus) {
+    for (const site of entry.projected) {
+      const key = site.declaredPatterns.length > 0
+        ? site.declaredPatterns.join('+')
+        : 'UNDECLARED — zod projected this on its own';
+      byPattern.set(key, (byPattern.get(key) ?? 0) + 1);
+    }
+  }
+  console.log(
+    `\n📣 ${projectedSiteTotal} refinement site(s) DO reach the published JSON Schema, by declared pattern:`,
+  );
+  for (const [pattern, n] of [...byPattern].sort((a, b) => b[1] - a[1])) {
+    console.log(`     ${String(n).padStart(4)}  ${pattern}`);
+  }
+}
+
 // ─── Generate Bundled Schema ─────────────────────────────────────────
 // Single-file bundled schema containing all generated schemas for IDE autocomplete
 

packages/spec/scripts/dropped-refinements.test.ts

@@ -225,7 +225,7 @@ describe('the differential isolates the refinement, not the node', () => {
 });
 
 describe('the ratchet adjudicates against the ledger', () => {
-  const site = (path: string) => ({ path, nodeType: 'string', count: 1, aborting: false, verdict: 'dropped' as const });
+  const site = (path: string) => ({ path, nodeType: 'string', count: 1, aborting: false, verdict: 'dropped' as const, declaredPatterns: [] });
   const census = (defKey: string, paths: string[]) => ({
     defKey,
     dropped: paths.map(site),

packages/spec/scripts/lib/dropped-refinements.ts

@@ -30,13 +30,17 @@
  *
  * It is a **visibility ratchet**, exactly like `unemitted-schemas.ts`: it
  * reports what is already true and refuses GROWTH of the population. It
- * narrows no published shape, removes no refinement, and changes nothing about
- * what the runtime accepts — the baseline is anchored to the tree as it stands,
- * so it is green the moment it lands.
+ * narrows nothing itself and removes no refinement — the baseline is anchored
+ * to the tree as it stands.
  *
- * It is **not** a fix. Teaching the projection to emit what a refinement
- * constrains, or declaring the published artifact a floor, both change the
- * published contract and are a maintainer's decision, not a generator's.
+ * The **fix** is a separate module and a separate decision, taken for #18670
+ * item 2: `refinement-projection.ts` publishes a CLOSED, named list of
+ * refinements, and this module now measures against that same projection (see
+ * `projectOrNull`). So the two halves compose in one direction only — a rule the
+ * closed list emits reads `projected` here and its ledger row goes; every other
+ * rule reads `dropped` and stays declared. ⛔ Neither half may be used to talk
+ * the other out of its reading: a site is `dropped` because THIS build's file
+ * says nothing about it, not because a PR body says the projection handles it.
  *
  * ## Why the verdict is MEASURED per instance, never assumed
  *
@@ -77,18 +81,30 @@
 import fs from 'fs';
 import path from 'path';
 import { z } from 'zod';
+// The closed list of refinements that DO reach the published file (#18670 item
+// 2), and the zod-check primitives both halves turn on. Imported rather than
+// re-derived because the differential below is only a statement about the real
+// published artifact if it runs the generator's own projection — see that
+// module's header.
+import {
+  CUSTOM_CHECK_KIND,
+  checkKindOf,
+  customChecksOf,
+  projectableRefinementsOf,
+  refinementProjectionOverride,
+  zodDefOf,
+} from './refinement-projection';
 
 /** File name of the committed ledger, resolved against the package root. */
 export const DROPPED_REFINEMENTS_BASELINE_FILE = 'dropped-refinements.baseline.json';
 
 /**
- * The check kind `.refine()`, `.superRefine()` and a bare `.check(fn)` all
- * compile to in zod 4. Named once here because it is the ONE string this whole
- * instrument turns on: a zod release that renames it must fail as "no
- * refinements found anywhere" against the lit control in the census, never as a
- * quiet zero.
+ * Re-exported so this module stays the one import for the census vocabulary.
+ * Its declaration — and the argument for why it is the ONE string this whole
+ * instrument turns on — lives in `refinement-projection.ts`, beside the two
+ * other readers of a zod check.
  */
-export const CUSTOM_CHECK_KIND = 'custom';
+export { CUSTOM_CHECK_KIND };
 
 /** How deep the graph walk goes before it stops descending. */
 const MAX_DEPTH = 14;
@@ -122,6 +138,17 @@ export interface RefinementSite {
    * comparison has no two sides. Reported, never counted as a gap.
    */
   readonly verdict: 'dropped' | 'projected' | 'undecidable';
+  /**
+   * Which arms of the closed projectable list (#18670 item 2) this node's
+   * refinements were DECLARED as, in declaration order — empty for every rule
+   * outside that list, which is what keeps it `dropped`.
+   *
+   * Reported so the generator can say which PATTERN closed a site rather than
+   * only that the count moved: a projection arm that silently stops emitting
+   * shows up here as a site that went back to `dropped` with its pattern still
+   * named, which reads differently from a refinement somebody deleted.
+   */
+  readonly declaredPatterns: readonly string[];
 }
 
 /** Every refinement site under one published schema. */
@@ -160,28 +187,11 @@ export interface DroppedRefinementsBaseline {
   readonly entries: Readonly<Record<string, DroppedRefinementsEntry>>;
 }
 
-function defOf(schema: z.ZodType): Record<string, unknown> | null {
-  const def = (schema as unknown as { _zod?: { def?: unknown } })._zod?.def;
-  return def && typeof def === 'object' ? (def as Record<string, unknown>) : null;
-}
-
-function checkKind(check: unknown): string | null {
-  const kind = (check as { _zod?: { def?: { check?: unknown } } } | null)?._zod?.def?.check;
-  return typeof kind === 'string' ? kind : null;
-}
-
 function checkAborts(check: unknown): boolean {
   const def = (check as { _zod?: { def?: Record<string, unknown> } } | null)?._zod?.def;
   return def?.abort === true || def?.fatal === true;
 }
 
-function customChecksOf(schema: z.ZodType): unknown[] {
-  const def = defOf(schema);
-  const checks = def?.checks;
-  if (!Array.isArray(checks)) return [];
-  return checks.filter((c) => checkKind(c) === CUSTOM_CHECK_KIND);
-}
-
 /**
  * The same node with its `custom` checks removed.
  *
@@ -196,9 +206,9 @@ function customChecksOf(schema: z.ZodType): unknown[] {
  * question.
  */
 function withoutCustomChecks(schema: z.ZodType): z.ZodType | null {
-  const def = defOf(schema);
+  const def = zodDefOf(schema);
   if (!def || !Array.isArray(def.checks)) return null;
-  const kept = def.checks.filter((c) => checkKind(c) !== CUSTOM_CHECK_KIND);
+  const kept = def.checks.filter((c) => checkKindOf(c) !== CUSTOM_CHECK_KIND);
   const cloneable = schema as unknown as { clone?: (d: unknown) => z.ZodType };
   if (typeof cloneable.clone !== 'function') return null;
   const stripped = cloneable.clone({ ...def, checks: kept });
@@ -207,11 +217,24 @@ function withoutCustomChecks(schema: z.ZodType): z.ZodType | null {
   return stripped;
 }
 
-/** `toJSONSchema` in the generator's own io ladder, or `null` when neither side has a JSON form. */
+/**
+ * `toJSONSchema` in the generator's own io ladder, or `null` when neither side
+ * has a JSON form.
+ *
+ * ⭐ It passes the generator's `override` (#18670 item 2). Without it this
+ * function would measure a projection nothing publishes: a node whose rule the
+ * closed list DOES emit would read byte-identical on both sides of the
+ * differential and stay in the ledger for ever, and the shrink-only ledger's
+ * whole use — a row deletion is the observable proof a site closed — would be
+ * unreachable. With it, `dropped` means "this build's own published file states
+ * nothing about this rule".
+ */
 function projectOrNull(schema: z.ZodType): string | null {
   for (const io of ['output', 'input'] as const) {
     try {
-      return JSON.stringify(z.toJSONSchema(schema, { target: 'draft-2020-12', io }));
+      return JSON.stringify(
+        z.toJSONSchema(schema, { target: 'draft-2020-12', io, override: refinementProjectionOverride }),
+      );
     } catch {
       // Try the other direction — the generator does the same, for the same reason.
     }
@@ -241,7 +264,7 @@ function verdictFor(schema: z.ZodType): RefinementSite['verdict'] {
  */
 function labelledChildren(schema: z.ZodType): Array<{ label: string; schema: z.ZodType }> {
   const out: Array<{ label: string; schema: z.ZodType }> = [];
-  const def = defOf(schema);
+  const def = zodDefOf(schema);
   if (!def) return out;
   const seen = new Set<unknown>();
   const walk = (label: string, value: unknown): void => {
@@ -375,10 +398,11 @@ export function collectDroppedRefinements(defKey: string, root: z.ZodType): Refi
     if (customs.length > 0) {
       const site: RefinementSite = {
         path: readablePath(path),
-        nodeType: String(defOf(schema)?.type ?? 'unknown'),
+        nodeType: String(zodDefOf(schema)?.type ?? 'unknown'),
         count: customs.length,
         aborting: customs.some(checkAborts),
         verdict: verdictFor(schema),
+        declaredPatterns: projectableRefinementsOf(schema).map((declared) => declared.pattern),
       };
       if (site.verdict === 'dropped') dropped.push(site);
       else if (site.verdict === 'projected') projected.push(site);