docs(skills): ship at least three json evals per published skill in the objectui fixture shape

scripts/check-skills-token-ratchet.mjs

@@ -472,6 +472,33 @@ export const CEILINGS = new Map([
   ['skills/objectstack-ui/rules/navigation.md', 2273],
   ['skills/objectstack-ui/rules/pages.md', 5501],
 
+  // ── 2026-09-22 evals fixtures — decision batch #213 item 2, PR #19721 ────
+  // Every published skill gained `evals/*.json` in the objectui fixture shape.
+  // The maintainer ruled the work in batch #213, verbatim and untranslated:
+  // 「1 2 4 同意」 on item 2 「objectstack 各 skill 先补 3 题 evals(复用 objectui 的
+  // json 形状),先于第 1 项,好量化『切了没变差』」. The eleven files landed
+  // unpriced on 02b931d2 and this gate redded on every one of them, as the
+  // header says it must; the maintainer's follow-up word, recorded as issue
+  // comment 5776586573 (2026-09-22), verbatim and untranslated —
+  // 「天花板行按照你的意见就行」 — adopts pricing them in. So: one row per file,
+  // each an INITIAL measurement taken on that same tree in the convention
+  // above (re-measured unchanged on the commit that adds the rows) and pinned
+  // AT it — zero headroom, on the same terms as the #12392 extension rows and
+  // the #14296 「NEW FILE … pinned at its landed count」 rows. Nothing else
+  // moves: no existing row is raised, the population is unchanged, and
+  // `evals/**` stays priced.
+  ['skills/objectstack-ai/evals/skills-tools-knowledge.json', 1074],
+  ['skills/objectstack-api/evals/endpoints-auth-routes.json', 1042],
+  ['skills/objectstack-automation/evals/flows-triggers-approvals.json', 1255],
+  ['skills/objectstack-data/evals/hooks-security-seeds-datasources.json', 894],
+  ['skills/objectstack-data/evals/objects-fields-relationships.json', 946],
+  ['skills/objectstack-formula/evals/cel-predicates-formulas.json', 1046],
+  ['skills/objectstack-i18n/evals/bundles-locales-coverage.json', 1004],
+  ['skills/objectstack-platform/evals/config-plugins-ops.json', 1211],
+  ['skills/objectstack-query/evals/filters-pagination-search.json', 955],
+  ['skills/objectstack-ui/evals/views-apps-actions-pages.json', 1505],
+  ['skills/objectstack-upgrade/evals/protocol-major-upgrade.json', 1033],
+
   // the remaining skills' eval notes
 ]);
 

skills/objectstack-ai/evals/skills-tools-knowledge.json

@@ -0,0 +1,55 @@
+{
+  "skill_name": "objectstack-ai",
+  "evals": [
+    {
+      "id": 1,
+      "prompt": "Add `src/skills/case-triage.skill.ts` for our support app: the assistant should be able to query and read support cases and run our existing `escalate_case` Action on a case. Only activate it when the user is working on the `support_case` object.",
+      "expected_output": "Emits a `defineSkill` with a snake_case `name`, `label`, `instructions`, and a `tools` list where every name resolves: platform tools `query_records` / `get_record` plus the action tool `action_escalate_case` materialised from the stack's own Action. Routing is a `triggerConditions` entry `{ field: 'objectName', operator: 'eq', value: 'support_case' }` (a string value for `eq`). No `permissions` key on the skill (removed in 17) and no `*.agent.ts` file.",
+      "files": [],
+      "assertions": {
+        "must_contain": ["defineSkill", "tools", "action_escalate_case", "triggerConditions"],
+        "must_not_contain": ["permissions:", "defineAgent("]
+      }
+    },
+    {
+      "id": 2,
+      "prompt": "Our copilot should use Anthropic with a low temperature (0.3) and cap responses at 2000 tokens. Where do these settings go in the metadata?",
+      "expected_output": "Places sampling under the agent's `model` block (`AIModelConfigSchema`): `provider: 'anthropic'`, a free-string `model`, `temperature: 0.3` (0–2, outside is a parse error) and `maxTokens: 2000`. States that there is no top-level `temperature` / `maxTokens` on an agent and that the inline `provider` enum is `openai | azure_openai | anthropic | local`.",
+      "files": [],
+      "assertions": {
+        "must_contain": ["provider", "anthropic", "temperature", "maxTokens"],
+        "must_not_contain": []
+      }
+    },
+    {
+      "id": 3,
+      "prompt": "Index our `kb_article` records (title and body) so the assistant can answer from them with RAG. Only published articles, and we want to filter by `category` at search time. We run RAGFlow.",
+      "expected_output": "Emits a `KnowledgeSourceSchema.parse({...})` record with `adapter: 'ragflow'` and `source: { kind: 'object', object: 'kb_article', contentFields: ['title', 'body'], metadataFields: ['category'], where: { published: true } }`, `refresh: { onRecordChange: true }`, registered at boot through `KnowledgeServicePlugin({ sources })`. Chunking / top-K / rerankers are adapter config, not platform metadata.",
+      "files": [],
+      "assertions": {
+        "must_contain": ["KnowledgeSourceSchema", "contentFields", "metadataFields", "where", "KnowledgeServicePlugin"],
+        "must_not_contain": ["chunkSize", "topK"]
+      }
+    },
+    {
+      "id": 4,
+      "prompt": "Write a `*.tool.ts` so the assistant can call our `close_case` action on a support case.",
+      "expected_output": "Does not author a tool record: the default path is to opt the Action in with `ai: { exposed: true, description: '<40+ chars>' }` on the object's action, which materialises `action_close_case`, and to name that tool in a skill's `tools`. `defineTool` is only for presentation refinement and cannot make anything executable (no `handler` / `implementation`; `ToolSchema` is strict).",
+      "files": [],
+      "assertions": {
+        "must_contain": ["exposed", "description", "action_", "tools"],
+        "must_not_contain": ["handler:", "implementation:"]
+      }
+    },
+    {
+      "id": 5,
+      "prompt": "Create `src/ai/support-bot.agent.ts` with defineAgent({ name: 'support_bot', tools: ['query_records', 'action_close_case'], knowledge: ['support_kb'] }) and register it under agents.",
+      "expected_output": "Refuses the agent: the `agent` type is closed to third parties (a stack-authored agent is filtered out, refused by `loadAgent()` and 404s on chat), and `agent.tools` / `agent.knowledge` are protocol-17 tombstones. Re-declares the two tools in a `defineSkill` (registered under `skills`) with a `surface`, and restricts retrieval at the knowledge-source level (`aiExposed`, `where`) rather than on an agent. The runtime's `ask` / `build` agents pick the skill up by surface.",
+      "files": [],
+      "assertions": {
+        "must_contain": ["defineSkill", "skills", "surface", "aiExposed"],
+        "must_not_contain": ["defineAgent(", "knowledge:"]
+      }
+    }
+  ]
+}

skills/objectstack-api/evals/endpoints-auth-routes.json

@@ -0,0 +1,55 @@
+{
+  "skill_name": "objectstack-api",
+  "evals": [
+    {
+      "id": 1,
+      "prompt": "Add `src/api/lead-feed.endpoint.ts`: a GET endpoint that returns our `acme_lead` records for a partner integration, cached for 30 seconds. The stack's manifest namespace is `acme`.",
+      "expected_output": "Emits a declarative `ApiEndpoint` (registered under `defineStack({ apis })`): `path: '/api/v1/apps/acme/leads'` (the ADR-0121 carve-out — only the subpath is free), `method: 'GET'`, `type: 'object_operation'`, `objectParams: { object: 'acme_lead', operation: 'find' }`, `cacheTtlSeconds: 30`. `authRequired` is omitted (defaults to true). No handler code, no `transform` in a mapping, and `manifest.namespace` is declared explicitly.",
+      "files": [],
+      "assertions": {
+        "must_contain": ["/api/v1/apps/", "object_operation", "objectParams", "cacheTtlSeconds"],
+        "must_not_contain": ["transform:", "'/api/v1/leads'"]
+      }
+    },
+    {
+      "id": 2,
+      "prompt": "The partner can't send a session token — make the lead feed callable without login.",
+      "expected_output": "Sets `authRequired: false` and, because ADR-0121 D6 pairs an anonymous entry point with an armed budget, adds `rateLimit: { enabled: true, windowMs: 60_000, maxRequests: 100 }`. Points out that the gate's predicate is `rateLimit.enabled === true`, so writing only `windowMs` / `maxRequests` declares a budget that meters nothing, and that the endpoint budget is independent of `server.security.rateLimit`.",
+      "files": [],
+      "assertions": {
+        "must_contain": ["authRequired", "rateLimit", "enabled: true", "maxRequests"],
+        "must_not_contain": []
+      }
+    },
+    {
+      "id": 3,
+      "prompt": "We receive Stripe webhooks and must verify the signature with real code before writing anything. Mount POST /api/v1/apps/acme/stripe in our plugin.",
+      "expected_output": "Uses the code-route pattern, not `apis:` (`script` / `proxy` targets do not execute in 17.x): in the plugin's `start(ctx)`, inside `ctx.hook('kernel:ready', ...)`, read `http.server` canonical-first with one `try` per name (`getService` is synchronous and throws on an empty slot), then `http.post('/api/v1/apps/acme/stripe', handler)`. Notes that `listen()` is deferred to `kernel:listening` so the route lands before Hono seals its matcher.",
+      "files": [],
+      "assertions": {
+        "must_contain": ["http.server", "kernel:ready", "getService"],
+        "must_not_contain": ["type: 'script'", "type: 'proxy'"]
+      }
+    },
+    {
+      "id": 4,
+      "prompt": "Enable Google and GitHub sign-in next to the existing email/password login.",
+      "expected_output": "Reads `node_modules/@objectstack/spec/src/api/auth.zod.ts` before wiring anything: `AuthProvider` is `'local' | 'google' | 'github' | 'microsoft' | 'ldap' | 'saml'`, and `LoginRequestSchema` carries `type` plus optional `email` / `username` / `password` / `provider` / `redirectTo`. Does not invent a nested `auth` block on an endpoint — endpoint auth is `authRequired` (declarative) or the flat `public` + `permissions` pair.",
+      "files": [],
+      "assertions": {
+        "must_contain": ["AuthProvider", "auth.zod.ts", "google", "github", "LoginRequestSchema"],
+        "must_not_contain": []
+      }
+    },
+    {
+      "id": 5,
+      "prompt": "Lock the `invoice` object's API down to read, list, search and export; add a `/bulk` route for imports and a `restore` operation so deleted invoices can be undeleted.",
+      "expected_output": "Authors only the primitives: `apiMethods: ['get', 'list']` — `search` and `export` derive from `list` automatically and are never declared (a declared derived verb is stripped with a warning). Explains there is no `/bulk` route (batch writes go through `POST /api/v1/data/{object}/batch`) and that `restore` / `purge` never derive because DELETE is a hard delete; recoverability is per-field `trackHistory` or a `lifecycle` archive policy.",
+      "files": [],
+      "assertions": {
+        "must_contain": ["apiMethods", "/batch", "trackHistory", "lifecycle"],
+        "must_not_contain": ["'search'", "'restore'"]
+      }
+    }
+  ]
+}

skills/objectstack-automation/evals/flows-triggers-approvals.json

@@ -0,0 +1,55 @@
+{
+  "skill_name": "objectstack-automation",
+  "evals": [
+    {
+      "id": 1,
+      "prompt": "Every night at 02:00, mark `subscription` records whose `renewal_date` is in the past as `lapsed`. `status` is a readonly field that only automation may change.",
+      "expected_output": "Emits a `defineFlow` with `type: 'schedule'`, the cadence on the START NODE's `config.schedule: { type: 'cron', expression: '0 2 * * *' }` (FlowSchema has no top-level `schedule`; no cron tagged template), `runAs: 'system'` because a scheduled run has no trigger user and a readonly field is stripped under `runAs: 'user'`. One `update_record` node with `filter` as an ObjectQL where-map (`renewal_date: { $lt: '{TODAY()}' }`) and `fields: { status: 'lapsed' }` (not `values`), `label` on every node, nodes wired with `edges` (never `next`), ending in an `end` node.",
+      "files": [],
+      "assertions": {
+        "must_contain": ["defineFlow", "type: 'schedule'", "runAs: 'system'", "update_record", "edges"],
+        "must_not_contain": ["next:", "values:", "cron`"]
+      }
+    },
+    {
+      "id": 2,
+      "prompt": "When an opportunity moves into stage `closed_won`, send the account owner an inbox notification with the deal name.",
+      "expected_output": "Emits a `record_change` flow whose binding lives on the `start` node `config`: `objectName: 'opportunity'`, `triggerType: 'record-after-update'`, and a bare-CEL `condition` `previous.stage != 'closed_won' && record.stage == 'closed_won'` so it fires only on the transition. A `notify` node with `recipients: '{record.owner_id}'`, `title` with single-brace interpolation, `channels: ['inbox']`. Reminds that `requires` must list `automation`, `triggers` and `messaging` (otherwise notify reports success with `skipped: true`).",
+      "files": [],
+      "assertions": {
+        "must_contain": ["triggerType", "record-after-update", "previous.", "notify", "recipients", "messaging"],
+        "must_not_contain": ["trigger:", "{{"]
+      }
+    },
+    {
+      "id": 3,
+      "prompt": "Build an autolaunched flow `budget_approval` on the `project` object: when `budget` increases past 100000, route to a manager for approval. The manager can approve, reject, or send the record back to the submitter for revision; after the submitter reworks and resubmits it returns to the manager for another round. Cap it at two send-backs.",
+      "expected_output": "Emits the ADR-0044 shape: an `approval` node (`type: 'approval'`, `approvers: [{ type: 'position', value: 'manager' }]`, `lockRecord: true`, `maxRevisions: 2`) with `approve` / `reject` out-edges plus a third `revise` out-edge targeting an `approval_revise` node (no config), and a resubmit edge back into the approval node typed `type: 'back'` — the only thing that legalises the cycle for `registerFlow`. The flow lives in `flows` (there is no `approvals` stack collection); the window is a node, not a re-suspend of the approval node.",
+      "files": [],
+      "assertions": {
+        "must_contain": ["type: 'approval'", "approval_revise", "type: 'back'", "maxRevisions", "revise"],
+        "must_not_contain": ["approvals: [", "maxRevisions: 0"]
+      }
+    },
+    {
+      "id": 4,
+      "prompt": "Build a guided 'new client onboarding' wizard: step 1 collects company details, step 2 asks whether a credit check is needed — if yes show a credit-check screen, otherwise skip it — then create the `account` record.",
+      "expected_output": "Because the steps BRANCH on logic, this is a `screen` flow (`type: 'screen'`), not a form-view wizard: `screen` nodes for each step, a `decision` node whose out-edges carry mutually exclusive `condition` predicates (an unguarded out-edge always runs, in parallel), a `create_record` node, explicit `variables` entries (`{ name, type, isInput, isOutput }`), and every path reaching an `end`. The flow stays `runAs: 'user'`.",
+      "files": [],
+      "assertions": {
+        "must_contain": ["screen", "decision", "condition", "create_record", "variables"],
+        "must_not_contain": ["type: 'wizard'", "next:"]
+      }
+    },
+    {
+      "id": 5,
+      "prompt": "Add a step that calls our `pricing.recalc` function to compute the discount and saves it on the quote, then posts the quote to our tax service over HTTP; if the tax call fails, route to a handler node instead of failing the run.",
+      "expected_output": "Emits a `script` node with `config.function: 'pricing.recalc'` (registered via `defineStack({ functions })`), `inputs`, and `outputVariable: 'recalc'`; the function is a pure compute step, so a following `update_record` persists `{recalc.discount}` via `fields`. The `http` node sets `timeoutMs`. Failure routing is an edge `{ source, target, type: 'fault' }` — a `label: 'error'` alone routes nothing.",
+      "files": [],
+      "assertions": {
+        "must_contain": ["function:", "outputVariable", "type: 'fault'", "timeoutMs", "update_record"],
+        "must_not_contain": ["label: 'error'", "query_record"]
+      }
+    }
+  ]
+}

skills/objectstack-data/evals/README.md

@@ -1,11 +1,12 @@
 # Evaluation Tests (evals/)
 
-⚠️ **Not yet implemented** — placeholder for future skill evals.
+JSON fixtures in the objectui shape (`skill_name`, `evals[]`: `prompt`,
+`expected_output`, `assertions.must_contain` / `must_not_contain`):
 
-Candidate data-domain scenarios: naming (snake_case names, lowercase option
-values), field-type selection (`secret` vs `password`, `lookup` vs
-`master_detail`), relationships (junction object vs multi-value lookup,
-`deleteBehavior`), validation (script inversion, `state_machine` transitions,
-unique **index** not a validation type), hooks (`before*` vs `after*`,
-sandboxed `body` capabilities), and seeds (`externalId` choice, natural-key
-lookups, CEL dynamic values).
+- `objects-fields-relationships.json` — naming, field types (`secret` vs
+  `password`), relationships (junction vs multi-value lookup,
+  `deleteBehavior`), validation (`state_machine`, unique **index**),
+  conditional field rules.
+- `hooks-security-seeds-datasources.json` — search mirrors via hooks,
+  permission-set scopes, external datasources, seeds (`externalId`,
+  CEL dynamic values).