diff --git a/scripts/check-skills-token-ratchet.mjs b/scripts/check-skills-token-ratchet.mjs index 5b984866059..fb4c4431240 100644 --- a/scripts/check-skills-token-ratchet.mjs +++ b/scripts/check-skills-token-ratchet.mjs @@ -472,6 +472,33 @@ export const CEILINGS = new Map([ ['skills/objectstack-ui/rules/navigation.md', 2273], ['skills/objectstack-ui/rules/pages.md', 5501], + // ── 2026-09-22 evals fixtures — decision batch #213 item 2, PR #19721 ──── + // Every published skill gained `evals/*.json` in the objectui fixture shape. + // The maintainer ruled the work in batch #213, verbatim and untranslated: + // 「1 2 4 同意」 on item 2 「objectstack 各 skill 先补 3 题 evals(复用 objectui 的 + // json 形状),先于第 1 项,好量化『切了没变差』」. The eleven files landed + // unpriced on 02b931d2 and this gate redded on every one of them, as the + // header says it must; the maintainer's follow-up word, recorded as issue + // comment 5776586573 (2026-09-22), verbatim and untranslated — + // 「天花板行按照你的意见就行」 — adopts pricing them in. So: one row per file, + // each an INITIAL measurement taken on that same tree in the convention + // above (re-measured unchanged on the commit that adds the rows) and pinned + // AT it — zero headroom, on the same terms as the #12392 extension rows and + // the #14296 「NEW FILE … pinned at its landed count」 rows. Nothing else + // moves: no existing row is raised, the population is unchanged, and + // `evals/**` stays priced. + ['skills/objectstack-ai/evals/skills-tools-knowledge.json', 1074], + ['skills/objectstack-api/evals/endpoints-auth-routes.json', 1042], + ['skills/objectstack-automation/evals/flows-triggers-approvals.json', 1255], + ['skills/objectstack-data/evals/hooks-security-seeds-datasources.json', 894], + ['skills/objectstack-data/evals/objects-fields-relationships.json', 946], + ['skills/objectstack-formula/evals/cel-predicates-formulas.json', 1046], + ['skills/objectstack-i18n/evals/bundles-locales-coverage.json', 1004], + ['skills/objectstack-platform/evals/config-plugins-ops.json', 1211], + ['skills/objectstack-query/evals/filters-pagination-search.json', 955], + ['skills/objectstack-ui/evals/views-apps-actions-pages.json', 1505], + ['skills/objectstack-upgrade/evals/protocol-major-upgrade.json', 1033], + // the remaining skills' eval notes ]); diff --git a/skills/objectstack-ai/evals/skills-tools-knowledge.json b/skills/objectstack-ai/evals/skills-tools-knowledge.json new file mode 100644 index 00000000000..3631cb7eb29 --- /dev/null +++ b/skills/objectstack-ai/evals/skills-tools-knowledge.json @@ -0,0 +1,55 @@ +{ + "skill_name": "objectstack-ai", + "evals": [ + { + "id": 1, + "prompt": "Add `src/skills/case-triage.skill.ts` for our support app: the assistant should be able to query and read support cases and run our existing `escalate_case` Action on a case. Only activate it when the user is working on the `support_case` object.", + "expected_output": "Emits a `defineSkill` with a snake_case `name`, `label`, `instructions`, and a `tools` list where every name resolves: platform tools `query_records` / `get_record` plus the action tool `action_escalate_case` materialised from the stack's own Action. Routing is a `triggerConditions` entry `{ field: 'objectName', operator: 'eq', value: 'support_case' }` (a string value for `eq`). No `permissions` key on the skill (removed in 17) and no `*.agent.ts` file.", + "files": [], + "assertions": { + "must_contain": ["defineSkill", "tools", "action_escalate_case", "triggerConditions"], + "must_not_contain": ["permissions:", "defineAgent("] + } + }, + { + "id": 2, + "prompt": "Our copilot should use Anthropic with a low temperature (0.3) and cap responses at 2000 tokens. Where do these settings go in the metadata?", + "expected_output": "Places sampling under the agent's `model` block (`AIModelConfigSchema`): `provider: 'anthropic'`, a free-string `model`, `temperature: 0.3` (0–2, outside is a parse error) and `maxTokens: 2000`. States that there is no top-level `temperature` / `maxTokens` on an agent and that the inline `provider` enum is `openai | azure_openai | anthropic | local`.", + "files": [], + "assertions": { + "must_contain": ["provider", "anthropic", "temperature", "maxTokens"], + "must_not_contain": [] + } + }, + { + "id": 3, + "prompt": "Index our `kb_article` records (title and body) so the assistant can answer from them with RAG. Only published articles, and we want to filter by `category` at search time. We run RAGFlow.", + "expected_output": "Emits a `KnowledgeSourceSchema.parse({...})` record with `adapter: 'ragflow'` and `source: { kind: 'object', object: 'kb_article', contentFields: ['title', 'body'], metadataFields: ['category'], where: { published: true } }`, `refresh: { onRecordChange: true }`, registered at boot through `KnowledgeServicePlugin({ sources })`. Chunking / top-K / rerankers are adapter config, not platform metadata.", + "files": [], + "assertions": { + "must_contain": ["KnowledgeSourceSchema", "contentFields", "metadataFields", "where", "KnowledgeServicePlugin"], + "must_not_contain": ["chunkSize", "topK"] + } + }, + { + "id": 4, + "prompt": "Write a `*.tool.ts` so the assistant can call our `close_case` action on a support case.", + "expected_output": "Does not author a tool record: the default path is to opt the Action in with `ai: { exposed: true, description: '<40+ chars>' }` on the object's action, which materialises `action_close_case`, and to name that tool in a skill's `tools`. `defineTool` is only for presentation refinement and cannot make anything executable (no `handler` / `implementation`; `ToolSchema` is strict).", + "files": [], + "assertions": { + "must_contain": ["exposed", "description", "action_", "tools"], + "must_not_contain": ["handler:", "implementation:"] + } + }, + { + "id": 5, + "prompt": "Create `src/ai/support-bot.agent.ts` with defineAgent({ name: 'support_bot', tools: ['query_records', 'action_close_case'], knowledge: ['support_kb'] }) and register it under agents.", + "expected_output": "Refuses the agent: the `agent` type is closed to third parties (a stack-authored agent is filtered out, refused by `loadAgent()` and 404s on chat), and `agent.tools` / `agent.knowledge` are protocol-17 tombstones. Re-declares the two tools in a `defineSkill` (registered under `skills`) with a `surface`, and restricts retrieval at the knowledge-source level (`aiExposed`, `where`) rather than on an agent. The runtime's `ask` / `build` agents pick the skill up by surface.", + "files": [], + "assertions": { + "must_contain": ["defineSkill", "skills", "surface", "aiExposed"], + "must_not_contain": ["defineAgent(", "knowledge:"] + } + } + ] +} diff --git a/skills/objectstack-api/evals/endpoints-auth-routes.json b/skills/objectstack-api/evals/endpoints-auth-routes.json new file mode 100644 index 00000000000..040c82d71c5 --- /dev/null +++ b/skills/objectstack-api/evals/endpoints-auth-routes.json @@ -0,0 +1,55 @@ +{ + "skill_name": "objectstack-api", + "evals": [ + { + "id": 1, + "prompt": "Add `src/api/lead-feed.endpoint.ts`: a GET endpoint that returns our `acme_lead` records for a partner integration, cached for 30 seconds. The stack's manifest namespace is `acme`.", + "expected_output": "Emits a declarative `ApiEndpoint` (registered under `defineStack({ apis })`): `path: '/api/v1/apps/acme/leads'` (the ADR-0121 carve-out — only the subpath is free), `method: 'GET'`, `type: 'object_operation'`, `objectParams: { object: 'acme_lead', operation: 'find' }`, `cacheTtlSeconds: 30`. `authRequired` is omitted (defaults to true). No handler code, no `transform` in a mapping, and `manifest.namespace` is declared explicitly.", + "files": [], + "assertions": { + "must_contain": ["/api/v1/apps/", "object_operation", "objectParams", "cacheTtlSeconds"], + "must_not_contain": ["transform:", "'/api/v1/leads'"] + } + }, + { + "id": 2, + "prompt": "The partner can't send a session token — make the lead feed callable without login.", + "expected_output": "Sets `authRequired: false` and, because ADR-0121 D6 pairs an anonymous entry point with an armed budget, adds `rateLimit: { enabled: true, windowMs: 60_000, maxRequests: 100 }`. Points out that the gate's predicate is `rateLimit.enabled === true`, so writing only `windowMs` / `maxRequests` declares a budget that meters nothing, and that the endpoint budget is independent of `server.security.rateLimit`.", + "files": [], + "assertions": { + "must_contain": ["authRequired", "rateLimit", "enabled: true", "maxRequests"], + "must_not_contain": [] + } + }, + { + "id": 3, + "prompt": "We receive Stripe webhooks and must verify the signature with real code before writing anything. Mount POST /api/v1/apps/acme/stripe in our plugin.", + "expected_output": "Uses the code-route pattern, not `apis:` (`script` / `proxy` targets do not execute in 17.x): in the plugin's `start(ctx)`, inside `ctx.hook('kernel:ready', ...)`, read `http.server` canonical-first with one `try` per name (`getService` is synchronous and throws on an empty slot), then `http.post('/api/v1/apps/acme/stripe', handler)`. Notes that `listen()` is deferred to `kernel:listening` so the route lands before Hono seals its matcher.", + "files": [], + "assertions": { + "must_contain": ["http.server", "kernel:ready", "getService"], + "must_not_contain": ["type: 'script'", "type: 'proxy'"] + } + }, + { + "id": 4, + "prompt": "Enable Google and GitHub sign-in next to the existing email/password login.", + "expected_output": "Reads `node_modules/@objectstack/spec/src/api/auth.zod.ts` before wiring anything: `AuthProvider` is `'local' | 'google' | 'github' | 'microsoft' | 'ldap' | 'saml'`, and `LoginRequestSchema` carries `type` plus optional `email` / `username` / `password` / `provider` / `redirectTo`. Does not invent a nested `auth` block on an endpoint — endpoint auth is `authRequired` (declarative) or the flat `public` + `permissions` pair.", + "files": [], + "assertions": { + "must_contain": ["AuthProvider", "auth.zod.ts", "google", "github", "LoginRequestSchema"], + "must_not_contain": [] + } + }, + { + "id": 5, + "prompt": "Lock the `invoice` object's API down to read, list, search and export; add a `/bulk` route for imports and a `restore` operation so deleted invoices can be undeleted.", + "expected_output": "Authors only the primitives: `apiMethods: ['get', 'list']` — `search` and `export` derive from `list` automatically and are never declared (a declared derived verb is stripped with a warning). Explains there is no `/bulk` route (batch writes go through `POST /api/v1/data/{object}/batch`) and that `restore` / `purge` never derive because DELETE is a hard delete; recoverability is per-field `trackHistory` or a `lifecycle` archive policy.", + "files": [], + "assertions": { + "must_contain": ["apiMethods", "/batch", "trackHistory", "lifecycle"], + "must_not_contain": ["'search'", "'restore'"] + } + } + ] +} diff --git a/skills/objectstack-automation/evals/flows-triggers-approvals.json b/skills/objectstack-automation/evals/flows-triggers-approvals.json new file mode 100644 index 00000000000..59c0fbf46ca --- /dev/null +++ b/skills/objectstack-automation/evals/flows-triggers-approvals.json @@ -0,0 +1,55 @@ +{ + "skill_name": "objectstack-automation", + "evals": [ + { + "id": 1, + "prompt": "Every night at 02:00, mark `subscription` records whose `renewal_date` is in the past as `lapsed`. `status` is a readonly field that only automation may change.", + "expected_output": "Emits a `defineFlow` with `type: 'schedule'`, the cadence on the START NODE's `config.schedule: { type: 'cron', expression: '0 2 * * *' }` (FlowSchema has no top-level `schedule`; no cron tagged template), `runAs: 'system'` because a scheduled run has no trigger user and a readonly field is stripped under `runAs: 'user'`. One `update_record` node with `filter` as an ObjectQL where-map (`renewal_date: { $lt: '{TODAY()}' }`) and `fields: { status: 'lapsed' }` (not `values`), `label` on every node, nodes wired with `edges` (never `next`), ending in an `end` node.", + "files": [], + "assertions": { + "must_contain": ["defineFlow", "type: 'schedule'", "runAs: 'system'", "update_record", "edges"], + "must_not_contain": ["next:", "values:", "cron`"] + } + }, + { + "id": 2, + "prompt": "When an opportunity moves into stage `closed_won`, send the account owner an inbox notification with the deal name.", + "expected_output": "Emits a `record_change` flow whose binding lives on the `start` node `config`: `objectName: 'opportunity'`, `triggerType: 'record-after-update'`, and a bare-CEL `condition` `previous.stage != 'closed_won' && record.stage == 'closed_won'` so it fires only on the transition. A `notify` node with `recipients: '{record.owner_id}'`, `title` with single-brace interpolation, `channels: ['inbox']`. Reminds that `requires` must list `automation`, `triggers` and `messaging` (otherwise notify reports success with `skipped: true`).", + "files": [], + "assertions": { + "must_contain": ["triggerType", "record-after-update", "previous.", "notify", "recipients", "messaging"], + "must_not_contain": ["trigger:", "{{"] + } + }, + { + "id": 3, + "prompt": "Build an autolaunched flow `budget_approval` on the `project` object: when `budget` increases past 100000, route to a manager for approval. The manager can approve, reject, or send the record back to the submitter for revision; after the submitter reworks and resubmits it returns to the manager for another round. Cap it at two send-backs.", + "expected_output": "Emits the ADR-0044 shape: an `approval` node (`type: 'approval'`, `approvers: [{ type: 'position', value: 'manager' }]`, `lockRecord: true`, `maxRevisions: 2`) with `approve` / `reject` out-edges plus a third `revise` out-edge targeting an `approval_revise` node (no config), and a resubmit edge back into the approval node typed `type: 'back'` — the only thing that legalises the cycle for `registerFlow`. The flow lives in `flows` (there is no `approvals` stack collection); the window is a node, not a re-suspend of the approval node.", + "files": [], + "assertions": { + "must_contain": ["type: 'approval'", "approval_revise", "type: 'back'", "maxRevisions", "revise"], + "must_not_contain": ["approvals: [", "maxRevisions: 0"] + } + }, + { + "id": 4, + "prompt": "Build a guided 'new client onboarding' wizard: step 1 collects company details, step 2 asks whether a credit check is needed — if yes show a credit-check screen, otherwise skip it — then create the `account` record.", + "expected_output": "Because the steps BRANCH on logic, this is a `screen` flow (`type: 'screen'`), not a form-view wizard: `screen` nodes for each step, a `decision` node whose out-edges carry mutually exclusive `condition` predicates (an unguarded out-edge always runs, in parallel), a `create_record` node, explicit `variables` entries (`{ name, type, isInput, isOutput }`), and every path reaching an `end`. The flow stays `runAs: 'user'`.", + "files": [], + "assertions": { + "must_contain": ["screen", "decision", "condition", "create_record", "variables"], + "must_not_contain": ["type: 'wizard'", "next:"] + } + }, + { + "id": 5, + "prompt": "Add a step that calls our `pricing.recalc` function to compute the discount and saves it on the quote, then posts the quote to our tax service over HTTP; if the tax call fails, route to a handler node instead of failing the run.", + "expected_output": "Emits a `script` node with `config.function: 'pricing.recalc'` (registered via `defineStack({ functions })`), `inputs`, and `outputVariable: 'recalc'`; the function is a pure compute step, so a following `update_record` persists `{recalc.discount}` via `fields`. The `http` node sets `timeoutMs`. Failure routing is an edge `{ source, target, type: 'fault' }` — a `label: 'error'` alone routes nothing.", + "files": [], + "assertions": { + "must_contain": ["function:", "outputVariable", "type: 'fault'", "timeoutMs", "update_record"], + "must_not_contain": ["label: 'error'", "query_record"] + } + } + ] +} diff --git a/skills/objectstack-data/evals/README.md b/skills/objectstack-data/evals/README.md index 7d003a00ab7..520a5e2be7f 100644 --- a/skills/objectstack-data/evals/README.md +++ b/skills/objectstack-data/evals/README.md @@ -1,11 +1,12 @@ # Evaluation Tests (evals/) -⚠️ **Not yet implemented** — placeholder for future skill evals. +JSON fixtures in the objectui shape (`skill_name`, `evals[]`: `prompt`, +`expected_output`, `assertions.must_contain` / `must_not_contain`): -Candidate data-domain scenarios: naming (snake_case names, lowercase option -values), field-type selection (`secret` vs `password`, `lookup` vs -`master_detail`), relationships (junction object vs multi-value lookup, -`deleteBehavior`), validation (script inversion, `state_machine` transitions, -unique **index** not a validation type), hooks (`before*` vs `after*`, -sandboxed `body` capabilities), and seeds (`externalId` choice, natural-key -lookups, CEL dynamic values). +- `objects-fields-relationships.json` — naming, field types (`secret` vs + `password`), relationships (junction vs multi-value lookup, + `deleteBehavior`), validation (`state_machine`, unique **index**), + conditional field rules. +- `hooks-security-seeds-datasources.json` — search mirrors via hooks, + permission-set scopes, external datasources, seeds (`externalId`, + CEL dynamic values). diff --git a/skills/objectstack-data/evals/hooks-security-seeds-datasources.json b/skills/objectstack-data/evals/hooks-security-seeds-datasources.json new file mode 100644 index 00000000000..6f4648942de --- /dev/null +++ b/skills/objectstack-data/evals/hooks-security-seeds-datasources.json @@ -0,0 +1,45 @@ +{ + "skill_name": "objectstack-data", + "evals": [ + { + "id": 1, + "prompt": "Support agents want to search tasks by the name of the project they belong to.", + "expected_output": "Applies the one prescription: a stored `text` mirror field `project_name` on `task`, listed in `searchableFields: ['name', 'project_name']`, maintained by a `beforeInsert` / `beforeUpdate` hook on `task` (stamp the parent's name for the incoming `project_id`) and an `afterUpdate` hook on `project` (re-stamp on rename), plus a one-off backfill for rows written outside hooks. Never a dotted path (`project_id.name` is refused, `400 INVALID_FIELD`) and never a `formula` mirror (virtual, nothing to scan).", + "files": [], + "assertions": { + "must_contain": ["searchableFields", "project_name", "beforeInsert", "afterUpdate"], + "must_not_contain": ["'project_id.name'", "type: 'formula'"] + } + }, + { + "id": 2, + "prompt": "Sales reps may only see their own leads, sales managers see their unit and everything below it, and every signed-in user can read the `product` catalog.", + "expected_output": "Access is authored in permission sets, not on the object (an object-level `permissions` key is rejected). `lead` gets `sharingModel: 'private'`; the rep set's grant on `lead` carries `readScope: 'own'`, the manager set's `readScope: 'unit_and_below'` — an ADR-0057 scope that needs the enterprise `@objectstack/security-enterprise` plugin and `requires: ['hierarchy-security']`, failing closed to `own` without it. `product` is `sharingModel: 'public_read'`.", + "files": [], + "assertions": { + "must_contain": ["sharingModel: 'private'", "readScope", "unit_and_below", "public_read", "hierarchy-security"], + "must_not_contain": ["permissions: {"] + } + }, + { + "id": 3, + "prompt": "Expose the existing `customers` table from our Postgres reporting database as a read-only `legacy_customer` object.", + "expected_output": "Declares a `defineDatasource` entry (credentials by `credentialsRef`, never inline) and points the object at it with `datasource: ''`, mapping the physical table via `remoteName` and column names via `columnMap` — there is no `field.columnName` on an external object. Sets the datasource's `schemaMode` so the platform never alters the remote schema, and keeps the external write gate closed.", + "files": [], + "assertions": { + "must_contain": ["defineDatasource", "datasource", "remoteName", "columnMap", "schemaMode"], + "must_not_contain": ["columnName:"] + } + }, + { + "id": 4, + "prompt": "Seed the `country` reference table for every environment, and add demo `opportunity` rows (dev/test only) that close 45 days after install and are owned by whoever installs the package.", + "expected_output": "Uses `defineSeed(Country, { externalId: 'code', mode: 'upsert', records })` for reference data and `defineSeed(Opportunity, { env: ['dev', 'test'], ... })` for demo rows, with lookups referenced by natural key and dynamic values as CEL: `close_date: cel`daysFromNow(45)``, `owner_id: cel`os.user.id``. Never `externalId: 'id'`, never `new Date()` / `Date.now()` (breaks build determinism), never `defineDataset` (the analytics layer), parents before children in the exported array.", + "files": [], + "assertions": { + "must_contain": ["defineSeed", "externalId", "upsert", "env:", "daysFromNow"], + "must_not_contain": ["new Date(", "defineDataset", "externalId: 'id'"] + } + } + ] +} diff --git a/skills/objectstack-data/evals/objects-fields-relationships.json b/skills/objectstack-data/evals/objects-fields-relationships.json new file mode 100644 index 00000000000..1fcdaaf6e22 --- /dev/null +++ b/skills/objectstack-data/evals/objects-fields-relationships.json @@ -0,0 +1,45 @@ +{ + "skill_name": "objectstack-data", + "evals": [ + { + "id": 1, + "prompt": "Create `src/objects/vendor.object.ts` for our vendors: company name, contact email, a status (Active / Inactive), the API key we use to call their system, and the password for their supplier portal.", + "expected_output": "Emits `ObjectSchema.create({...})` with a snake_case `name`, an authored `sharingModel` (a publish with none is refused), `label` / `pluralLabel` / `icon`, `nameField`, and fields with snake_case keys and camelCase config properties: `text` (required), `email`, a `select` whose `options` carry lowercase `value`s (`active`, `inactive`), and BOTH credentials as `type: 'secret'` (encrypted at rest) — never `type: 'password'`, which is plaintext at rest on a generic object. No `namespace` or `tableName` key.", + "files": [], + "assertions": { + "must_contain": ["ObjectSchema.create", "sharingModel", "type: 'secret'", "options", "nameField"], + "must_not_contain": ["type: 'password'", "namespace:", "vendorName"] + } + }, + { + "id": 2, + "prompt": "Model invoices with line items, and let invoices be tagged with many tags where each tagging records who added it and when.", + "expected_output": "Line items: `invoice_line` with a `master_detail` field to `invoice`, `required: true`, `deleteBehavior: 'cascade'` (or `'restrict'` — `set_null` is refused on master_detail), `inlineEdit` on that relationship, and a `summary` roll-up field on the parent. Tagging carries attributes, so it is a junction object (`invoice_tag`) with two `lookup` fields plus `added_by` / `added_at` — not a `multiple: true` lookup (an array of ids) and not a `many_to_many` type, which does not exist.", + "files": [], + "assertions": { + "must_contain": ["master_detail", "deleteBehavior", "inlineEdit", "summary", "lookup"], + "must_not_contain": ["many_to_many", "deleteBehavior: 'set_null'"] + } + }, + { + "id": 3, + "prompt": "On `employee`: email must be unique within each organization, `status` may only move draft → active → terminated, and a `contract_end` earlier than `contract_start` must be rejected.", + "expected_output": "Uniqueness is an index, not a validation type: `indexes: [{ fields: ['email'], unique: 'organization' }]` (the scope is stated; `type` / `partial` are retired). The lifecycle is one `state_machine` validation with `field: 'status'`, `transitions: { draft: ['active'], active: ['terminated'], terminated: [] }` and a `message`. The date rule is a `script` validation (inverted: fails when the expression is true) or `cross_field`, written as bare CEL over `record.` with explicit `!= null` guards.", + "files": [], + "assertions": { + "must_contain": ["unique: 'organization'", "state_machine", "transitions", "record."], + "must_not_contain": ["type: 'unique'", "partial:"] + } + }, + { + "id": 4, + "prompt": "On `invoice`, `paid_at` should only appear — and become mandatory — once the status is `paid`, and `total` must lock after payment.", + "expected_output": "Declares the rules on the FIELD definitions (they belong to the data model, so every form and API write honours them): `visibleWhen: P`record.status == 'paid'``, `requiredWhen: P`record.status == 'paid'`` on `paid_at`, and `readonlyWhen: P`record.status == 'paid'`` on `total`. Bare CEL with `record.` — never brace-wrapped, never Salesforce syntax, never the removed `conditionalRequired` alias, and not in a form view.", + "files": [], + "assertions": { + "must_contain": ["visibleWhen", "requiredWhen", "readonlyWhen", "record.status"], + "must_not_contain": ["conditionalRequired:", "{record."] + } + } + ] +} diff --git a/skills/objectstack-formula/evals/cel-predicates-formulas.json b/skills/objectstack-formula/evals/cel-predicates-formulas.json new file mode 100644 index 00000000000..6c335d1299d --- /dev/null +++ b/skills/objectstack-formula/evals/cel-predicates-formulas.json @@ -0,0 +1,55 @@ +{ + "skill_name": "objectstack-formula", + "evals": [ + { + "id": 1, + "prompt": "Add a `full_name` formula on `contact` built from salutation, first name and last name, and a `margin_pct` formula on `deal` = (revenue - cost) / cost * 100.", + "expected_output": "Two `type: 'formula'` fields whose `expression` is CEL via the `F` tag. The text formula coalesces every nullable operand — `F`coalesce(record.salutation, '') + ' ' + coalesce(record.first_name, '') + ' ' + coalesce(record.last_name, '')`` — because `null + 'foo'` throws. The numeric one guards the divisor: `F`coalesce(record.cost, 0) > 0 ? ((coalesce(record.revenue, 0) - record.cost) * 100) / record.cost : 0.0``. Field references are bare `record.`, never brace-wrapped.", + "files": [], + "assertions": { + "must_contain": ["F`", "coalesce(", "record."], + "must_not_contain": ["{record.", "CONCAT("] + } + }, + { + "id": 2, + "prompt": "Translate this Salesforce validation rule into ObjectStack: AND(ISBLANK(po_number), amount > 10000, NOT(ISPICKVAL(status, 'draft'))).", + "expected_output": "Applies the mechanical legacy → CEL table: `P`isBlank(record.po_number) && record.amount > 10000 && record.status != 'draft'``, noting that a `script` validation is inverted (it FAILS when the expression is true, so keep the failing condition) and that `record.amount > 10000` on a declared-nullable field needs an explicit `record.amount != null &&` guard or the publish gate rejects it. Uses `'draft'` single quotes and bare `record.` references.", + "files": [], + "assertions": { + "must_contain": ["isBlank(", "&&", "record.", "P`"], + "must_not_contain": ["{record.", "ISBLANK(record."] + } + }, + { + "id": 3, + "prompt": "Run an automation only when a support case has just become `escalated` — on single updates and on bulk updates alike.", + "expected_output": "Writes the transition once as `P`previous.status != 'escalated' && record.status == 'escalated'``: `record` is the record's post-write state (true on every update of an already-escalated case), so only a `previous` comparison says \"just became\"; `previous` is bound per matched row on `before*` and `after*` update events, bulk included. Not usable on insert events (unbound `previous` aborts the write). There is no `ISCHANGED()` function.", + "files": [], + "assertions": { + "must_contain": ["previous.status", "record.status", "!="], + "must_not_contain": ["ISCHANGED(", "{previous."] + } + }, + { + "id": 4, + "prompt": "Demo opportunities should close 45 days after the package is installed, and a formula should show how many days remain until close.", + "expected_output": "Seed value as a CEL envelope, `close_date: cel`daysFromNow(45)``, never `new Date()` / `Date.now()` (the determinism gate: two `os build` runs must be byte-identical). Days remaining is `daysBetween(today(), record.close_date)` — never `record.close_date - today()` (a duration stored as `{}`) and never `today() + 45` (date arithmetic faults).", + "files": [], + "assertions": { + "must_contain": ["cel`", "daysFromNow(", "daysBetween("], + "must_not_contain": ["new Date(", "today() + "] + } + }, + { + "id": 5, + "prompt": "Hide `discount_reason` unless a discount amount was entered, and reject any `end_date` earlier than `start_date`.", + "expected_output": "`visibleWhen: P`record.discount != null`` (or `!isBlank(record.discount)`) on the field — `has(record.discount)` is not a null check: every declared field exists, so it is always true. The date rule is a `script` validation whose predicate dominates the comparison with explicit null tests: `record.start_date != null && record.end_date != null && record.end_date < record.start_date`; a `has()` guard does not satisfy the publish-time gate. Canonical property names only — never `conditionalRequired`.", + "files": [], + "assertions": { + "must_contain": ["visibleWhen", "!= null", "record.end_date"], + "must_not_contain": ["conditionalRequired:", "{record."] + } + } + ] +} diff --git a/skills/objectstack-i18n/evals/bundles-locales-coverage.json b/skills/objectstack-i18n/evals/bundles-locales-coverage.json new file mode 100644 index 00000000000..df8410377c7 --- /dev/null +++ b/skills/objectstack-i18n/evals/bundles-locales-coverage.json @@ -0,0 +1,55 @@ +{ + "skill_name": "objectstack-i18n", + "evals": [ + { + "id": 1, + "prompt": "Add Simplified Chinese for the `support_case` object: its label and plural, the `status` field with options New / Open / Closed, the `close` action, and a welcome message that greets the user by name.", + "expected_output": "A `TranslationData` per locale under `objects.support_case` — `label`, `pluralLabel`, `fields.status.label`, `fields.status.options` keyed by the STORED option value (`new`, `open`, `closed`, never the display label), `_actions.close.label` — assembled with `defineTranslationBundle({ en, 'zh-CN': zhCN })` and registered under `defineStack({ translations })`. The message uses double-brace interpolation, `'welcome.user': '欢迎,{{userName}}!'`; there is no ICU plural/select.", + "files": [], + "assertions": { + "must_contain": ["defineTranslationBundle", "objects", "pluralLabel", "options", "_actions", "{{"], + "must_not_contain": ["o: {", "{count, plural"] + } + }, + { + "id": 2, + "prompt": "We are adding Japanese to the app.", + "expected_output": "Adds `'ja-JP'` (BCP-47) to `i18n.supportedLocales`, keeps `defaultLocale` / `fallbackLocale`, scaffolds the skeleton with `os i18n extract --locales=ja-JP --out=./src/translations`, adds the `'ja-JP'` key to the bundle, and verifies with `os i18n check --locales=ja-JP`.", + "files": [], + "assertions": { + "must_contain": ["supportedLocales", "ja-JP", "os i18n extract", "fallbackLocale"], + "must_not_contain": ["'jp'", "ja_JP"] + } + }, + { + "id": 3, + "prompt": "A customer admin wants to relabel `account` as 客户 in their Studio without waiting for a redeploy.", + "expected_output": "A `translation` metadata item (author with `defineTranslation`, shape `TranslationItemSchema`) carrying `locale: 'zh-CN'` and the same `objects.account.label` group as a file bundle; published items layer over the shipped bundles at boot and on every publish, and deleting the item restores the shipped value. One locale per item; the retired `o.*` dialect is rejected on this door too.", + "files": [], + "assertions": { + "must_contain": ["translation", "locale", "objects", "defineTranslation"], + "must_not_contain": ["o: {", "_meta"] + } + }, + { + "id": 4, + "prompt": "Our storage plugin ships its own generated `objects` translations for `en` and `zh-CN`. Wire them in.", + "expected_output": "Wraps the generated subtree into a `TranslationData` (`{ objects: enObjects }`), builds the bundle with `defineTranslationBundle`, and wraps the translated locale with `withSourceFallback(..., enSource, undefined, zhCNGeneratedSourceHashes)` so a leaf whose source moved is not served forever under a green check. A plugin that owns its objects loads it from its own `kernel:ready` hook via `i18n.loadTranslations(locale, data)` (deep-merge) rather than the stack.", + "files": [], + "assertions": { + "must_contain": ["defineTranslationBundle", "withSourceFallback", "loadTranslations", "kernel:ready"], + "must_not_contain": ["getAppBundle", "loadAppBundle"] + } + }, + { + "id": 5, + "prompt": "`os i18n check` reports zh-CN is missing app, navigation and dashboard keys even though we just ran `os i18n extract`. What is wrong?", + "expected_output": "Diagnoses that `--objects-only` is on by default, so extract wrote only the `objects` / `globalActions` subtree while check also demands `apps`, `navigation`, `dashboard`, `widget`, `page` and `flow` keys. Re-runs `os i18n extract --no-objects-only` (or hand-authors those groups), re-checks with `os i18n check --locales=zh-CN`, and adds `--strict` / `--threshold` in CI; `extract --check` is the separate freshness gate.", + "files": [], + "assertions": { + "must_contain": ["--no-objects-only", "os i18n check", "apps", "--strict"], + "must_not_contain": ["messageFormat"] + } + } + ] +} diff --git a/skills/objectstack-platform/evals/config-plugins-ops.json b/skills/objectstack-platform/evals/config-plugins-ops.json new file mode 100644 index 00000000000..7355f340fc3 --- /dev/null +++ b/skills/objectstack-platform/evals/config-plugins-ops.json @@ -0,0 +1,65 @@ +{ + "skill_name": "objectstack-platform", + "evals": [ + { + "id": 1, + "prompt": "Write `objectstack.config.ts` for our CRM app: objects, views and flows live under src/, the package namespace is `crm`, and it targets protocol 17.", + "expected_output": "A `defineStack` whose `manifest` carries `id`, `version`, `type: 'app'`, `name`, `namespace: 'crm'` and `engines: { protocol: '^17' }`, with barrel imports and `objects: Object.values(objects)` (same for `views` / `flows`). Object names carry the prefix in `name` (`crm_account`) — no object-level `namespace`, no `tableName`. No driver in `plugins:` (the CLI resolves it from the DB URL) and no `workflows` / `approvals` top-level keys (refused at load).", + "files": [], + "assertions": { + "must_contain": ["defineStack", "manifest", "engines", "Object.values("], + "must_not_contain": ["workflows:", "approvals:", "tableName"] + } + }, + { + "id": 2, + "prompt": "Build a plugin that registers an `audit` service and records every metadata reload; it should also record every `task` insert.", + "expected_output": "A `Plugin` object with `name`, `init(ctx)` calling `ctx.registerService('audit', ...)` and `ctx.hook('metadata:reloaded', ...)`, `start(ctx)`, `destroy()`. Record-level events are NOT kernel hooks — `ctx.hook('data:beforeInsert')` registers and silently never fires — so the task insert is a data hook (`hooks:` collection / `ql.on('beforeInsert', 'task', ...)`, objectstack-data). No `onInstall` / `onEnable` / `onDisable` plugin hooks exist.", + "files": [], + "assertions": { + "must_contain": ["registerService", "ctx.hook(", "metadata:reloaded", "init(", "destroy"], + "must_not_contain": ["data:beforeInsert", "onInstall"] + } + }, + { + "id": 3, + "prompt": "Our scheduled flows never launch, and a `notify` step reports success but nobody receives anything.", + "expected_output": "Both are missing capability tokens: `requires: ['automation', 'triggers', 'job', 'messaging']` on the stack root — `triggers` alone arms record-change triggers, schedule and time-relative triggers run on the `job` service, and without `messaging` notify returns success with `skipped: true`. Declaring is a demand (a missing provider is a hard boot error); an unknown token is rejected by `defineStack` at authoring time.", + "files": [], + "assertions": { + "must_contain": ["requires", "job", "messaging", "triggers"], + "must_not_contain": ["plugins: [new ServiceJobPlugin", "@objectstack/adapter-"] + } + }, + { + "id": 4, + "prompt": "We deploy on Vercel serverless functions inside our own Hono app. Mount the ObjectStack API under /api.", + "expected_output": "`createHonoApp({ kernel })` from `@objectstack/hono` (prefix defaults to `/api`) mounted into the host app — `HonoServerPlugin` is for when ObjectStack owns the server. Uses `LiteKernel` for the cold-start-sensitive host, warns that `/auth/*` follows the auth `basePath`, not `prefix`, and that there are no `@objectstack/adapter-*` packages.", + "files": [], + "assertions": { + "must_contain": ["createHonoApp", "@objectstack/hono", "LiteKernel"], + "must_not_contain": ["@objectstack/adapter-", "HonoServerPlugin("] + } + }, + { + "id": 5, + "prompt": "I changed field types on an object that already has production data. Show me what the database needs, verify the metadata, and give me a quality score.", + "expected_output": "`os migrate plan` to preview the metadata↔DB drift, `os migrate apply` (with `--allow-destructive` for type changes / drops) to reconcile, then `os validate` (schema + CEL + widget bindings) and `os lint --score`. Does not report the change done until `os validate` passes; `os lint` does not replace it.", + "files": [], + "assertions": { + "must_contain": ["os migrate plan", "os migrate apply", "os validate", "os lint"], + "must_not_contain": ["os rollback", "os data seed"] + } + }, + { + "id": 6, + "prompt": "Plan the production deployment: Docker on Kubernetes behind nginx, PostgreSQL.", + "expected_output": "`os start` as the container entrypoint with `PORT` pinned explicitly (production fails loudly on a busy port; only dev auto-hops), `driver-sql` selected by the database URL (no `DriverPlugin` in `plugins:`), TLS terminated on the proxy with the app on an internal port, and `OS_AUTH_URL` / `OS_TRUSTED_ORIGINS` / the app `hostname` kept in sync with the proxy. Health via `GET /health` and `GET /ready`.", + "files": [], + "assertions": { + "must_contain": ["os start", "PORT", "OS_TRUSTED_ORIGINS", "driver-sql"], + "must_not_contain": ["new DriverPlugin", "@objectstack/plugin-prometheus"] + } + } + ] +} diff --git a/skills/objectstack-query/evals/filters-pagination-search.json b/skills/objectstack-query/evals/filters-pagination-search.json new file mode 100644 index 00000000000..13c60ed0ccf --- /dev/null +++ b/skills/objectstack-query/evals/filters-pagination-search.json @@ -0,0 +1,55 @@ +{ + "skill_name": "objectstack-query", + "evals": [ + { + "id": 1, + "prompt": "Find active enterprise accounts in the US or EU whose email is at company.com (case-insensitive), highest revenue first, newest first as a tiebreak.", + "expected_output": "`engine.find('account', { where: { status: 'active', type: 'enterprise', region: { $in: ['us', 'eu'] }, email: { $icontains: '@company.com' } }, orderBy: [{ field: 'revenue', order: 'desc' }, { field: 'created_at', order: 'desc' }] })` — the object is the FIRST argument, `$icontains` (not `$contains`, which is case-sensitive), `orderBy` as an array of sort nodes.", + "files": [], + "assertions": { + "must_contain": ["engine.find('account'", "$in", "$icontains", "orderBy"], + "must_not_contain": ["$contains:", "joins:"] + } + }, + { + "id": 2, + "prompt": "An infinite-scroll activity feed over millions of `activity` rows, newest first.", + "expected_output": "Keyset paging: a `where` on the sort key past the last row seen (`created_at: { $lt: lastSeenCreatedAt }`), `orderBy: [{ field: 'created_at', order: 'desc' }]` in the same direction, and a `limit`. Not `offset` (the database still scans the skipped rows) and not `query.cursor`, which was removed in 17.", + "files": [], + "assertions": { + "must_contain": ["orderBy", "limit", "$lt", "keyset"], + "must_not_contain": ["cursor:", "offset: page"] + } + }, + { + "id": 3, + "prompt": "Total and average deal amount per region for deals won this year, plus how many distinct owners closed them.", + "expected_output": "`engine.aggregate('deal', { where: { stage: 'closed_won', closed_at: { $gte: ... } }, groupBy: ['region'], aggregations: [{ function: 'sum', field: 'amount', alias: 'total' }, { function: 'avg', field: 'amount', alias: 'avg_amount' }, { function: 'count_distinct', field: 'owner_id', alias: 'owners' }] })`. No `fields` / `orderBy` in the aggregate bag, no `distinct: true` (use `count_distinct`), measures read under their `alias`.", + "files": [], + "assertions": { + "must_contain": ["engine.aggregate(", "groupBy", "aggregations", "count_distinct", "alias"], + "must_not_contain": ["distinct: true", "orderBy:"] + } + }, + { + "id": 4, + "prompt": "Search tasks by keyword across the title and the parent project's name, and show each task's assignee name and email.", + "expected_output": "`engine.find('task', { search: 'term', searchFields: ['title', 'project_name'], fields: ['title', 'status', 'assignee'], expand: { assignee: { object: 'user', fields: ['name', 'email'] } } })`: `search` never traverses, so the project name is a stored mirror field on `task` (objectstack-data), never a dotted `project_id.name`; the projection keeps the `assignee` FK column or `expand` resolves nothing. No `fuzzy` / `boost` / `highlight` knobs — validated then ignored.", + "files": [], + "assertions": { + "must_contain": ["searchFields", "expand", "fields", "stored"], + "must_not_contain": ["'project_id.name'", "fuzzy"] + } + }, + { + "id": 5, + "prompt": "In a `beforeInsert` hook I need to check whether a project with this name already exists, regardless of which organization the caller belongs to.", + "expected_output": "Passes an execution `context`: `engine.find('project', { where: { name }, limit: 1, context: { isSystem: true } })` — without it the read runs as the caller's identity and org-scoping returns fewer rows, indistinguishable from no data. `context` may sit in the query bag or the trailing options argument on reads.", + "files": [], + "assertions": { + "must_contain": ["context", "isSystem: true", "limit: 1"], + "must_not_contain": [] + } + } + ] +} diff --git a/skills/objectstack-ui/evals/README.md b/skills/objectstack-ui/evals/README.md index 4e3f82cd113..4b66ef5f3da 100644 --- a/skills/objectstack-ui/evals/README.md +++ b/skills/objectstack-ui/evals/README.md @@ -1,15 +1,17 @@ # Evaluation Tests (evals/) -Evals validate that AI assistants correctly apply this skill's UI rules when -generating view / page / dashboard / report metadata. +Evals check that AI assistants apply this skill's UI rules when generating +view / page / dashboard / report metadata. ## Current evals - `analytics-inline-vs-dataset.json` — dataset-envelope decisions for - dashboard/report widgets: when a data need fits a `defineDataset` - (dimensions + measures selected by name), when it must escalate to a Cube or - a stored rollup field, and when an ad-hoc in-page `` needs no - dataset at all. + dashboard/report widgets: when a data need fits a `defineDataset`, when it + must escalate to a Cube or a stored rollup field, and when an ad-hoc + in-page `` needs no dataset at all. +- `views-apps-actions-pages.json` — `defineView` containers, `App.create` + navigation, `defineAction` surfaces, record / interface pages, package + docs, wizard forms. ## Format @@ -24,8 +26,6 @@ Each eval file is a JSON fixture with a list of evals, each carrying: When adding evals: -1. Each eval should test a single, specific rule from SKILL.md (e.g. page-owned - `columns`/`sort`/`filterBy` on interface pages, dataset-bound widgets, - action `locations` values) +1. Each eval should test a single, specific rule from SKILL.md 2. Include both positive (correct) and negative (incorrect) patterns 3. Use realistic scenarios from actual ObjectStack projects diff --git a/skills/objectstack-ui/evals/views-apps-actions-pages.json b/skills/objectstack-ui/evals/views-apps-actions-pages.json new file mode 100644 index 00000000000..7cc43608bbe --- /dev/null +++ b/skills/objectstack-ui/evals/views-apps-actions-pages.json @@ -0,0 +1,65 @@ +{ + "skill_name": "objectstack-ui", + "evals": [ + { + "id": 1, + "prompt": "Create `src/views/support-case.view.ts`: a default grid (subject linking to the record, status, priority, owner, due date), a 'My Open Cases' list, a kanban by status, and a simple edit form.", + "expected_output": "One `defineView({ list, listViews, formViews })` container — never a bare flat view object. `data: { provider: 'object', object: 'support_case' }`, `columns` with `link: true` on `subject`, a `listViews.my_open` entry with `filter` triples (`{ field: 'assigned_to', operator: 'equals', value: '{current_user_id}' }`), a kanban entry with the required top-level `columns` plus `kanban: { groupByField: 'status' }` (no top-level `groupBy`), and `formViews.edit` with `type: 'simple'` and `sections` referencing a declared field group.", + "files": [], + "assertions": { + "must_contain": ["defineView", "provider: 'object'", "listViews", "groupByField", "link: true", "{current_user_id}"], + "must_not_contain": ["groupBy:", "conditionalRequired:"] + } + }, + { + "id": 2, + "prompt": "Build `src/apps/crm.app.ts`: a Sales group with Leads, Opportunities, the pipeline kanban, a 'My open deals' entry and the sales dashboard; an Approvals group that only shows when approvals are installed; and a second app for the support team.", + "expected_output": "`App.create({...})` with a `navigation` tree of `group` items: `type: 'object'` entries with `objectName`, a `viewName: 'pipeline_kanban'` shortcut, a bare `filters: { owner_id: '{current_user_id}', status: 'open' }` slice for 'My open deals' (no view authored for a one-off), `type: 'dashboard'` with `dashboardName`, and `requiresObject: 'sys_approval_request'` on the approvals item. Exactly one App per `type: 'app'` package: the support audience is a `requiredPermissions`-gated group inside it, not a second `App.create`. Registered under `defineStack({ apps })`.", + "files": [], + "assertions": { + "must_contain": ["App.create", "navigation", "objectName", "viewName", "filters", "requiresObject", "requiredPermissions"], + "must_not_contain": ["\"path\":", "pageName"] + } + }, + { + "id": 3, + "prompt": "Add a Convert action on `lead` — only for qualified, not-yet-converted leads, from the record header and each list row, launching the `lead_conversion` flow — and a bulk 'Add to campaign' that asks which campaign, for the selected rows.", + "expected_output": "Two `defineAction` items. Convert: `type: 'flow'`, `target: 'lead_conversion'`, `locations: ['record_header', 'list_item']`, `visible: P`record.status == \"qualified\" && record.is_converted == false`` (bare CEL, never brace-wrapped), `confirmText`, `successMessage`, `refreshAfter: true`. Bulk: `type: 'modal'`, `locations: ['list_toolbar']`, a field-backed `params: [{ field: 'campaign_id', objectOverride: 'campaign', required: true }]`, and a `body` that reads `input.selectedIds`.", + "files": [], + "assertions": { + "must_contain": ["defineAction", "locations", "record_header", "list_toolbar", "visible", "successMessage", "selectedIds"], + "must_not_contain": ["{record."] + } + }, + { + "id": 4, + "prompt": "Design a custom lead record page: the Convert action inline in the header, a highlights strip, a stage path over `status`, and the related contacts. Also write a user guide for the `crm` package under src/docs and link it from the nav.", + "expected_output": "`definePage({ name, label, type: 'record', object: 'lead', template: 'three-column', regions: [...] })` with `page:header` (the Action object passed in `properties.actions`, no sibling action node), `record:highlights`, `record:path` (`statusField`, `stages`), `record:related_list`. The guide is a flat, namespace-prefixed `src/docs/crm_user_guide.md` in pure Markdown (MDX and image references are rejected at build), surfaced by a `type: 'url'` nav item pointing at `/docs/crm_user_guide`.", + "files": [], + "assertions": { + "must_contain": ["definePage", "type: 'record'", "regions", "page:header", "record:path", "src/docs/", "/docs/"], + "must_not_contain": ["record_detail", ".mdx"] + } + }, + { + "id": 5, + "prompt": "Sales reps need a locked-down task workbench: only open tasks, sorted by due date, grid only, no switching to other views; plus a report of opportunity amounts by stage.", + "expected_output": "Capability narrowing is the signal for an interface page: `definePage({ type: 'list', object: 'task', interfaceConfig: { source: 'task', columns: [...], sort: [...], filterBy: [...], userFilters: { element: 'dropdown', ... }, appearance: { allowedVisualizations: ['grid'] }, userActions: {...} } })` — the page IS the view definition (no deprecated `sourceView`). The report is `type: 'summary'` selecting `rows` / `values` from a declared `dataset`; there is no `chart` report type.", + "files": [], + "assertions": { + "must_contain": ["definePage", "interfaceConfig", "columns", "allowedVisualizations", "summary", "dataset"], + "must_not_contain": ["sourceView", "type: 'chart'"] + } + }, + { + "id": 6, + "prompt": "Invoice entry as a stepped form — header, then line items, then review — saving the invoice and its lines together.", + "expected_output": "A stepped form over ONE object is `formViews` with `type: 'wizard'` and `sections`, not a screen flow (nothing branches). Line items come from the DATA MODEL: `inlineEdit` on the child's `master_detail` field, which renders the editable grid and saves parent + children in one atomic `/api/v1/batch`; `subforms` only to override derived columns. No custom page, and no 'Add line' button (the ghost row exists).", + "files": [], + "assertions": { + "must_contain": ["type: 'wizard'", "inlineEdit", "formViews", "master_detail"], + "must_not_contain": ["type: 'screen'"] + } + } + ] +} diff --git a/skills/objectstack-upgrade/evals/protocol-major-upgrade.json b/skills/objectstack-upgrade/evals/protocol-major-upgrade.json new file mode 100644 index 00000000000..ebd472a15a7 --- /dev/null +++ b/skills/objectstack-upgrade/evals/protocol-major-upgrade.json @@ -0,0 +1,45 @@ +{ + "skill_name": "objectstack-upgrade", + "evals": [ + { + "id": 1, + "prompt": "把这个项目升级到 v17。objectstack.config.ts 的 manifest 里写着 engines: { protocol: '^16' }。", + "expected_output": "Preflight: `--from 16` read off `manifest.engines.protocol` (the floor of the range, never guessed from the installed spec); keep `os validate > .upgrade/validate-before.txt` as the control. Mechanical: `os migrate meta --from 16 --step`, `--json`, and `--out .upgrade/migrated.stack.json` — the command rewrites nothing on disk, so port the printed `path: from → to (conversionId)` edits into the sources, one `conversionId` per commit, then replay `os migrate meta --from 17 --json` until `applied` is `[]`. Acceptance: `os validate` green, `tsc --noEmit` green, and `.upgrade/REPORT.md` with the mechanical table, decided residue, open decisions and pending per-deployment gates.", + "files": [], + "assertions": { + "must_contain": ["os migrate meta --from 16", "--out", "tsc --noEmit", "REPORT.md", "applied"], + "must_not_contain": ["sed -i", "--from 17 --apply"] + } + }, + { + "id": 2, + "prompt": "After bumping @objectstack/spec, tsc fails: `conditionalRequired` does not exist ... [REMOVED] use `requiredWhen`. Just fix it for me.", + "expected_output": "Does not hand-write the rewrite: the chain already knows the rename, so it runs `os migrate meta --from ` to attribute the site to its `conversionId`, ports `requiredWhen` with the value unchanged, and confirms with `tsc --noEmit` (tombstones type the retired key as `never`). It is a decide-alone item (one target, evidence in the repo, being wrong fails a gate). Never adds a tolerant read or `??` alias so the old key keeps loading; finishes with the prose sweep for `conditionalRequired` in READMEs, comments and the project's AI conventions file.", + "files": [], + "assertions": { + "must_contain": ["requiredWhen", "os migrate meta", "conversionId", "tsc --noEmit"], + "must_not_contain": ["??"] + } + }, + { + "id": 3, + "prompt": "The chain dropped `connectors[sap_erp].fieldMappings[].transform` = { type: 'javascript', expression: 'value / 100' }. Decide what to do with it.", + "expected_output": "Classifies it as R1 (intent choice): the tombstone names one live target (import-mapping `transform`, a string enum with no `javascript` member) while the real options are import mapping, an ETL step where it actually runs, or delete because the member never executed — so the choice is a business statement and the owner decides. Asks with one message carrying the site, the prescription verbatim, the options and their costs, a recommendation and what will be verified; parks the item in the report as `AWAITING DECISION` and keeps the mechanical layer moving.", + "files": [], + "assertions": { + "must_contain": ["AWAITING DECISION", "owner", "never executed", "transform"], + "must_not_contain": ["type: 'javascript'"] + } + }, + { + "id": 4, + "prompt": "We bumped @objectstack/spec 16 → 17 and `os validate` is green, so there was nothing to upgrade, right? Production also has `sys_metadata` rows written under 16.", + "expected_output": "No: `validate` green is necessary, not sufficient — migration-chain-only conversions (16 → 17 has `field-required-notnull-explicit`) reject nothing at parse. Replays `os migrate meta --from 17 --json` and reads `applied` (must be `[]`). Stored rows rehydrate through `applyConversionsToStoredItem` at read time; to make it durable runs `os migrate meta --stored` (read-only) then `os migrate meta --stored --apply --yes` (`--yes` is mandatory in a non-TTY), and never hand-edits `sys_metadata`. Any per-deployment data-migration gates the run prints go into the report as pending, by name.", + "files": [], + "assertions": { + "must_contain": ["applied", "--stored", "--yes", "field-required-notnull-explicit", "applyConversionsToStoredItem"], + "must_not_contain": ["UPDATE sys_metadata"] + } + } + ] +}