From 684b22a154f04a4e3f015eacff739f65ca65a5d7 Mon Sep 17 00:00:00 2001 From: Alice Moore <86723305+3mdistal@users.noreply.github.com> Date: Fri, 21 Aug 2026 00:41:02 +0000 Subject: [PATCH 1/5] fix: preserve Content database property mutations --- .changeset/calm-tools-preserve-inputs.md | 5 ++ packages/core/src/eval/agent-runner.ts | 3 + packages/core/src/eval/runner.spec.ts | 1 + packages/core/src/eval/types.ts | 5 ++ .../content/actions/add-database-item.ts | 4 +- .../content/actions/update-database-item.ts | 2 +- .../actions/upsert-database-item-by-key.ts | 4 +- templates/content/package.json | 1 + templates/content/parity/README.md | 13 +++- .../database-row-property-input.test.ts | 29 ++++++++ .../__tests__/eval-scenario-coverage.test.ts | 68 ++++++++++++++++++- templates/content/parity/eval-scenarios.ts | 22 ++++++ ...n-database-create-property-preservation.ts | 45 ++++++++++++ templates/content/parity/scenario-to-eval.ts | 56 +++++++++++++++ 14 files changed, 250 insertions(+), 8 deletions(-) create mode 100644 .changeset/calm-tools-preserve-inputs.md create mode 100644 templates/content/parity/__tests__/database-row-property-input.test.ts create mode 100644 templates/content/parity/run-database-create-property-preservation.ts diff --git a/.changeset/calm-tools-preserve-inputs.md b/.changeset/calm-tools-preserve-inputs.md new file mode 100644 index 0000000000..268e397a4a --- /dev/null +++ b/.changeset/calm-tools-preserve-inputs.md @@ -0,0 +1,5 @@ +--- +"@agent-native/core": patch +--- + +Expose model-produced tool inputs to eval scorers so argument-level agent behavior can be verified. diff --git a/packages/core/src/eval/agent-runner.ts b/packages/core/src/eval/agent-runner.ts index b8599bd85b..3b370ae7f2 100644 --- a/packages/core/src/eval/agent-runner.ts +++ b/packages/core/src/eval/agent-runner.ts @@ -120,6 +120,7 @@ export async function createAgentRunner( let text = ""; const toolCalls: string[] = []; + const toolCallDetails: Array<{ name: string; input: unknown }> = []; let ok = true; let error: string | undefined; @@ -134,6 +135,7 @@ export async function createAgentRunner( break; case "tool_start": toolCalls.push(event.tool); + toolCallDetails.push({ name: event.tool, input: event.input }); break; case "error": ok = false; @@ -165,6 +167,7 @@ export async function createAgentRunner( return { text, toolCalls, + toolCallDetails, ok, error, runId, diff --git a/packages/core/src/eval/runner.spec.ts b/packages/core/src/eval/runner.spec.ts index fef74b3690..3fa997625c 100644 --- a/packages/core/src/eval/runner.spec.ts +++ b/packages/core/src/eval/runner.spec.ts @@ -299,6 +299,7 @@ describe("createAgentRunner over a mocked runAgentLoop (no real model)", () => { const out = await runner.runAgent({ prompt: "hi" }); expect(out.text).toBe("Hello world"); expect(out.toolCalls).toEqual(["search"]); + expect(out.toolCallDetails).toEqual([{ name: "search", input: {} }]); expect(out.ok).toBe(true); // End-to-end: a contains scorer over the real collected text. diff --git a/packages/core/src/eval/types.ts b/packages/core/src/eval/types.ts index 3648416981..a99a0849a9 100644 --- a/packages/core/src/eval/types.ts +++ b/packages/core/src/eval/types.ts @@ -31,6 +31,11 @@ export interface AgentRunOutput { readonly text: string; /** Names of tools/actions the agent invoked, in call order. */ readonly toolCalls: readonly string[]; + /** Tool names and model-produced inputs, in call order. */ + readonly toolCallDetails?: readonly { + readonly name: string; + readonly input: unknown; + }[]; /** Whether the run completed without a terminal error event. */ readonly ok: boolean; /** Terminal error message, if the run errored. */ diff --git a/templates/content/actions/add-database-item.ts b/templates/content/actions/add-database-item.ts index 318f9984a2..35ca108741 100644 --- a/templates/content/actions/add-database-item.ts +++ b/templates/content/actions/add-database-item.ts @@ -20,7 +20,9 @@ const schema = databaseMutationEnvelopeSchema.extend({ propertyValues: z .record(z.string(), z.unknown()) .optional() - .describe("Strict property values keyed by property definition ID"), + .describe( + "Strict property values keyed by property definition ID. Use exact property definition IDs as keys. Include every schema-valid writable property value the user explicitly requested; when the request contains at least one such value, never pass an empty object. Do not invent or clear unmentioned properties.", + ), }); export default defineAction({ diff --git a/templates/content/actions/update-database-item.ts b/templates/content/actions/update-database-item.ts index 889c33c15d..3f79784ffc 100644 --- a/templates/content/actions/update-database-item.ts +++ b/templates/content/actions/update-database-item.ts @@ -20,7 +20,7 @@ const schema = databaseMutationEnvelopeSchema.extend({ .record(z.string(), z.unknown()) .optional() .describe( - "Sparse strict patch keyed by property definition ID; omitted fields are preserved and explicit null clears a value", + "Sparse strict patch keyed by property definition ID; omitted fields are preserved and explicit null clears a value. Use exact property definition IDs as keys. Include every schema-valid writable property value the user explicitly requested; when the request contains at least one such value, never pass an empty object. Do not invent or clear unmentioned properties.", ), }); diff --git a/templates/content/actions/upsert-database-item-by-key.ts b/templates/content/actions/upsert-database-item-by-key.ts index c9a86fb303..334d57f055 100644 --- a/templates/content/actions/upsert-database-item-by-key.ts +++ b/templates/content/actions/upsert-database-item-by-key.ts @@ -21,7 +21,9 @@ const schema = databaseMutationEnvelopeSchema.extend({ propertyValues: z .record(z.string(), z.unknown()) .optional() - .describe("Sparse strict values keyed by property definition ID"), + .describe( + "Sparse strict values keyed by property definition ID. Use exact property definition IDs as keys. Include every schema-valid writable property value the user explicitly requested; when the request contains at least one such value, never pass an empty object. Do not invent or clear unmentioned properties.", + ), }); export default defineAction({ diff --git a/templates/content/package.json b/templates/content/package.json index 5c5e767d47..6cead972bc 100644 --- a/templates/content/package.json +++ b/templates/content/package.json @@ -12,6 +12,7 @@ "test:parity": "vitest --run parity", "test:parity-capabilities": "vitest --run parity actions/content-database-lifecycle.db.test.ts actions/bind-content-database-source-field.db.test.ts actions/_local-file-documents.test.ts actions/builder-source-review-gates.db.test.ts", "eval:parity": "agent-native eval parity", + "eval:property-preservation": "tsx parity/run-database-create-property-preservation.ts", "format.fix": "oxfmt --write .", "typecheck": "agent-native typecheck", "migrate:production": "tsx scripts/migrate-production.ts", diff --git a/templates/content/parity/README.md b/templates/content/parity/README.md index 05ac6c5b35..4c84876e57 100644 --- a/templates/content/parity/README.md +++ b/templates/content/parity/README.md @@ -10,7 +10,7 @@ PR 2.2 adds two executable tiers: existing action tests, make no model calls, and require no private provider credentials. - Gated agent evals run through `agent-native eval parity`. They are opt-in via - `CONTENT_PARITY_EVALS=1`, capped to four initial scenarios, and should be + `CONTENT_PARITY_EVALS=1`, kept to a small explicit scenario set, and should be reserved for manual or nightly checks. ## Deterministic Checks @@ -33,14 +33,23 @@ cd templates/content CONTENT_PARITY_EVALS=1 ANTHROPIC_API_KEY=... ./node_modules/.bin/agent-native eval parity ``` +The database-create property-preservation regression has a dedicated +fixture-only runner so it can inspect model-produced arguments without loading +or executing unrelated Content actions: + +```bash +CONTENT_PARITY_EVALS=1 pnpm eval:property-preservation +``` + With `CONTENT_PARITY_EVALS` unset, parity evals return skipped rows and do not call the agent runner. The CLI still exits `0`, but both readable and JSON reports mark each row with `status: "skipped"` and a `skipReason` such as `Skipped because CONTENT_PARITY_EVALS is unset`. -With the gate set, the eval files run the four PR 2.2 scenarios: +With the gate set, the eval files include these scenarios: - `database-source-scope` +- `database-create-property-preservation` - `document-search-edit` - `local-file-source-truth` - `builder-source-review-readonly` diff --git a/templates/content/parity/__tests__/database-row-property-input.test.ts b/templates/content/parity/__tests__/database-row-property-input.test.ts new file mode 100644 index 0000000000..e3ed8af8a2 --- /dev/null +++ b/templates/content/parity/__tests__/database-row-property-input.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from "vitest"; + +import addDatabaseItem from "../../actions/add-database-item"; +import updateDatabaseItem from "../../actions/update-database-item"; +import upsertDatabaseItemByKey from "../../actions/upsert-database-item-by-key"; + +const rowMutationActions = [ + ["add-database-item", addDatabaseItem], + ["update-database-item", updateDatabaseItem], + ["upsert-database-item-by-key", upsertDatabaseItemByKey], +] as const; + +describe("database row property inputs", () => { + it.each(rowMutationActions)( + "%s tells the agent to preserve explicitly requested writable values", + (_name, action) => { + const propertyValues = action.tool.parameters.properties.propertyValues; + expect(propertyValues.description).toContain( + "Include every schema-valid writable property value the user explicitly requested", + ); + expect(propertyValues.description).toContain( + "never pass an empty object", + ); + expect(propertyValues.description).toContain( + "Do not invent or clear unmentioned properties", + ); + }, + ); +}); diff --git a/templates/content/parity/__tests__/eval-scenario-coverage.test.ts b/templates/content/parity/__tests__/eval-scenario-coverage.test.ts index 65fb938762..b3d39b7935 100644 --- a/templates/content/parity/__tests__/eval-scenario-coverage.test.ts +++ b/templates/content/parity/__tests__/eval-scenario-coverage.test.ts @@ -37,10 +37,11 @@ describe("Content parity eval scenarios", () => { expect(invalid).toEqual([]); }); - it("keeps PR 2.5 capped to the five bundled gated scenarios", () => { + it("keeps the bundled gated scenarios explicit", () => { expect(parityEvalScenarios.map((scenario) => scenario.id).sort()).toEqual([ "builder-source-review-readonly", "database-bulk-row-reliability", + "database-create-property-preservation", "database-source-scope", "document-search-edit", "local-file-source-truth", @@ -83,14 +84,16 @@ describe("Content parity eval scenarios", () => { ); expect(report.failed).toBe(0); - expect(report.skipped).toBe(5); + expect(report.skipped).toBe(6); expect(report.results.every((row) => row.status === "skipped")).toBe(true); }); it("runs scorer-backed evals when the gate is set", async () => { process.env.CONTENT_PARITY_EVALS = "1"; - const scenario = parityEvalScenarios[0]; + const scenario = parityEvalScenarios.find( + (candidate) => candidate.id === "database-source-scope", + )!; const evalCase = scenarioToEval(scenario); const row = await scoreEval(evalCase, { runAgent: vi.fn(async () => ({ @@ -141,4 +144,63 @@ describe("Content parity eval scenarios", () => { expect(expectedToolsScore?.reason).toContain("remove-database-items"); expect(row.status).toBe("failed"); }); + + it("fails when database creation drops explicitly requested properties", async () => { + process.env.CONTENT_PARITY_EVALS = "1"; + const scenario = parityEvalScenarios.find( + (candidate) => candidate.id === "database-create-property-preservation", + )!; + const evalCase = scenarioToEval(scenario); + const row = await scoreEval(evalCase, { + runAgent: vi.fn(async () => ({ + text: scenario.successSignals.join("\n"), + toolCalls: ["add-database-item"], + toolCallDetails: [ + { name: "add-database-item", input: { propertyValues: {} } }, + ], + ok: true, + runId: "content-parity:empty-property-values", + durationMs: 1, + })), + engine: {} as never, + model: "test-model", + analyzeContext: vi.fn(), + }); + + expect( + row.scores.find((score) => score.scorer === "expected_property_values"), + ).toMatchObject({ passed: false, score: 0 }); + expect(row.status).toBe("failed"); + }); + + it("accepts exact database creation properties without extras", async () => { + process.env.CONTENT_PARITY_EVALS = "1"; + const scenario = parityEvalScenarios.find( + (candidate) => candidate.id === "database-create-property-preservation", + )!; + const evalCase = scenarioToEval(scenario); + const row = await scoreEval(evalCase, { + runAgent: vi.fn(async () => ({ + text: scenario.successSignals.join("\n"), + toolCalls: ["add-database-item"], + toolCallDetails: [ + { + name: "add-database-item", + input: { propertyValues: scenario.expectedPropertyValues }, + }, + ], + ok: true, + runId: "content-parity:exact-property-values", + durationMs: 1, + })), + engine: {} as never, + model: "test-model", + analyzeContext: vi.fn(), + }); + + expect( + row.scores.find((score) => score.scorer === "expected_property_values"), + ).toMatchObject({ passed: true, score: 1 }); + expect(row.status).toBe("passed"); + }); }); diff --git a/templates/content/parity/eval-scenarios.ts b/templates/content/parity/eval-scenarios.ts index b8bcfb76dd..996de81c62 100644 --- a/templates/content/parity/eval-scenarios.ts +++ b/templates/content/parity/eval-scenarios.ts @@ -8,9 +8,31 @@ export interface ParityEvalScenario { prompt: string; successSignals: string[]; expectedTools?: string[]; + expectedPropertyValues?: Record; } export const parityEvalScenarios: ParityEvalScenario[] = [ + { + id: "database-create-property-preservation", + title: "Database create property preservation", + capabilityIds: ["database.rows"], + gateEnv: "CONTENT_PARITY_EVALS", + defaultState: "skipped", + requiresPrivateCredentials: false, + prompt: + "Create exactly one row in the already-verified fixture Content database. Do not rediscover or alter the target. Use target spaceId fixture_personal_space, databaseId fixture_feedback_database, and documentId fixture_feedback_document; expectedSchemaRevision fixture_schema_revision; title [FIXTURE] preserve explicit properties; idempotencyKey fixture-create-property-preservation-v1. Set property fixture_status_property to status-cannot-verify and property fixture_evidence_property to Baseline fixture preserve-me. No Blocks value was requested. Call add-database-item once with every exact target constraint and both property values, then report its result truthfully.", + successSignals: [ + "Uses add-database-item once for the exact fixture target.", + "Preserves both explicitly requested writable property values.", + "Does not invent a Blocks value or another property.", + "Reports an action failure rather than claiming a row was created if the fixture is unavailable.", + ], + expectedTools: ["add-database-item"], + expectedPropertyValues: { + fixture_status_property: "status-cannot-verify", + fixture_evidence_property: "Baseline fixture preserve-me", + }, + }, { id: "database-bulk-row-reliability", title: "Bulk database row reliability", diff --git a/templates/content/parity/run-database-create-property-preservation.ts b/templates/content/parity/run-database-create-property-preservation.ts new file mode 100644 index 0000000000..1713a9fd52 --- /dev/null +++ b/templates/content/parity/run-database-create-property-preservation.ts @@ -0,0 +1,45 @@ +import { createAgentRunner, runEvals } from "@agent-native/core/eval"; + +import addDatabaseItem from "../actions/add-database-item.ts"; +import { parityEvalScenarios } from "./eval-scenarios.ts"; +import { scenarioToEval } from "./scenario-to-eval.ts"; + +const scenario = parityEvalScenarios.find( + (candidate) => candidate.id === "database-create-property-preservation", +); +if (!scenario) { + throw new Error("Missing database create property preservation scenario."); +} + +const evalCase = scenarioToEval(scenario); +evalCase.scorers = evalCase.scorers.filter( + (scorer) => + scorer.name === "expected_tools" || + scorer.name === "expected_property_values", +); + +const runner = await createAgentRunner({ + actions: { + "add-database-item": { + ...addDatabaseItem, + run: async (input) => ({ fixtureOnly: true, received: input }), + }, + }, + systemPrompt: + "You are Content's AI document assistant. Use the registered Content action and preserve exact user-supplied target constraints, property IDs, and property values. Never invent fields or claim an action succeeded when it failed.", +}); + +const report = await runEvals([evalCase], runner, { persist: false }); +console.log( + JSON.stringify( + { + engine: runner.engine.name, + model: runner.model, + report, + }, + null, + 2, + ), +); + +process.exitCode = report.failed === 0 ? 0 : 1; diff --git a/templates/content/parity/scenario-to-eval.ts b/templates/content/parity/scenario-to-eval.ts index 6385b1d8c2..103d45cab1 100644 --- a/templates/content/parity/scenario-to-eval.ts +++ b/templates/content/parity/scenario-to-eval.ts @@ -30,6 +30,59 @@ function expectedToolScorer(expectedTools: string[]) { }); } +function normalizePropertyValues(input: unknown): Record { + if (!input || typeof input !== "object" || Array.isArray(input)) return {}; + const propertyValues = (input as Record).propertyValues; + if (!propertyValues) return {}; + if (!Array.isArray(propertyValues)) { + return typeof propertyValues === "object" + ? (propertyValues as Record) + : {}; + } + return Object.fromEntries( + propertyValues.flatMap((entry) => { + if (!entry || typeof entry !== "object") return []; + const { propertyId, value } = entry as Record; + return typeof propertyId === "string" ? [[propertyId, value]] : []; + }), + ); +} + +function expectedPropertyValuesScorer(expected: Record) { + return createScorer< + AgentRunOutput, + { + received: Record; + missing: string[]; + unexpected: string[]; + } + >({ + name: "expected_property_values", + analyze(run) { + const detail = run.toolCallDetails?.find( + (call) => call.name === "add-database-item", + ); + const received = normalizePropertyValues(detail?.input); + const missing = Object.entries(expected) + .filter(([propertyId, value]) => received[propertyId] !== value) + .map(([propertyId]) => propertyId); + const unexpected = Object.keys(received).filter( + (propertyId) => !(propertyId in expected), + ); + return { received, missing, unexpected }; + }, + generateScore({ missing, unexpected }) { + return missing.length === 0 && unexpected.length === 0 ? 1 : 0; + }, + generateReason({ analysis: { received, missing, unexpected } }) { + if (missing.length === 0 && unexpected.length === 0) { + return "Agent preserved every expected property ID and exact value without inventing another property."; + } + return `Received propertyValues ${JSON.stringify(received)}; missing or changed: ${missing.join(", ") || "none"}; unexpected: ${unexpected.join(", ") || "none"}`; + }, + }); +} + export function scenarioToEval(scenario: ParityEvalScenario): Eval { const name = `content-parity:${scenario.id}`; @@ -52,6 +105,9 @@ export function scenarioToEval(scenario: ParityEvalScenario): Eval { ...(scenario.expectedTools?.length ? [expectedToolScorer(scenario.expectedTools)] : []), + ...(scenario.expectedPropertyValues + ? [expectedPropertyValuesScorer(scenario.expectedPropertyValues)] + : []), ], }); } From 74eb67f387a624a2555159a14c0c2f4cf086d6e5 Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Thu, 20 Aug 2026 21:17:39 -0400 Subject: [PATCH 2/5] fix: normalize Content AI property entries --- .../actions/_database-property-input.ts | 53 +++++++++++++++++++ .../content/actions/add-database-item.ts | 19 ++++--- .../content/actions/update-database-item.ts | 22 +++++--- .../actions/upsert-database-item-by-key.ts | 22 +++++--- .../database-row-property-input.test.ts | 53 ++++++++++++++++--- templates/content/parity/eval-scenarios.ts | 2 +- templates/content/parity/scenario-to-eval.ts | 4 +- 7 files changed, 146 insertions(+), 29 deletions(-) create mode 100644 templates/content/actions/_database-property-input.ts diff --git a/templates/content/actions/_database-property-input.ts b/templates/content/actions/_database-property-input.ts new file mode 100644 index 0000000000..dd98815eb2 --- /dev/null +++ b/templates/content/actions/_database-property-input.ts @@ -0,0 +1,53 @@ +import { ActionContractError } from "@agent-native/core"; +import { z } from "zod"; + +export const databasePropertyValuesSchema = z + .record(z.string(), z.unknown()) + .optional() + .describe( + "Programmatic property values keyed by exact property definition ID.", + ); + +export const databasePropertyEntriesSchema = z + .array( + z.object({ + propertyId: z + .string() + .min(1) + .describe("Exact immutable property definition ID"), + value: z.unknown().describe("Schema-valid value for this property"), + }), + ) + .max(1_000) + .optional() + .describe( + "Property values as explicit entries. Include one entry for every schema-valid writable property value the user requested, using the exact immutable property definition ID. When at least one value was requested, never pass an empty array. Do not invent or clear unmentioned properties.", + ); + +export function normalizeDatabasePropertyInput(input: { + propertyEntries?: Array<{ propertyId: string; value: unknown }>; + propertyValues?: Record; +}): Record | undefined { + if (input.propertyEntries && input.propertyValues) { + throw new ActionContractError( + "Provide propertyEntries or propertyValues, not both.", + { errorCode: "AMBIGUOUS_PROPERTY_INPUT" }, + ); + } + if (!input.propertyEntries) return input.propertyValues; + + const values: Record = {}; + for (const entry of input.propertyEntries) { + if (Object.prototype.hasOwnProperty.call(values, entry.propertyId)) { + throw new ActionContractError( + `Property entry ${entry.propertyId} was provided more than once.`, + { + errorCode: "DUPLICATE_PROPERTY_INPUT", + details: { propertyId: entry.propertyId }, + }, + ); + } + values[entry.propertyId] = entry.value; + } + return values; +} diff --git a/templates/content/actions/add-database-item.ts b/templates/content/actions/add-database-item.ts index 35ca108741..629872af8b 100644 --- a/templates/content/actions/add-database-item.ts +++ b/templates/content/actions/add-database-item.ts @@ -3,6 +3,11 @@ import { buildDeepLink } from "@agent-native/core/server"; import { z } from "zod"; import type { ContentDatabaseRowMutationResult } from "../shared/api.js"; +import { + databasePropertyEntriesSchema, + databasePropertyValuesSchema, + normalizeDatabasePropertyInput, +} from "./_database-property-input.js"; import { createDatabaseRow, databaseMutationEnvelopeSchema, @@ -17,17 +22,14 @@ const schema = databaseMutationEnvelopeSchema.extend({ .max(500) .optional() .describe("New row page title"), - propertyValues: z - .record(z.string(), z.unknown()) - .optional() - .describe( - "Strict property values keyed by property definition ID. Use exact property definition IDs as keys. Include every schema-valid writable property value the user explicitly requested; when the request contains at least one such value, never pass an empty object. Do not invent or clear unmentioned properties.", - ), + propertyValues: databasePropertyValuesSchema, + propertyEntries: databasePropertyEntriesSchema, }); export default defineAction({ description: "Create one row in an exact ordinary Content database using its discovered schema revision. Strictly validates every non-Blocks property, applies the side effect once per idempotency key, and returns a verified receipt with stable row identity.", + agentInputSchema: schema.omit({ propertyValues: true }), publicAgent: { expose: true, readOnly: false, @@ -54,7 +56,10 @@ export default defineAction({ }, }, run: async (args): Promise => { - const result = await createDatabaseRow(args); + const result = await createDatabaseRow({ + ...args, + propertyValues: normalizeDatabasePropertyInput(args), + }); const response = await getContentDatabaseResponse( result.receipt.target.databaseId, { diff --git a/templates/content/actions/update-database-item.ts b/templates/content/actions/update-database-item.ts index 3f79784ffc..66bac08d6b 100644 --- a/templates/content/actions/update-database-item.ts +++ b/templates/content/actions/update-database-item.ts @@ -3,6 +3,11 @@ import { buildDeepLink } from "@agent-native/core/server"; import { z } from "zod"; import type { ContentDatabaseRowMutationResult } from "../shared/api.js"; +import { + databasePropertyEntriesSchema, + databasePropertyValuesSchema, + normalizeDatabasePropertyInput, +} from "./_database-property-input.js"; import { databaseMutationEnvelopeSchema, updateDatabaseRow, @@ -16,17 +21,16 @@ const schema = databaseMutationEnvelopeSchema.extend({ .min(1) .describe("Row revision returned by get-content-database"), title: z.string().trim().min(1).max(500).optional(), - propertyValues: z - .record(z.string(), z.unknown()) - .optional() - .describe( - "Sparse strict patch keyed by property definition ID; omitted fields are preserved and explicit null clears a value. Use exact property definition IDs as keys. Include every schema-valid writable property value the user explicitly requested; when the request contains at least one such value, never pass an empty object. Do not invent or clear unmentioned properties.", - ), + propertyValues: databasePropertyValuesSchema, + propertyEntries: databasePropertyEntriesSchema.describe( + "Sparse property patch as explicit entries; omitted fields are preserved and explicit null clears a value. Include one entry for every schema-valid writable property value the user requested, using the exact immutable property definition ID. When at least one value was requested, never pass an empty array. Do not invent or clear unmentioned properties.", + ), }); export default defineAction({ description: "Sparsely update one exact Content database row by stable item and document IDs. Requires schema and row revisions, validates every non-Blocks property, and returns a verified idempotent receipt.", + agentInputSchema: schema.omit({ propertyValues: true }), schema, http: { method: "PUT" }, audit: { @@ -44,7 +48,11 @@ export default defineAction({ : "Updated Content database row"; }, }, - run: updateDatabaseRow, + run: (args) => + updateDatabaseRow({ + ...args, + propertyValues: normalizeDatabasePropertyInput(args), + }), link: ({ result }) => { const documentId = (result as ContentDatabaseRowMutationResult | null) ?.receipt.row.documentId; diff --git a/templates/content/actions/upsert-database-item-by-key.ts b/templates/content/actions/upsert-database-item-by-key.ts index 334d57f055..8dbcb7f603 100644 --- a/templates/content/actions/upsert-database-item-by-key.ts +++ b/templates/content/actions/upsert-database-item-by-key.ts @@ -3,6 +3,11 @@ import { buildDeepLink } from "@agent-native/core/server"; import { z } from "zod"; import type { ContentDatabaseRowMutationResult } from "../shared/api.js"; +import { + databasePropertyEntriesSchema, + databasePropertyValuesSchema, + normalizeDatabasePropertyInput, +} from "./_database-property-input.js"; import { databaseMutationEnvelopeSchema, upsertDatabaseRow, @@ -18,17 +23,16 @@ const schema = databaseMutationEnvelopeSchema.extend({ "Use null to assert the key is absent and create; use the discovered row revision to update an existing key", ), title: z.string().trim().min(1).max(500).optional(), - propertyValues: z - .record(z.string(), z.unknown()) - .optional() - .describe( - "Sparse strict values keyed by property definition ID. Use exact property definition IDs as keys. Include every schema-valid writable property value the user explicitly requested; when the request contains at least one such value, never pass an empty object. Do not invent or clear unmentioned properties.", - ), + propertyValues: databasePropertyValuesSchema, + propertyEntries: databasePropertyEntriesSchema.describe( + "Sparse property values as explicit entries. Include one entry for every schema-valid writable property value the user requested, using the exact immutable property definition ID. When at least one value was requested, never pass an empty array. Do not invent or clear unmentioned properties.", + ), }); export default defineAction({ description: "Create or sparsely update one Content database row by that database's explicitly configured natural key. Requires schema and row compare-and-swap revisions and returns a verified idempotent receipt.", + agentInputSchema: schema.omit({ propertyValues: true }), schema, audit: { recordInputs: false, @@ -45,7 +49,11 @@ export default defineAction({ : "Upserted Content database row by natural key"; }, }, - run: upsertDatabaseRow, + run: (args) => + upsertDatabaseRow({ + ...args, + propertyValues: normalizeDatabasePropertyInput(args), + }), link: ({ result }) => { const documentId = (result as ContentDatabaseRowMutationResult | null) ?.receipt.row.documentId; diff --git a/templates/content/parity/__tests__/database-row-property-input.test.ts b/templates/content/parity/__tests__/database-row-property-input.test.ts index e3ed8af8a2..2c2338289e 100644 --- a/templates/content/parity/__tests__/database-row-property-input.test.ts +++ b/templates/content/parity/__tests__/database-row-property-input.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; +import { normalizeDatabasePropertyInput } from "../../actions/_database-property-input"; import addDatabaseItem from "../../actions/add-database-item"; import updateDatabaseItem from "../../actions/update-database-item"; import upsertDatabaseItemByKey from "../../actions/upsert-database-item-by-key"; @@ -14,16 +15,56 @@ describe("database row property inputs", () => { it.each(rowMutationActions)( "%s tells the agent to preserve explicitly requested writable values", (_name, action) => { - const propertyValues = action.tool.parameters.properties.propertyValues; - expect(propertyValues.description).toContain( - "Include every schema-valid writable property value the user explicitly requested", + const properties = action.tool.parameters.properties; + expect(properties).not.toHaveProperty("propertyValues"); + const propertyEntries = properties.propertyEntries; + expect(propertyEntries.type).toBe("array"); + expect(propertyEntries.items.properties.propertyId.description).toContain( + "Exact immutable property definition ID", ); - expect(propertyValues.description).toContain( - "never pass an empty object", + expect(propertyEntries.description).toContain( + "Include one entry for every schema-valid writable property value the user requested", ); - expect(propertyValues.description).toContain( + expect(propertyEntries.description).toContain( + "never pass an empty array", + ); + expect(propertyEntries.description).toContain( "Do not invent or clear unmentioned properties", ); }, ); + + it("normalizes model-friendly entries into the strict action contract", () => { + expect( + normalizeDatabasePropertyInput({ + propertyEntries: [ + { propertyId: "status-id", value: "ready" }, + { propertyId: "evidence-id", value: "preserve me" }, + ], + }), + ).toEqual({ + "status-id": "ready", + "evidence-id": "preserve me", + }); + }); + + it("rejects duplicate property entries instead of silently overwriting", () => { + expect(() => + normalizeDatabasePropertyInput({ + propertyEntries: [ + { propertyId: "status-id", value: "ready" }, + { propertyId: "status-id", value: "changed" }, + ], + }), + ).toThrow(/provided more than once/); + }); + + it("rejects ambiguous entry and record inputs", () => { + expect(() => + normalizeDatabasePropertyInput({ + propertyEntries: [{ propertyId: "status-id", value: "ready" }], + propertyValues: { "status-id": "ready" }, + }), + ).toThrow(/not both/); + }); }); diff --git a/templates/content/parity/eval-scenarios.ts b/templates/content/parity/eval-scenarios.ts index 996de81c62..c459e1f376 100644 --- a/templates/content/parity/eval-scenarios.ts +++ b/templates/content/parity/eval-scenarios.ts @@ -20,7 +20,7 @@ export const parityEvalScenarios: ParityEvalScenario[] = [ defaultState: "skipped", requiresPrivateCredentials: false, prompt: - "Create exactly one row in the already-verified fixture Content database. Do not rediscover or alter the target. Use target spaceId fixture_personal_space, databaseId fixture_feedback_database, and documentId fixture_feedback_document; expectedSchemaRevision fixture_schema_revision; title [FIXTURE] preserve explicit properties; idempotencyKey fixture-create-property-preservation-v1. Set property fixture_status_property to status-cannot-verify and property fixture_evidence_property to Baseline fixture preserve-me. No Blocks value was requested. Call add-database-item once with every exact target constraint and both property values, then report its result truthfully.", + "Create exactly one row in the already-verified fixture Content database. Do not rediscover or alter the target. Use target spaceId fixture_personal_space, databaseId fixture_feedback_database, and documentId fixture_feedback_document; expectedSchemaRevision fixture_schema_revision; title [FIXTURE] preserve explicit properties; idempotencyKey fixture-create-property-preservation-v1. Set property fixture_status_property to status-cannot-verify and property fixture_evidence_property to Baseline fixture preserve-me. No Blocks value was requested. Call add-database-item once with every exact target constraint and both property entries, then report its result truthfully.", successSignals: [ "Uses add-database-item once for the exact fixture target.", "Preserves both explicitly requested writable property values.", diff --git a/templates/content/parity/scenario-to-eval.ts b/templates/content/parity/scenario-to-eval.ts index 103d45cab1..c657048da4 100644 --- a/templates/content/parity/scenario-to-eval.ts +++ b/templates/content/parity/scenario-to-eval.ts @@ -32,7 +32,9 @@ function expectedToolScorer(expectedTools: string[]) { function normalizePropertyValues(input: unknown): Record { if (!input || typeof input !== "object" || Array.isArray(input)) return {}; - const propertyValues = (input as Record).propertyValues; + const propertyValues = + (input as Record).propertyEntries ?? + (input as Record).propertyValues; if (!propertyValues) return {}; if (!Array.isArray(propertyValues)) { return typeof propertyValues === "object" From fd85e17054cb04954cd150e1485e44bc94b36f0b Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Fri, 21 Aug 2026 07:43:48 -0400 Subject: [PATCH 3/5] fix: canonicalize Content mutation inputs --- .../actions/_database-property-input.ts | 20 +++ .../content/actions/add-database-item.ts | 9 +- .../content/actions/update-database-item.ts | 8 +- .../actions/upsert-database-item-by-key.ts | 8 +- .../database-row-property-input.test.ts | 44 ++++++- .../__tests__/eval-scenario-coverage.test.ts | 75 +++++++++++ templates/content/parity/scenario-to-eval.ts | 119 ++++++++++++++---- 7 files changed, 240 insertions(+), 43 deletions(-) diff --git a/templates/content/actions/_database-property-input.ts b/templates/content/actions/_database-property-input.ts index dd98815eb2..30552617c3 100644 --- a/templates/content/actions/_database-property-input.ts +++ b/templates/content/actions/_database-property-input.ts @@ -51,3 +51,23 @@ export function normalizeDatabasePropertyInput(input: { } return values; } + +export function canonicalizeDatabasePropertyInput< + T extends { + propertyEntries?: Array<{ propertyId: string; value: unknown }>; + propertyValues?: Record; + }, +>( + input: T, +): Omit & { + propertyValues?: Record; +} { + const { propertyEntries, propertyValues, ...canonicalInput } = input; + return { + ...canonicalInput, + propertyValues: normalizeDatabasePropertyInput({ + propertyEntries, + propertyValues, + }), + }; +} diff --git a/templates/content/actions/add-database-item.ts b/templates/content/actions/add-database-item.ts index 629872af8b..8666c33bdf 100644 --- a/templates/content/actions/add-database-item.ts +++ b/templates/content/actions/add-database-item.ts @@ -4,9 +4,9 @@ import { z } from "zod"; import type { ContentDatabaseRowMutationResult } from "../shared/api.js"; import { + canonicalizeDatabasePropertyInput, databasePropertyEntriesSchema, databasePropertyValuesSchema, - normalizeDatabasePropertyInput, } from "./_database-property-input.js"; import { createDatabaseRow, @@ -56,10 +56,9 @@ export default defineAction({ }, }, run: async (args): Promise => { - const result = await createDatabaseRow({ - ...args, - propertyValues: normalizeDatabasePropertyInput(args), - }); + const result = await createDatabaseRow( + canonicalizeDatabasePropertyInput(args), + ); const response = await getContentDatabaseResponse( result.receipt.target.databaseId, { diff --git a/templates/content/actions/update-database-item.ts b/templates/content/actions/update-database-item.ts index 66bac08d6b..4fcd904e53 100644 --- a/templates/content/actions/update-database-item.ts +++ b/templates/content/actions/update-database-item.ts @@ -4,9 +4,9 @@ import { z } from "zod"; import type { ContentDatabaseRowMutationResult } from "../shared/api.js"; import { + canonicalizeDatabasePropertyInput, databasePropertyEntriesSchema, databasePropertyValuesSchema, - normalizeDatabasePropertyInput, } from "./_database-property-input.js"; import { databaseMutationEnvelopeSchema, @@ -48,11 +48,7 @@ export default defineAction({ : "Updated Content database row"; }, }, - run: (args) => - updateDatabaseRow({ - ...args, - propertyValues: normalizeDatabasePropertyInput(args), - }), + run: (args) => updateDatabaseRow(canonicalizeDatabasePropertyInput(args)), link: ({ result }) => { const documentId = (result as ContentDatabaseRowMutationResult | null) ?.receipt.row.documentId; diff --git a/templates/content/actions/upsert-database-item-by-key.ts b/templates/content/actions/upsert-database-item-by-key.ts index 8dbcb7f603..93dcb1ef43 100644 --- a/templates/content/actions/upsert-database-item-by-key.ts +++ b/templates/content/actions/upsert-database-item-by-key.ts @@ -4,9 +4,9 @@ import { z } from "zod"; import type { ContentDatabaseRowMutationResult } from "../shared/api.js"; import { + canonicalizeDatabasePropertyInput, databasePropertyEntriesSchema, databasePropertyValuesSchema, - normalizeDatabasePropertyInput, } from "./_database-property-input.js"; import { databaseMutationEnvelopeSchema, @@ -49,11 +49,7 @@ export default defineAction({ : "Upserted Content database row by natural key"; }, }, - run: (args) => - upsertDatabaseRow({ - ...args, - propertyValues: normalizeDatabasePropertyInput(args), - }), + run: (args) => upsertDatabaseRow(canonicalizeDatabasePropertyInput(args)), link: ({ result }) => { const documentId = (result as ContentDatabaseRowMutationResult | null) ?.receipt.row.documentId; diff --git a/templates/content/parity/__tests__/database-row-property-input.test.ts b/templates/content/parity/__tests__/database-row-property-input.test.ts index 2c2338289e..aaa40f1e3d 100644 --- a/templates/content/parity/__tests__/database-row-property-input.test.ts +++ b/templates/content/parity/__tests__/database-row-property-input.test.ts @@ -1,6 +1,10 @@ import { describe, expect, it } from "vitest"; -import { normalizeDatabasePropertyInput } from "../../actions/_database-property-input"; +import { + canonicalizeDatabasePropertyInput, + normalizeDatabasePropertyInput, +} from "../../actions/_database-property-input"; +import { digest } from "../../actions/_database-row-mutation"; import addDatabaseItem from "../../actions/add-database-item"; import updateDatabaseItem from "../../actions/update-database-item"; import upsertDatabaseItemByKey from "../../actions/upsert-database-item-by-key"; @@ -67,4 +71,42 @@ describe("database row property inputs", () => { }), ).toThrow(/not both/); }); + + it("removes the model-only representation before canonical hashing", () => { + const canonical = canonicalizeDatabasePropertyInput({ + idempotencyKey: "same-intent", + propertyEntries: [ + { propertyId: "status-id", value: "ready" }, + { propertyId: "evidence-id", value: "preserve me" }, + ], + }); + + expect(canonical).toEqual({ + idempotencyKey: "same-intent", + propertyValues: { + "status-id": "ready", + "evidence-id": "preserve me", + }, + }); + expect(canonical).not.toHaveProperty("propertyEntries"); + }); + + it("gives equivalent entry and record inputs the same canonical digest", () => { + const fromEntries = canonicalizeDatabasePropertyInput({ + idempotencyKey: "same-intent", + propertyEntries: [ + { propertyId: "status-id", value: "ready" }, + { propertyId: "evidence-id", value: "preserve me" }, + ], + }); + const fromRecord = canonicalizeDatabasePropertyInput({ + idempotencyKey: "same-intent", + propertyValues: { + "evidence-id": "preserve me", + "status-id": "ready", + }, + }); + + expect(digest(fromEntries)).toBe(digest(fromRecord)); + }); }); diff --git a/templates/content/parity/__tests__/eval-scenario-coverage.test.ts b/templates/content/parity/__tests__/eval-scenario-coverage.test.ts index b3d39b7935..f7aba9894c 100644 --- a/templates/content/parity/__tests__/eval-scenario-coverage.test.ts +++ b/templates/content/parity/__tests__/eval-scenario-coverage.test.ts @@ -203,4 +203,79 @@ describe("Content parity eval scenarios", () => { ).toMatchObject({ passed: true, score: 1 }); expect(row.status).toBe("passed"); }); + + it.each([ + { + name: "duplicate property entries", + toolCallDetails: [ + { + name: "add-database-item", + input: { + propertyEntries: [ + { propertyId: "parity-text-property-id", value: "preserve me" }, + { propertyId: "parity-text-property-id", value: "preserve me" }, + { propertyId: "parity-status-property-id", value: "ready" }, + ], + }, + }, + ], + }, + { + name: "ambiguous property formats", + toolCallDetails: [ + { + name: "add-database-item", + input: { + propertyEntries: [ + { propertyId: "parity-text-property-id", value: "preserve me" }, + { propertyId: "parity-status-property-id", value: "ready" }, + ], + propertyValues: { + "parity-text-property-id": "preserve me", + "parity-status-property-id": "ready", + }, + }, + }, + ], + }, + { + name: "an extra row mutation", + toolCallDetails: [ + { + name: "add-database-item", + input: { + propertyEntries: [ + { propertyId: "parity-text-property-id", value: "preserve me" }, + { propertyId: "parity-status-property-id", value: "ready" }, + ], + }, + }, + { name: "update-database-item", input: {} }, + ], + }, + ])("rejects $name", async ({ toolCallDetails }) => { + process.env.CONTENT_PARITY_EVALS = "1"; + const scenario = parityEvalScenarios.find( + (candidate) => candidate.id === "database-create-property-preservation", + )!; + const evalCase = scenarioToEval(scenario); + const row = await scoreEval(evalCase, { + runAgent: vi.fn(async () => ({ + text: scenario.successSignals.join("\n"), + toolCalls: toolCallDetails.map((call) => call.name), + toolCallDetails, + ok: true, + runId: "content-parity:invalid-property-input", + durationMs: 1, + })), + engine: {} as never, + model: "test-model", + analyzeContext: vi.fn(), + }); + + expect( + row.scores.find((score) => score.scorer === "expected_property_values"), + ).toMatchObject({ passed: false, score: 0 }); + expect(row.status).toBe("failed"); + }); }); diff --git a/templates/content/parity/scenario-to-eval.ts b/templates/content/parity/scenario-to-eval.ts index c657048da4..d6766456a3 100644 --- a/templates/content/parity/scenario-to-eval.ts +++ b/templates/content/parity/scenario-to-eval.ts @@ -30,26 +30,68 @@ function expectedToolScorer(expectedTools: string[]) { }); } -function normalizePropertyValues(input: unknown): Record { - if (!input || typeof input !== "object" || Array.isArray(input)) return {}; - const propertyValues = - (input as Record).propertyEntries ?? - (input as Record).propertyValues; - if (!propertyValues) return {}; - if (!Array.isArray(propertyValues)) { - return typeof propertyValues === "object" - ? (propertyValues as Record) - : {}; +function analyzePropertyValues(input: unknown): { + received: Record; + invalid: string[]; +} { + if (!input || typeof input !== "object" || Array.isArray(input)) { + return { received: {}, invalid: ["tool input is not an object"] }; } - return Object.fromEntries( - propertyValues.flatMap((entry) => { - if (!entry || typeof entry !== "object") return []; - const { propertyId, value } = entry as Record; - return typeof propertyId === "string" ? [[propertyId, value]] : []; - }), - ); + const record = input as Record; + const hasEntries = record.propertyEntries !== undefined; + const hasValues = record.propertyValues !== undefined; + if (hasEntries && hasValues) { + return { + received: {}, + invalid: ["propertyEntries and propertyValues were both provided"], + }; + } + if (hasValues) { + if ( + !record.propertyValues || + typeof record.propertyValues !== "object" || + Array.isArray(record.propertyValues) + ) { + return { received: {}, invalid: ["propertyValues is not a record"] }; + } + return { + received: record.propertyValues as Record, + invalid: [], + }; + } + if (!hasEntries || !Array.isArray(record.propertyEntries)) { + return { received: {}, invalid: ["propertyEntries is not an array"] }; + } + + const received: Record = {}; + const invalid: string[] = []; + for (const entry of record.propertyEntries) { + if (!entry || typeof entry !== "object" || Array.isArray(entry)) { + invalid.push("propertyEntries contains a non-object entry"); + continue; + } + const { propertyId, value } = entry as Record; + if (typeof propertyId !== "string" || propertyId.length === 0) { + invalid.push("propertyEntries contains an invalid propertyId"); + continue; + } + if (Object.prototype.hasOwnProperty.call(received, propertyId)) { + invalid.push(`propertyEntries contains duplicate ID ${propertyId}`); + continue; + } + received[propertyId] = value; + } + return { received, invalid }; } +const databaseRowMutationTools = new Set([ + "add-database-item", + "update-database-item", + "upsert-database-item-by-key", + "duplicate-database-items", + "remove-database-items", +]); + function expectedPropertyValuesScorer(expected: Record) { return createScorer< AgentRunOutput, @@ -57,30 +99,57 @@ function expectedPropertyValuesScorer(expected: Record) { received: Record; missing: string[]; unexpected: string[]; + invalid: string[]; + mutationCalls: string[]; } >({ name: "expected_property_values", analyze(run) { - const detail = run.toolCallDetails?.find( + const mutationCalls = (run.toolCallDetails ?? []) + .filter((call) => databaseRowMutationTools.has(call.name)) + .map((call) => call.name); + const createCalls = (run.toolCallDetails ?? []).filter( (call) => call.name === "add-database-item", ); - const received = normalizePropertyValues(detail?.input); + const analysis = analyzePropertyValues(createCalls[0]?.input); + const invalid = [...analysis.invalid]; + if (createCalls.length !== 1) { + invalid.push( + `expected exactly one add-database-item call, received ${createCalls.length}`, + ); + } + if (mutationCalls.length !== 1) { + invalid.push( + `expected exactly one row mutation, received ${mutationCalls.length}`, + ); + } + const received = analysis.received; const missing = Object.entries(expected) .filter(([propertyId, value]) => received[propertyId] !== value) .map(([propertyId]) => propertyId); const unexpected = Object.keys(received).filter( (propertyId) => !(propertyId in expected), ); - return { received, missing, unexpected }; + return { received, missing, unexpected, invalid, mutationCalls }; }, - generateScore({ missing, unexpected }) { - return missing.length === 0 && unexpected.length === 0 ? 1 : 0; + generateScore({ missing, unexpected, invalid }) { + return missing.length === 0 && + unexpected.length === 0 && + invalid.length === 0 + ? 1 + : 0; }, - generateReason({ analysis: { received, missing, unexpected } }) { - if (missing.length === 0 && unexpected.length === 0) { + generateReason({ + analysis: { received, missing, unexpected, invalid, mutationCalls }, + }) { + if ( + missing.length === 0 && + unexpected.length === 0 && + invalid.length === 0 + ) { return "Agent preserved every expected property ID and exact value without inventing another property."; } - return `Received propertyValues ${JSON.stringify(received)}; missing or changed: ${missing.join(", ") || "none"}; unexpected: ${unexpected.join(", ") || "none"}`; + return `Received propertyValues ${JSON.stringify(received)}; mutations: ${mutationCalls.join(", ") || "none"}; missing or changed: ${missing.join(", ") || "none"}; unexpected: ${unexpected.join(", ") || "none"}; invalid: ${invalid.join("; ") || "none"}`; }, }); } From 75344400b32aa39343e775df620697882050fc07 Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Fri, 21 Aug 2026 07:52:10 -0400 Subject: [PATCH 4/5] test: require successful Content mutation evals --- packages/core/src/eval/agent-runner.ts | 30 +++- packages/core/src/eval/runner.spec.ts | 23 ++- packages/core/src/eval/types.ts | 5 +- .../__tests__/eval-scenario-coverage.test.ts | 153 +++++++++++++----- templates/content/parity/eval-scenarios.ts | 27 +++- templates/content/parity/scenario-to-eval.ts | 63 +++++++- 6 files changed, 250 insertions(+), 51 deletions(-) diff --git a/packages/core/src/eval/agent-runner.ts b/packages/core/src/eval/agent-runner.ts index 3b370ae7f2..0c316ec3e7 100644 --- a/packages/core/src/eval/agent-runner.ts +++ b/packages/core/src/eval/agent-runner.ts @@ -120,7 +120,14 @@ export async function createAgentRunner( let text = ""; const toolCalls: string[] = []; - const toolCallDetails: Array<{ name: string; input: unknown }> = []; + const toolCallDetails: Array<{ + name: string; + id?: string; + input: unknown; + completed?: boolean; + isError?: boolean; + result?: string; + }> = []; let ok = true; let error: string | undefined; @@ -135,8 +142,25 @@ export async function createAgentRunner( break; case "tool_start": toolCalls.push(event.tool); - toolCallDetails.push({ name: event.tool, input: event.input }); + toolCallDetails.push({ + name: event.tool, + id: event.id, + input: event.input, + }); + break; + case "tool_done": { + const detail = event.id + ? toolCallDetails.find((call) => call.id === event.id) + : toolCallDetails.find( + (call) => call.name === event.tool && !call.completed, + ); + if (detail) { + detail.completed = true; + detail.isError = event.isError === true; + detail.result = event.result; + } break; + } case "error": ok = false; error = event.error; @@ -167,7 +191,7 @@ export async function createAgentRunner( return { text, toolCalls, - toolCallDetails, + toolCallDetails: toolCallDetails.map(({ id: _id, ...detail }) => detail), ok, error, runId, diff --git a/packages/core/src/eval/runner.spec.ts b/packages/core/src/eval/runner.spec.ts index 3fa997625c..d0b5df129c 100644 --- a/packages/core/src/eval/runner.spec.ts +++ b/packages/core/src/eval/runner.spec.ts @@ -276,7 +276,18 @@ describe("createAgentRunner over a mocked runAgentLoop (no real model)", () => { const runLoop = vi.fn( async (opts: { send: (e: AgentChatEvent) => void }) => { opts.send({ type: "text", text: "Hello " }); - opts.send({ type: "tool_start", tool: "search", input: {} }); + opts.send({ + type: "tool_start", + tool: "search", + id: "search-1", + input: {}, + }); + opts.send({ + type: "tool_done", + tool: "search", + id: "search-1", + result: '{"ok":true}', + }); opts.send({ type: "text", text: "world" }); return { inputTokens: 0, @@ -299,7 +310,15 @@ describe("createAgentRunner over a mocked runAgentLoop (no real model)", () => { const out = await runner.runAgent({ prompt: "hi" }); expect(out.text).toBe("Hello world"); expect(out.toolCalls).toEqual(["search"]); - expect(out.toolCallDetails).toEqual([{ name: "search", input: {} }]); + expect(out.toolCallDetails).toEqual([ + { + name: "search", + input: {}, + completed: true, + isError: false, + result: '{"ok":true}', + }, + ]); expect(out.ok).toBe(true); // End-to-end: a contains scorer over the real collected text. diff --git a/packages/core/src/eval/types.ts b/packages/core/src/eval/types.ts index a99a0849a9..d9f17f8fef 100644 --- a/packages/core/src/eval/types.ts +++ b/packages/core/src/eval/types.ts @@ -31,10 +31,13 @@ export interface AgentRunOutput { readonly text: string; /** Names of tools/actions the agent invoked, in call order. */ readonly toolCalls: readonly string[]; - /** Tool names and model-produced inputs, in call order. */ + /** Tool names, model-produced inputs, and execution outcomes in call order. */ readonly toolCallDetails?: readonly { readonly name: string; readonly input: unknown; + readonly completed?: boolean; + readonly isError?: boolean; + readonly result?: string; }[]; /** Whether the run completed without a terminal error event. */ readonly ok: boolean; diff --git a/templates/content/parity/__tests__/eval-scenario-coverage.test.ts b/templates/content/parity/__tests__/eval-scenario-coverage.test.ts index f7aba9894c..02bc290662 100644 --- a/templates/content/parity/__tests__/eval-scenario-coverage.test.ts +++ b/templates/content/parity/__tests__/eval-scenario-coverage.test.ts @@ -7,6 +7,22 @@ import { scenarioToEval } from "../scenario-to-eval"; const OLD_GATE = process.env.CONTENT_PARITY_EVALS; +function successfulCreateCall( + scenario: (typeof parityEvalScenarios)[number], + propertyInput: Record, +) { + return { + name: "add-database-item", + input: { + ...scenario.expectedCreateEnvelope, + ...propertyInput, + }, + completed: true, + isError: false, + result: '{"fixtureOnly":true}', + }; +} + afterEach(() => { if (OLD_GATE === undefined) { delete process.env.CONTENT_PARITY_EVALS; @@ -156,7 +172,7 @@ describe("Content parity eval scenarios", () => { text: scenario.successSignals.join("\n"), toolCalls: ["add-database-item"], toolCallDetails: [ - { name: "add-database-item", input: { propertyValues: {} } }, + successfulCreateCall(scenario, { propertyValues: {} }), ], ok: true, runId: "content-parity:empty-property-values", @@ -184,10 +200,9 @@ describe("Content parity eval scenarios", () => { text: scenario.successSignals.join("\n"), toolCalls: ["add-database-item"], toolCallDetails: [ - { - name: "add-database-item", - input: { propertyValues: scenario.expectedPropertyValues }, - }, + successfulCreateCall(scenario, { + propertyValues: scenario.expectedPropertyValues, + }), ], ok: true, runId: "content-parity:exact-property-values", @@ -205,59 +220,57 @@ describe("Content parity eval scenarios", () => { }); it.each([ - { + (scenario: (typeof parityEvalScenarios)[number]) => ({ name: "duplicate property entries", toolCallDetails: [ - { - name: "add-database-item", - input: { - propertyEntries: [ - { propertyId: "parity-text-property-id", value: "preserve me" }, - { propertyId: "parity-text-property-id", value: "preserve me" }, - { propertyId: "parity-status-property-id", value: "ready" }, - ], - }, - }, + successfulCreateCall(scenario, { + propertyEntries: [ + { propertyId: "parity-text-property-id", value: "preserve me" }, + { propertyId: "parity-text-property-id", value: "preserve me" }, + { propertyId: "parity-status-property-id", value: "ready" }, + ], + }), ], - }, - { + }), + (scenario: (typeof parityEvalScenarios)[number]) => ({ name: "ambiguous property formats", toolCallDetails: [ - { - name: "add-database-item", - input: { - propertyEntries: [ - { propertyId: "parity-text-property-id", value: "preserve me" }, - { propertyId: "parity-status-property-id", value: "ready" }, - ], - propertyValues: { - "parity-text-property-id": "preserve me", - "parity-status-property-id": "ready", - }, + successfulCreateCall(scenario, { + propertyEntries: [ + { propertyId: "parity-text-property-id", value: "preserve me" }, + { propertyId: "parity-status-property-id", value: "ready" }, + ], + propertyValues: { + "parity-text-property-id": "preserve me", + "parity-status-property-id": "ready", }, - }, + }), ], - }, - { + }), + (scenario: (typeof parityEvalScenarios)[number]) => ({ name: "an extra row mutation", toolCallDetails: [ + successfulCreateCall(scenario, { + propertyEntries: [ + { propertyId: "parity-text-property-id", value: "preserve me" }, + { propertyId: "parity-status-property-id", value: "ready" }, + ], + }), { - name: "add-database-item", - input: { - propertyEntries: [ - { propertyId: "parity-text-property-id", value: "preserve me" }, - { propertyId: "parity-status-property-id", value: "ready" }, - ], - }, + name: "update-database-item", + input: {}, + completed: true, + isError: false, + result: "{}", }, - { name: "update-database-item", input: {} }, ], - }, - ])("rejects $name", async ({ toolCallDetails }) => { + }), + ])("rejects invalid property behavior", async (buildCase) => { process.env.CONTENT_PARITY_EVALS = "1"; const scenario = parityEvalScenarios.find( (candidate) => candidate.id === "database-create-property-preservation", )!; + const { toolCallDetails } = buildCase(scenario); const evalCase = scenarioToEval(scenario); const row = await scoreEval(evalCase, { runAgent: vi.fn(async () => ({ @@ -278,4 +291,60 @@ describe("Content parity eval scenarios", () => { ).toMatchObject({ passed: false, score: 0 }); expect(row.status).toBe("failed"); }); + + it.each([ + { + name: "wrong target", + mutate(call: ReturnType) { + return { + ...call, + input: { + ...(call.input as Record), + target: { + ...((call.input as Record).target as Record< + string, + unknown + >), + databaseId: "wrong-database", + }, + }, + }; + }, + }, + { + name: "failed execution", + mutate(call: ReturnType) { + return { ...call, isError: true, result: "fixture rejected" }; + }, + }, + ])("rejects $name", async ({ mutate }) => { + process.env.CONTENT_PARITY_EVALS = "1"; + const scenario = parityEvalScenarios.find( + (candidate) => candidate.id === "database-create-property-preservation", + )!; + const call = mutate( + successfulCreateCall(scenario, { + propertyValues: scenario.expectedPropertyValues, + }), + ); + const evalCase = scenarioToEval(scenario); + const row = await scoreEval(evalCase, { + runAgent: vi.fn(async () => ({ + text: scenario.successSignals.join("\n"), + toolCalls: ["add-database-item"], + toolCallDetails: [call], + ok: true, + runId: "content-parity:rejected-create", + durationMs: 1, + })), + engine: {} as never, + model: "test-model", + analyzeContext: vi.fn(), + }); + + expect( + row.scores.find((score) => score.scorer === "expected_property_values"), + ).toMatchObject({ passed: false, score: 0 }); + expect(row.status).toBe("failed"); + }); }); diff --git a/templates/content/parity/eval-scenarios.ts b/templates/content/parity/eval-scenarios.ts index c459e1f376..3c26553e2b 100644 --- a/templates/content/parity/eval-scenarios.ts +++ b/templates/content/parity/eval-scenarios.ts @@ -9,6 +9,17 @@ export interface ParityEvalScenario { successSignals: string[]; expectedTools?: string[]; expectedPropertyValues?: Record; + expectedCreateEnvelope?: { + target: { + authorityScope: { kind: "personal"; id: string }; + spaceId: string; + databaseId: string; + databaseDocumentId: string; + }; + expectedSchemaRevision: string; + idempotencyKey: string; + title: string; + }; } export const parityEvalScenarios: ParityEvalScenario[] = [ @@ -20,7 +31,7 @@ export const parityEvalScenarios: ParityEvalScenario[] = [ defaultState: "skipped", requiresPrivateCredentials: false, prompt: - "Create exactly one row in the already-verified fixture Content database. Do not rediscover or alter the target. Use target spaceId fixture_personal_space, databaseId fixture_feedback_database, and documentId fixture_feedback_document; expectedSchemaRevision fixture_schema_revision; title [FIXTURE] preserve explicit properties; idempotencyKey fixture-create-property-preservation-v1. Set property fixture_status_property to status-cannot-verify and property fixture_evidence_property to Baseline fixture preserve-me. No Blocks value was requested. Call add-database-item once with every exact target constraint and both property entries, then report its result truthfully.", + "Create exactly one row in the already-verified fixture Content database. Do not rediscover or alter the target. Use target authorityScope { kind: personal, id: fixture-owner@example.test }, spaceId fixture_personal_space, databaseId fixture_feedback_database, and databaseDocumentId fixture_feedback_document; expectedSchemaRevision fixture_schema_revision; title [FIXTURE] preserve explicit properties; idempotencyKey fixture-create-property-preservation-v1. Set property fixture_status_property to status-cannot-verify and property fixture_evidence_property to Baseline fixture preserve-me. No Blocks value was requested. Call add-database-item once with every exact target constraint and both property entries, then report its result truthfully.", successSignals: [ "Uses add-database-item once for the exact fixture target.", "Preserves both explicitly requested writable property values.", @@ -32,6 +43,20 @@ export const parityEvalScenarios: ParityEvalScenario[] = [ fixture_status_property: "status-cannot-verify", fixture_evidence_property: "Baseline fixture preserve-me", }, + expectedCreateEnvelope: { + target: { + authorityScope: { + kind: "personal", + id: "fixture-owner@example.test", + }, + spaceId: "fixture_personal_space", + databaseId: "fixture_feedback_database", + databaseDocumentId: "fixture_feedback_document", + }, + expectedSchemaRevision: "fixture_schema_revision", + idempotencyKey: "fixture-create-property-preservation-v1", + title: "[FIXTURE] preserve explicit properties", + }, }, { id: "database-bulk-row-reliability", diff --git a/templates/content/parity/scenario-to-eval.ts b/templates/content/parity/scenario-to-eval.ts index d6766456a3..514fa94cea 100644 --- a/templates/content/parity/scenario-to-eval.ts +++ b/templates/content/parity/scenario-to-eval.ts @@ -92,7 +92,40 @@ const databaseRowMutationTools = new Set([ "remove-database-items", ]); -function expectedPropertyValuesScorer(expected: Record) { +function matchesCreateEnvelope( + input: Record, + expected: NonNullable, +): boolean { + const target = input.target; + if (!target || typeof target !== "object" || Array.isArray(target)) { + return false; + } + const actualTarget = target as Record; + const authorityScope = actualTarget.authorityScope; + if ( + !authorityScope || + typeof authorityScope !== "object" || + Array.isArray(authorityScope) + ) { + return false; + } + const actualAuthority = authorityScope as Record; + return ( + actualAuthority.kind === expected.target.authorityScope.kind && + actualAuthority.id === expected.target.authorityScope.id && + actualTarget.spaceId === expected.target.spaceId && + actualTarget.databaseId === expected.target.databaseId && + actualTarget.databaseDocumentId === expected.target.databaseDocumentId && + input.expectedSchemaRevision === expected.expectedSchemaRevision && + input.idempotencyKey === expected.idempotencyKey && + input.title === expected.title + ); +} + +function expectedPropertyValuesScorer( + expected: Record, + expectedEnvelope?: ParityEvalScenario["expectedCreateEnvelope"], +) { return createScorer< AgentRunOutput, { @@ -123,6 +156,27 @@ function expectedPropertyValuesScorer(expected: Record) { `expected exactly one row mutation, received ${mutationCalls.length}`, ); } + const createInput = createCalls[0]?.input; + if ( + expectedEnvelope && + (!createInput || + typeof createInput !== "object" || + Array.isArray(createInput) || + !matchesCreateEnvelope( + createInput as Record, + expectedEnvelope, + )) + ) { + invalid.push( + "create target, schema revision, idempotency key, or title did not match the fixture", + ); + } + if (!createCalls[0]?.completed || createCalls[0]?.isError) { + invalid.push("add-database-item did not complete successfully"); + } + if (!run.ok) { + invalid.push("agent run did not complete successfully"); + } const received = analysis.received; const missing = Object.entries(expected) .filter(([propertyId, value]) => received[propertyId] !== value) @@ -177,7 +231,12 @@ export function scenarioToEval(scenario: ParityEvalScenario): Eval { ? [expectedToolScorer(scenario.expectedTools)] : []), ...(scenario.expectedPropertyValues - ? [expectedPropertyValuesScorer(scenario.expectedPropertyValues)] + ? [ + expectedPropertyValuesScorer( + scenario.expectedPropertyValues, + scenario.expectedCreateEnvelope, + ), + ] : []), ], }); From 0e7b09d59debbbb8154acbeb6c038fe52f7824e8 Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Fri, 21 Aug 2026 16:06:51 -0400 Subject: [PATCH 5/5] chore: publish branch work in packages/core, templates/content (7 files) --- packages/core/src/eval/agent-runner.ts | 2 + packages/core/src/eval/runner.spec.ts | 25 ++++++- packages/core/src/eval/types.ts | 1 + .../actions/_database-property-input.ts | 5 +- .../database-row-property-input.test.ts | 33 ++++++++ .../__tests__/eval-scenario-coverage.test.ts | 75 +++++++++++++++++++ templates/content/parity/scenario-to-eval.ts | 50 ++++++++++++- 7 files changed, 186 insertions(+), 5 deletions(-) diff --git a/packages/core/src/eval/agent-runner.ts b/packages/core/src/eval/agent-runner.ts index 0c316ec3e7..c986194a0e 100644 --- a/packages/core/src/eval/agent-runner.ts +++ b/packages/core/src/eval/agent-runner.ts @@ -125,6 +125,7 @@ export async function createAgentRunner( id?: string; input: unknown; completed?: boolean; + completedSideEffect?: boolean; isError?: boolean; result?: string; }> = []; @@ -156,6 +157,7 @@ export async function createAgentRunner( ); if (detail) { detail.completed = true; + detail.completedSideEffect = event.completedSideEffect; detail.isError = event.isError === true; detail.result = event.result; } diff --git a/packages/core/src/eval/runner.spec.ts b/packages/core/src/eval/runner.spec.ts index d0b5df129c..389844d1dd 100644 --- a/packages/core/src/eval/runner.spec.ts +++ b/packages/core/src/eval/runner.spec.ts @@ -287,6 +287,20 @@ describe("createAgentRunner over a mocked runAgentLoop (no real model)", () => { tool: "search", id: "search-1", result: '{"ok":true}', + completedSideEffect: true, + }); + opts.send({ + type: "tool_start", + tool: "update", + id: "update-1", + input: {}, + }); + opts.send({ + type: "tool_done", + tool: "update", + id: "update-1", + result: '{"ok":false}', + completedSideEffect: false, }); opts.send({ type: "text", text: "world" }); return { @@ -309,15 +323,24 @@ describe("createAgentRunner over a mocked runAgentLoop (no real model)", () => { const out = await runner.runAgent({ prompt: "hi" }); expect(out.text).toBe("Hello world"); - expect(out.toolCalls).toEqual(["search"]); + expect(out.toolCalls).toEqual(["search", "update"]); expect(out.toolCallDetails).toEqual([ { name: "search", input: {}, completed: true, + completedSideEffect: true, isError: false, result: '{"ok":true}', }, + { + name: "update", + input: {}, + completed: true, + completedSideEffect: false, + isError: false, + result: '{"ok":false}', + }, ]); expect(out.ok).toBe(true); diff --git a/packages/core/src/eval/types.ts b/packages/core/src/eval/types.ts index d9f17f8fef..81e32e03a8 100644 --- a/packages/core/src/eval/types.ts +++ b/packages/core/src/eval/types.ts @@ -36,6 +36,7 @@ export interface AgentRunOutput { readonly name: string; readonly input: unknown; readonly completed?: boolean; + readonly completedSideEffect?: boolean; readonly isError?: boolean; readonly result?: string; }[]; diff --git a/templates/content/actions/_database-property-input.ts b/templates/content/actions/_database-property-input.ts index 30552617c3..053de8f77a 100644 --- a/templates/content/actions/_database-property-input.ts +++ b/templates/content/actions/_database-property-input.ts @@ -36,7 +36,10 @@ export function normalizeDatabasePropertyInput(input: { } if (!input.propertyEntries) return input.propertyValues; - const values: Record = {}; + const values: Record = Object.create(null) as Record< + string, + unknown + >; for (const entry of input.propertyEntries) { if (Object.prototype.hasOwnProperty.call(values, entry.propertyId)) { throw new ActionContractError( diff --git a/templates/content/parity/__tests__/database-row-property-input.test.ts b/templates/content/parity/__tests__/database-row-property-input.test.ts index aaa40f1e3d..6e31ef055b 100644 --- a/templates/content/parity/__tests__/database-row-property-input.test.ts +++ b/templates/content/parity/__tests__/database-row-property-input.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { canonicalizeDatabasePropertyInput, + databasePropertyEntriesSchema, normalizeDatabasePropertyInput, } from "../../actions/_database-property-input"; import { digest } from "../../actions/_database-row-mutation"; @@ -72,6 +73,23 @@ describe("database row property inputs", () => { ).toThrow(/not both/); }); + it("preserves __proto__ as an ordinary property definition ID", () => { + const propertyEntries = databasePropertyEntriesSchema.parse([ + { propertyId: "__proto__", value: "preserve me" }, + ]); + const normalized = normalizeDatabasePropertyInput({ + propertyEntries, + }); + + expect(normalized).toBeDefined(); + expect(Object.getPrototypeOf(normalized)).toBeNull(); + expect(Object.keys(normalized!)).toEqual(["__proto__"]); + expect(Object.prototype.hasOwnProperty.call(normalized, "__proto__")).toBe( + true, + ); + expect(normalized?.["__proto__"]).toBe("preserve me"); + }); + it("removes the model-only representation before canonical hashing", () => { const canonical = canonicalizeDatabasePropertyInput({ idempotencyKey: "same-intent", @@ -109,4 +127,19 @@ describe("database row property inputs", () => { expect(digest(fromEntries)).toBe(digest(fromRecord)); }); + + it("includes __proto__ property values in the canonical digest", () => { + const withPrototypeNamedProperty = canonicalizeDatabasePropertyInput({ + idempotencyKey: "same-intent", + propertyEntries: [{ propertyId: "__proto__", value: "preserve me" }], + }); + const withoutProperty = canonicalizeDatabasePropertyInput({ + idempotencyKey: "same-intent", + propertyValues: {}, + }); + + expect(digest(withPrototypeNamedProperty)).not.toBe( + digest(withoutProperty), + ); + }); }); diff --git a/templates/content/parity/__tests__/eval-scenario-coverage.test.ts b/templates/content/parity/__tests__/eval-scenario-coverage.test.ts index 02bc290662..bf0892c049 100644 --- a/templates/content/parity/__tests__/eval-scenario-coverage.test.ts +++ b/templates/content/parity/__tests__/eval-scenario-coverage.test.ts @@ -18,6 +18,7 @@ function successfulCreateCall( ...propertyInput, }, completed: true, + completedSideEffect: true, isError: false, result: '{"fixtureOnly":true}', }; @@ -317,6 +318,80 @@ describe("Content parity eval scenarios", () => { return { ...call, isError: true, result: "fixture rejected" }; }, }, + { + name: "skipped side effect", + mutate(call: ReturnType) { + return { ...call, completedSideEffect: false }; + }, + }, + { + name: "an extra top-level field", + mutate(call: ReturnType) { + return { + ...call, + input: { + ...(call.input as Record), + hallucinated: true, + }, + }; + }, + }, + { + name: "an extra target field", + mutate(call: ReturnType) { + const input = call.input as Record; + return { + ...call, + input: { + ...input, + target: { + ...(input.target as Record), + hallucinated: true, + }, + }, + }; + }, + }, + { + name: "an extra authority field", + mutate(call: ReturnType) { + const input = call.input as Record; + const target = input.target as Record; + return { + ...call, + input: { + ...input, + target: { + ...target, + authorityScope: { + ...(target.authorityScope as Record), + hallucinated: true, + }, + }, + }, + }; + }, + }, + { + name: "an extra property-entry field", + mutate(call: ReturnType) { + const input = call.input as Record; + const { propertyValues, ...withoutPropertyValues } = input; + return { + ...call, + input: { + ...withoutPropertyValues, + propertyEntries: Object.entries( + propertyValues as Record, + ).map(([propertyId, value]) => ({ + propertyId, + value, + hallucinated: true, + })), + }, + }; + }, + }, ])("rejects $name", async ({ mutate }) => { process.env.CONTENT_PARITY_EVALS = "1"; const scenario = parityEvalScenarios.find( diff --git a/templates/content/parity/scenario-to-eval.ts b/templates/content/parity/scenario-to-eval.ts index 514fa94cea..6f7e50d378 100644 --- a/templates/content/parity/scenario-to-eval.ts +++ b/templates/content/parity/scenario-to-eval.ts @@ -30,6 +30,18 @@ function expectedToolScorer(expectedTools: string[]) { }); } +function hasExactKeys( + record: Record, + expectedKeys: readonly string[], +): boolean { + const actualKeys = Object.keys(record).sort(); + const sortedExpectedKeys = [...expectedKeys].sort(); + return ( + actualKeys.length === sortedExpectedKeys.length && + actualKeys.every((key, index) => key === sortedExpectedKeys[index]) + ); +} + function analyzePropertyValues(input: unknown): { received: Record; invalid: string[]; @@ -63,7 +75,10 @@ function analyzePropertyValues(input: unknown): { return { received: {}, invalid: ["propertyEntries is not an array"] }; } - const received: Record = {}; + const received: Record = Object.create(null) as Record< + string, + unknown + >; const invalid: string[] = []; for (const entry of record.propertyEntries) { if (!entry || typeof entry !== "object" || Array.isArray(entry)) { @@ -71,6 +86,14 @@ function analyzePropertyValues(input: unknown): { continue; } const { propertyId, value } = entry as Record; + if ( + !hasExactKeys(entry as Record, ["propertyId", "value"]) + ) { + invalid.push( + "propertyEntries contains an entry with unrecognized fields", + ); + continue; + } if (typeof propertyId !== "string" || propertyId.length === 0) { invalid.push("propertyEntries contains an invalid propertyId"); continue; @@ -111,6 +134,22 @@ function matchesCreateEnvelope( } const actualAuthority = authorityScope as Record; return ( + hasExactKeys(input, [ + "target", + "expectedSchemaRevision", + "idempotencyKey", + "title", + input.propertyEntries === undefined + ? "propertyValues" + : "propertyEntries", + ]) && + hasExactKeys(actualTarget, [ + "authorityScope", + "spaceId", + "databaseId", + "databaseDocumentId", + ]) && + hasExactKeys(actualAuthority, ["kind", "id"]) && actualAuthority.kind === expected.target.authorityScope.kind && actualAuthority.id === expected.target.authorityScope.id && actualTarget.spaceId === expected.target.spaceId && @@ -171,7 +210,11 @@ function expectedPropertyValuesScorer( "create target, schema revision, idempotency key, or title did not match the fixture", ); } - if (!createCalls[0]?.completed || createCalls[0]?.isError) { + if ( + !createCalls[0]?.completed || + createCalls[0]?.completedSideEffect !== true || + createCalls[0]?.isError + ) { invalid.push("add-database-item did not complete successfully"); } if (!run.ok) { @@ -182,7 +225,8 @@ function expectedPropertyValuesScorer( .filter(([propertyId, value]) => received[propertyId] !== value) .map(([propertyId]) => propertyId); const unexpected = Object.keys(received).filter( - (propertyId) => !(propertyId in expected), + (propertyId) => + !Object.prototype.hasOwnProperty.call(expected, propertyId), ); return { received, missing, unexpected, invalid, mutationCalls }; },