diff --git a/.changeset/calm-tools-preserve-inputs.md b/.changeset/calm-tools-preserve-inputs.md new file mode 100644 index 0000000000..268e397a4a --- /dev/null +++ b/.changeset/calm-tools-preserve-inputs.md @@ -0,0 +1,5 @@ +--- +"@agent-native/core": patch +--- + +Expose model-produced tool inputs to eval scorers so argument-level agent behavior can be verified. diff --git a/packages/core/src/eval/agent-runner.ts b/packages/core/src/eval/agent-runner.ts index b8599bd85b..c986194a0e 100644 --- a/packages/core/src/eval/agent-runner.ts +++ b/packages/core/src/eval/agent-runner.ts @@ -120,6 +120,15 @@ export async function createAgentRunner( let text = ""; const toolCalls: string[] = []; + const toolCallDetails: Array<{ + name: string; + id?: string; + input: unknown; + completed?: boolean; + completedSideEffect?: boolean; + isError?: boolean; + result?: string; + }> = []; let ok = true; let error: string | undefined; @@ -134,7 +143,26 @@ export async function createAgentRunner( break; case "tool_start": toolCalls.push(event.tool); + toolCallDetails.push({ + name: event.tool, + id: event.id, + input: event.input, + }); + break; + case "tool_done": { + const detail = event.id + ? toolCallDetails.find((call) => call.id === event.id) + : toolCallDetails.find( + (call) => call.name === event.tool && !call.completed, + ); + if (detail) { + detail.completed = true; + detail.completedSideEffect = event.completedSideEffect; + detail.isError = event.isError === true; + detail.result = event.result; + } break; + } case "error": ok = false; error = event.error; @@ -165,6 +193,7 @@ export async function createAgentRunner( return { text, toolCalls, + toolCallDetails: toolCallDetails.map(({ id: _id, ...detail }) => detail), ok, error, runId, diff --git a/packages/core/src/eval/runner.spec.ts b/packages/core/src/eval/runner.spec.ts index fef74b3690..389844d1dd 100644 --- a/packages/core/src/eval/runner.spec.ts +++ b/packages/core/src/eval/runner.spec.ts @@ -276,7 +276,32 @@ describe("createAgentRunner over a mocked runAgentLoop (no real model)", () => { const runLoop = vi.fn( async (opts: { send: (e: AgentChatEvent) => void }) => { opts.send({ type: "text", text: "Hello " }); - opts.send({ type: "tool_start", tool: "search", input: {} }); + opts.send({ + type: "tool_start", + tool: "search", + id: "search-1", + input: {}, + }); + opts.send({ + type: "tool_done", + tool: "search", + id: "search-1", + result: '{"ok":true}', + completedSideEffect: true, + }); + opts.send({ + type: "tool_start", + tool: "update", + id: "update-1", + input: {}, + }); + opts.send({ + type: "tool_done", + tool: "update", + id: "update-1", + result: '{"ok":false}', + completedSideEffect: false, + }); opts.send({ type: "text", text: "world" }); return { inputTokens: 0, @@ -298,7 +323,25 @@ describe("createAgentRunner over a mocked runAgentLoop (no real model)", () => { const out = await runner.runAgent({ prompt: "hi" }); expect(out.text).toBe("Hello world"); - expect(out.toolCalls).toEqual(["search"]); + expect(out.toolCalls).toEqual(["search", "update"]); + expect(out.toolCallDetails).toEqual([ + { + name: "search", + input: {}, + completed: true, + completedSideEffect: true, + isError: false, + result: '{"ok":true}', + }, + { + name: "update", + input: {}, + completed: true, + completedSideEffect: false, + isError: false, + result: '{"ok":false}', + }, + ]); expect(out.ok).toBe(true); // End-to-end: a contains scorer over the real collected text. diff --git a/packages/core/src/eval/types.ts b/packages/core/src/eval/types.ts index 3648416981..81e32e03a8 100644 --- a/packages/core/src/eval/types.ts +++ b/packages/core/src/eval/types.ts @@ -31,6 +31,15 @@ export interface AgentRunOutput { readonly text: string; /** Names of tools/actions the agent invoked, in call order. */ readonly toolCalls: readonly string[]; + /** Tool names, model-produced inputs, and execution outcomes in call order. */ + readonly toolCallDetails?: readonly { + readonly name: string; + readonly input: unknown; + readonly completed?: boolean; + readonly completedSideEffect?: boolean; + readonly isError?: boolean; + readonly result?: string; + }[]; /** Whether the run completed without a terminal error event. */ readonly ok: boolean; /** Terminal error message, if the run errored. */ diff --git a/templates/content/actions/_database-property-input.ts b/templates/content/actions/_database-property-input.ts new file mode 100644 index 0000000000..053de8f77a --- /dev/null +++ b/templates/content/actions/_database-property-input.ts @@ -0,0 +1,76 @@ +import { ActionContractError } from "@agent-native/core"; +import { z } from "zod"; + +export const databasePropertyValuesSchema = z + .record(z.string(), z.unknown()) + .optional() + .describe( + "Programmatic property values keyed by exact property definition ID.", + ); + +export const databasePropertyEntriesSchema = z + .array( + z.object({ + propertyId: z + .string() + .min(1) + .describe("Exact immutable property definition ID"), + value: z.unknown().describe("Schema-valid value for this property"), + }), + ) + .max(1_000) + .optional() + .describe( + "Property values as explicit entries. Include one entry for every schema-valid writable property value the user requested, using the exact immutable property definition ID. When at least one value was requested, never pass an empty array. Do not invent or clear unmentioned properties.", + ); + +export function normalizeDatabasePropertyInput(input: { + propertyEntries?: Array<{ propertyId: string; value: unknown }>; + propertyValues?: Record; +}): Record | undefined { + if (input.propertyEntries && input.propertyValues) { + throw new ActionContractError( + "Provide propertyEntries or propertyValues, not both.", + { errorCode: "AMBIGUOUS_PROPERTY_INPUT" }, + ); + } + if (!input.propertyEntries) return input.propertyValues; + + const values: Record = Object.create(null) as Record< + string, + unknown + >; + for (const entry of input.propertyEntries) { + if (Object.prototype.hasOwnProperty.call(values, entry.propertyId)) { + throw new ActionContractError( + `Property entry ${entry.propertyId} was provided more than once.`, + { + errorCode: "DUPLICATE_PROPERTY_INPUT", + details: { propertyId: entry.propertyId }, + }, + ); + } + values[entry.propertyId] = entry.value; + } + return values; +} + +export function canonicalizeDatabasePropertyInput< + T extends { + propertyEntries?: Array<{ propertyId: string; value: unknown }>; + propertyValues?: Record; + }, +>( + input: T, +): Omit & { + propertyValues?: Record; +} { + const { propertyEntries, propertyValues, ...canonicalInput } = input; + return { + ...canonicalInput, + propertyValues: normalizeDatabasePropertyInput({ + propertyEntries, + propertyValues, + }), + }; +} diff --git a/templates/content/actions/add-database-item.ts b/templates/content/actions/add-database-item.ts index 318f9984a2..8666c33bdf 100644 --- a/templates/content/actions/add-database-item.ts +++ b/templates/content/actions/add-database-item.ts @@ -3,6 +3,11 @@ import { buildDeepLink } from "@agent-native/core/server"; import { z } from "zod"; import type { ContentDatabaseRowMutationResult } from "../shared/api.js"; +import { + canonicalizeDatabasePropertyInput, + databasePropertyEntriesSchema, + databasePropertyValuesSchema, +} from "./_database-property-input.js"; import { createDatabaseRow, databaseMutationEnvelopeSchema, @@ -17,15 +22,14 @@ const schema = databaseMutationEnvelopeSchema.extend({ .max(500) .optional() .describe("New row page title"), - propertyValues: z - .record(z.string(), z.unknown()) - .optional() - .describe("Strict property values keyed by property definition ID"), + propertyValues: databasePropertyValuesSchema, + propertyEntries: databasePropertyEntriesSchema, }); export default defineAction({ description: "Create one row in an exact ordinary Content database using its discovered schema revision. Strictly validates every non-Blocks property, applies the side effect once per idempotency key, and returns a verified receipt with stable row identity.", + agentInputSchema: schema.omit({ propertyValues: true }), publicAgent: { expose: true, readOnly: false, @@ -52,7 +56,9 @@ export default defineAction({ }, }, run: async (args): Promise => { - const result = await createDatabaseRow(args); + const result = await createDatabaseRow( + canonicalizeDatabasePropertyInput(args), + ); const response = await getContentDatabaseResponse( result.receipt.target.databaseId, { diff --git a/templates/content/actions/update-database-item.ts b/templates/content/actions/update-database-item.ts index 889c33c15d..4fcd904e53 100644 --- a/templates/content/actions/update-database-item.ts +++ b/templates/content/actions/update-database-item.ts @@ -3,6 +3,11 @@ import { buildDeepLink } from "@agent-native/core/server"; import { z } from "zod"; import type { ContentDatabaseRowMutationResult } from "../shared/api.js"; +import { + canonicalizeDatabasePropertyInput, + databasePropertyEntriesSchema, + databasePropertyValuesSchema, +} from "./_database-property-input.js"; import { databaseMutationEnvelopeSchema, updateDatabaseRow, @@ -16,17 +21,16 @@ const schema = databaseMutationEnvelopeSchema.extend({ .min(1) .describe("Row revision returned by get-content-database"), title: z.string().trim().min(1).max(500).optional(), - propertyValues: z - .record(z.string(), z.unknown()) - .optional() - .describe( - "Sparse strict patch keyed by property definition ID; omitted fields are preserved and explicit null clears a value", - ), + propertyValues: databasePropertyValuesSchema, + propertyEntries: databasePropertyEntriesSchema.describe( + "Sparse property patch as explicit entries; omitted fields are preserved and explicit null clears a value. Include one entry for every schema-valid writable property value the user requested, using the exact immutable property definition ID. When at least one value was requested, never pass an empty array. Do not invent or clear unmentioned properties.", + ), }); export default defineAction({ description: "Sparsely update one exact Content database row by stable item and document IDs. Requires schema and row revisions, validates every non-Blocks property, and returns a verified idempotent receipt.", + agentInputSchema: schema.omit({ propertyValues: true }), schema, http: { method: "PUT" }, audit: { @@ -44,7 +48,7 @@ export default defineAction({ : "Updated Content database row"; }, }, - run: updateDatabaseRow, + run: (args) => updateDatabaseRow(canonicalizeDatabasePropertyInput(args)), link: ({ result }) => { const documentId = (result as ContentDatabaseRowMutationResult | null) ?.receipt.row.documentId; diff --git a/templates/content/actions/upsert-database-item-by-key.ts b/templates/content/actions/upsert-database-item-by-key.ts index c9a86fb303..93dcb1ef43 100644 --- a/templates/content/actions/upsert-database-item-by-key.ts +++ b/templates/content/actions/upsert-database-item-by-key.ts @@ -3,6 +3,11 @@ import { buildDeepLink } from "@agent-native/core/server"; import { z } from "zod"; import type { ContentDatabaseRowMutationResult } from "../shared/api.js"; +import { + canonicalizeDatabasePropertyInput, + databasePropertyEntriesSchema, + databasePropertyValuesSchema, +} from "./_database-property-input.js"; import { databaseMutationEnvelopeSchema, upsertDatabaseRow, @@ -18,15 +23,16 @@ const schema = databaseMutationEnvelopeSchema.extend({ "Use null to assert the key is absent and create; use the discovered row revision to update an existing key", ), title: z.string().trim().min(1).max(500).optional(), - propertyValues: z - .record(z.string(), z.unknown()) - .optional() - .describe("Sparse strict values keyed by property definition ID"), + propertyValues: databasePropertyValuesSchema, + propertyEntries: databasePropertyEntriesSchema.describe( + "Sparse property values as explicit entries. Include one entry for every schema-valid writable property value the user requested, using the exact immutable property definition ID. When at least one value was requested, never pass an empty array. Do not invent or clear unmentioned properties.", + ), }); export default defineAction({ description: "Create or sparsely update one Content database row by that database's explicitly configured natural key. Requires schema and row compare-and-swap revisions and returns a verified idempotent receipt.", + agentInputSchema: schema.omit({ propertyValues: true }), schema, audit: { recordInputs: false, @@ -43,7 +49,7 @@ export default defineAction({ : "Upserted Content database row by natural key"; }, }, - run: upsertDatabaseRow, + run: (args) => upsertDatabaseRow(canonicalizeDatabasePropertyInput(args)), link: ({ result }) => { const documentId = (result as ContentDatabaseRowMutationResult | null) ?.receipt.row.documentId; diff --git a/templates/content/package.json b/templates/content/package.json index af38011c90..b470021575 100644 --- a/templates/content/package.json +++ b/templates/content/package.json @@ -12,6 +12,7 @@ "test:parity": "vitest --run parity", "test:parity-capabilities": "vitest --run parity actions/content-database-lifecycle.db.test.ts actions/bind-content-database-source-field.db.test.ts actions/_local-file-documents.test.ts actions/builder-source-review-gates.db.test.ts", "eval:parity": "agent-native eval parity", + "eval:property-preservation": "tsx parity/run-database-create-property-preservation.ts", "format.fix": "oxfmt --write .", "typecheck": "agent-native typecheck", "migrate:production": "tsx scripts/migrate-production.ts", diff --git a/templates/content/parity/README.md b/templates/content/parity/README.md index 05ac6c5b35..4c84876e57 100644 --- a/templates/content/parity/README.md +++ b/templates/content/parity/README.md @@ -10,7 +10,7 @@ PR 2.2 adds two executable tiers: existing action tests, make no model calls, and require no private provider credentials. - Gated agent evals run through `agent-native eval parity`. They are opt-in via - `CONTENT_PARITY_EVALS=1`, capped to four initial scenarios, and should be + `CONTENT_PARITY_EVALS=1`, kept to a small explicit scenario set, and should be reserved for manual or nightly checks. ## Deterministic Checks @@ -33,14 +33,23 @@ cd templates/content CONTENT_PARITY_EVALS=1 ANTHROPIC_API_KEY=... ./node_modules/.bin/agent-native eval parity ``` +The database-create property-preservation regression has a dedicated +fixture-only runner so it can inspect model-produced arguments without loading +or executing unrelated Content actions: + +```bash +CONTENT_PARITY_EVALS=1 pnpm eval:property-preservation +``` + With `CONTENT_PARITY_EVALS` unset, parity evals return skipped rows and do not call the agent runner. The CLI still exits `0`, but both readable and JSON reports mark each row with `status: "skipped"` and a `skipReason` such as `Skipped because CONTENT_PARITY_EVALS is unset`. -With the gate set, the eval files run the four PR 2.2 scenarios: +With the gate set, the eval files include these scenarios: - `database-source-scope` +- `database-create-property-preservation` - `document-search-edit` - `local-file-source-truth` - `builder-source-review-readonly` diff --git a/templates/content/parity/__tests__/database-row-property-input.test.ts b/templates/content/parity/__tests__/database-row-property-input.test.ts new file mode 100644 index 0000000000..6e31ef055b --- /dev/null +++ b/templates/content/parity/__tests__/database-row-property-input.test.ts @@ -0,0 +1,145 @@ +import { describe, expect, it } from "vitest"; + +import { + canonicalizeDatabasePropertyInput, + databasePropertyEntriesSchema, + normalizeDatabasePropertyInput, +} from "../../actions/_database-property-input"; +import { digest } from "../../actions/_database-row-mutation"; +import addDatabaseItem from "../../actions/add-database-item"; +import updateDatabaseItem from "../../actions/update-database-item"; +import upsertDatabaseItemByKey from "../../actions/upsert-database-item-by-key"; + +const rowMutationActions = [ + ["add-database-item", addDatabaseItem], + ["update-database-item", updateDatabaseItem], + ["upsert-database-item-by-key", upsertDatabaseItemByKey], +] as const; + +describe("database row property inputs", () => { + it.each(rowMutationActions)( + "%s tells the agent to preserve explicitly requested writable values", + (_name, action) => { + const properties = action.tool.parameters.properties; + expect(properties).not.toHaveProperty("propertyValues"); + const propertyEntries = properties.propertyEntries; + expect(propertyEntries.type).toBe("array"); + expect(propertyEntries.items.properties.propertyId.description).toContain( + "Exact immutable property definition ID", + ); + expect(propertyEntries.description).toContain( + "Include one entry for every schema-valid writable property value the user requested", + ); + expect(propertyEntries.description).toContain( + "never pass an empty array", + ); + expect(propertyEntries.description).toContain( + "Do not invent or clear unmentioned properties", + ); + }, + ); + + it("normalizes model-friendly entries into the strict action contract", () => { + expect( + normalizeDatabasePropertyInput({ + propertyEntries: [ + { propertyId: "status-id", value: "ready" }, + { propertyId: "evidence-id", value: "preserve me" }, + ], + }), + ).toEqual({ + "status-id": "ready", + "evidence-id": "preserve me", + }); + }); + + it("rejects duplicate property entries instead of silently overwriting", () => { + expect(() => + normalizeDatabasePropertyInput({ + propertyEntries: [ + { propertyId: "status-id", value: "ready" }, + { propertyId: "status-id", value: "changed" }, + ], + }), + ).toThrow(/provided more than once/); + }); + + it("rejects ambiguous entry and record inputs", () => { + expect(() => + normalizeDatabasePropertyInput({ + propertyEntries: [{ propertyId: "status-id", value: "ready" }], + propertyValues: { "status-id": "ready" }, + }), + ).toThrow(/not both/); + }); + + it("preserves __proto__ as an ordinary property definition ID", () => { + const propertyEntries = databasePropertyEntriesSchema.parse([ + { propertyId: "__proto__", value: "preserve me" }, + ]); + const normalized = normalizeDatabasePropertyInput({ + propertyEntries, + }); + + expect(normalized).toBeDefined(); + expect(Object.getPrototypeOf(normalized)).toBeNull(); + expect(Object.keys(normalized!)).toEqual(["__proto__"]); + expect(Object.prototype.hasOwnProperty.call(normalized, "__proto__")).toBe( + true, + ); + expect(normalized?.["__proto__"]).toBe("preserve me"); + }); + + it("removes the model-only representation before canonical hashing", () => { + const canonical = canonicalizeDatabasePropertyInput({ + idempotencyKey: "same-intent", + propertyEntries: [ + { propertyId: "status-id", value: "ready" }, + { propertyId: "evidence-id", value: "preserve me" }, + ], + }); + + expect(canonical).toEqual({ + idempotencyKey: "same-intent", + propertyValues: { + "status-id": "ready", + "evidence-id": "preserve me", + }, + }); + expect(canonical).not.toHaveProperty("propertyEntries"); + }); + + it("gives equivalent entry and record inputs the same canonical digest", () => { + const fromEntries = canonicalizeDatabasePropertyInput({ + idempotencyKey: "same-intent", + propertyEntries: [ + { propertyId: "status-id", value: "ready" }, + { propertyId: "evidence-id", value: "preserve me" }, + ], + }); + const fromRecord = canonicalizeDatabasePropertyInput({ + idempotencyKey: "same-intent", + propertyValues: { + "evidence-id": "preserve me", + "status-id": "ready", + }, + }); + + expect(digest(fromEntries)).toBe(digest(fromRecord)); + }); + + it("includes __proto__ property values in the canonical digest", () => { + const withPrototypeNamedProperty = canonicalizeDatabasePropertyInput({ + idempotencyKey: "same-intent", + propertyEntries: [{ propertyId: "__proto__", value: "preserve me" }], + }); + const withoutProperty = canonicalizeDatabasePropertyInput({ + idempotencyKey: "same-intent", + propertyValues: {}, + }); + + expect(digest(withPrototypeNamedProperty)).not.toBe( + digest(withoutProperty), + ); + }); +}); diff --git a/templates/content/parity/__tests__/eval-scenario-coverage.test.ts b/templates/content/parity/__tests__/eval-scenario-coverage.test.ts index 65fb938762..bf0892c049 100644 --- a/templates/content/parity/__tests__/eval-scenario-coverage.test.ts +++ b/templates/content/parity/__tests__/eval-scenario-coverage.test.ts @@ -7,6 +7,23 @@ import { scenarioToEval } from "../scenario-to-eval"; const OLD_GATE = process.env.CONTENT_PARITY_EVALS; +function successfulCreateCall( + scenario: (typeof parityEvalScenarios)[number], + propertyInput: Record, +) { + return { + name: "add-database-item", + input: { + ...scenario.expectedCreateEnvelope, + ...propertyInput, + }, + completed: true, + completedSideEffect: true, + isError: false, + result: '{"fixtureOnly":true}', + }; +} + afterEach(() => { if (OLD_GATE === undefined) { delete process.env.CONTENT_PARITY_EVALS; @@ -37,10 +54,11 @@ describe("Content parity eval scenarios", () => { expect(invalid).toEqual([]); }); - it("keeps PR 2.5 capped to the five bundled gated scenarios", () => { + it("keeps the bundled gated scenarios explicit", () => { expect(parityEvalScenarios.map((scenario) => scenario.id).sort()).toEqual([ "builder-source-review-readonly", "database-bulk-row-reliability", + "database-create-property-preservation", "database-source-scope", "document-search-edit", "local-file-source-truth", @@ -83,14 +101,16 @@ describe("Content parity eval scenarios", () => { ); expect(report.failed).toBe(0); - expect(report.skipped).toBe(5); + expect(report.skipped).toBe(6); expect(report.results.every((row) => row.status === "skipped")).toBe(true); }); it("runs scorer-backed evals when the gate is set", async () => { process.env.CONTENT_PARITY_EVALS = "1"; - const scenario = parityEvalScenarios[0]; + const scenario = parityEvalScenarios.find( + (candidate) => candidate.id === "database-source-scope", + )!; const evalCase = scenarioToEval(scenario); const row = await scoreEval(evalCase, { runAgent: vi.fn(async () => ({ @@ -141,4 +161,265 @@ describe("Content parity eval scenarios", () => { expect(expectedToolsScore?.reason).toContain("remove-database-items"); expect(row.status).toBe("failed"); }); + + it("fails when database creation drops explicitly requested properties", async () => { + process.env.CONTENT_PARITY_EVALS = "1"; + const scenario = parityEvalScenarios.find( + (candidate) => candidate.id === "database-create-property-preservation", + )!; + const evalCase = scenarioToEval(scenario); + const row = await scoreEval(evalCase, { + runAgent: vi.fn(async () => ({ + text: scenario.successSignals.join("\n"), + toolCalls: ["add-database-item"], + toolCallDetails: [ + successfulCreateCall(scenario, { propertyValues: {} }), + ], + ok: true, + runId: "content-parity:empty-property-values", + durationMs: 1, + })), + engine: {} as never, + model: "test-model", + analyzeContext: vi.fn(), + }); + + expect( + row.scores.find((score) => score.scorer === "expected_property_values"), + ).toMatchObject({ passed: false, score: 0 }); + expect(row.status).toBe("failed"); + }); + + it("accepts exact database creation properties without extras", async () => { + process.env.CONTENT_PARITY_EVALS = "1"; + const scenario = parityEvalScenarios.find( + (candidate) => candidate.id === "database-create-property-preservation", + )!; + const evalCase = scenarioToEval(scenario); + const row = await scoreEval(evalCase, { + runAgent: vi.fn(async () => ({ + text: scenario.successSignals.join("\n"), + toolCalls: ["add-database-item"], + toolCallDetails: [ + successfulCreateCall(scenario, { + propertyValues: scenario.expectedPropertyValues, + }), + ], + ok: true, + runId: "content-parity:exact-property-values", + durationMs: 1, + })), + engine: {} as never, + model: "test-model", + analyzeContext: vi.fn(), + }); + + expect( + row.scores.find((score) => score.scorer === "expected_property_values"), + ).toMatchObject({ passed: true, score: 1 }); + expect(row.status).toBe("passed"); + }); + + it.each([ + (scenario: (typeof parityEvalScenarios)[number]) => ({ + name: "duplicate property entries", + toolCallDetails: [ + successfulCreateCall(scenario, { + propertyEntries: [ + { propertyId: "parity-text-property-id", value: "preserve me" }, + { propertyId: "parity-text-property-id", value: "preserve me" }, + { propertyId: "parity-status-property-id", value: "ready" }, + ], + }), + ], + }), + (scenario: (typeof parityEvalScenarios)[number]) => ({ + name: "ambiguous property formats", + toolCallDetails: [ + successfulCreateCall(scenario, { + propertyEntries: [ + { propertyId: "parity-text-property-id", value: "preserve me" }, + { propertyId: "parity-status-property-id", value: "ready" }, + ], + propertyValues: { + "parity-text-property-id": "preserve me", + "parity-status-property-id": "ready", + }, + }), + ], + }), + (scenario: (typeof parityEvalScenarios)[number]) => ({ + name: "an extra row mutation", + toolCallDetails: [ + successfulCreateCall(scenario, { + propertyEntries: [ + { propertyId: "parity-text-property-id", value: "preserve me" }, + { propertyId: "parity-status-property-id", value: "ready" }, + ], + }), + { + name: "update-database-item", + input: {}, + completed: true, + isError: false, + result: "{}", + }, + ], + }), + ])("rejects invalid property behavior", async (buildCase) => { + process.env.CONTENT_PARITY_EVALS = "1"; + const scenario = parityEvalScenarios.find( + (candidate) => candidate.id === "database-create-property-preservation", + )!; + const { toolCallDetails } = buildCase(scenario); + const evalCase = scenarioToEval(scenario); + const row = await scoreEval(evalCase, { + runAgent: vi.fn(async () => ({ + text: scenario.successSignals.join("\n"), + toolCalls: toolCallDetails.map((call) => call.name), + toolCallDetails, + ok: true, + runId: "content-parity:invalid-property-input", + durationMs: 1, + })), + engine: {} as never, + model: "test-model", + analyzeContext: vi.fn(), + }); + + expect( + row.scores.find((score) => score.scorer === "expected_property_values"), + ).toMatchObject({ passed: false, score: 0 }); + expect(row.status).toBe("failed"); + }); + + it.each([ + { + name: "wrong target", + mutate(call: ReturnType) { + return { + ...call, + input: { + ...(call.input as Record), + target: { + ...((call.input as Record).target as Record< + string, + unknown + >), + databaseId: "wrong-database", + }, + }, + }; + }, + }, + { + name: "failed execution", + mutate(call: ReturnType) { + return { ...call, isError: true, result: "fixture rejected" }; + }, + }, + { + name: "skipped side effect", + mutate(call: ReturnType) { + return { ...call, completedSideEffect: false }; + }, + }, + { + name: "an extra top-level field", + mutate(call: ReturnType) { + return { + ...call, + input: { + ...(call.input as Record), + hallucinated: true, + }, + }; + }, + }, + { + name: "an extra target field", + mutate(call: ReturnType) { + const input = call.input as Record; + return { + ...call, + input: { + ...input, + target: { + ...(input.target as Record), + hallucinated: true, + }, + }, + }; + }, + }, + { + name: "an extra authority field", + mutate(call: ReturnType) { + const input = call.input as Record; + const target = input.target as Record; + return { + ...call, + input: { + ...input, + target: { + ...target, + authorityScope: { + ...(target.authorityScope as Record), + hallucinated: true, + }, + }, + }, + }; + }, + }, + { + name: "an extra property-entry field", + mutate(call: ReturnType) { + const input = call.input as Record; + const { propertyValues, ...withoutPropertyValues } = input; + return { + ...call, + input: { + ...withoutPropertyValues, + propertyEntries: Object.entries( + propertyValues as Record, + ).map(([propertyId, value]) => ({ + propertyId, + value, + hallucinated: true, + })), + }, + }; + }, + }, + ])("rejects $name", async ({ mutate }) => { + process.env.CONTENT_PARITY_EVALS = "1"; + const scenario = parityEvalScenarios.find( + (candidate) => candidate.id === "database-create-property-preservation", + )!; + const call = mutate( + successfulCreateCall(scenario, { + propertyValues: scenario.expectedPropertyValues, + }), + ); + const evalCase = scenarioToEval(scenario); + const row = await scoreEval(evalCase, { + runAgent: vi.fn(async () => ({ + text: scenario.successSignals.join("\n"), + toolCalls: ["add-database-item"], + toolCallDetails: [call], + ok: true, + runId: "content-parity:rejected-create", + durationMs: 1, + })), + engine: {} as never, + model: "test-model", + analyzeContext: vi.fn(), + }); + + expect( + row.scores.find((score) => score.scorer === "expected_property_values"), + ).toMatchObject({ passed: false, score: 0 }); + expect(row.status).toBe("failed"); + }); }); diff --git a/templates/content/parity/eval-scenarios.ts b/templates/content/parity/eval-scenarios.ts index b8bcfb76dd..3c26553e2b 100644 --- a/templates/content/parity/eval-scenarios.ts +++ b/templates/content/parity/eval-scenarios.ts @@ -8,9 +8,56 @@ export interface ParityEvalScenario { prompt: string; successSignals: string[]; expectedTools?: string[]; + expectedPropertyValues?: Record; + expectedCreateEnvelope?: { + target: { + authorityScope: { kind: "personal"; id: string }; + spaceId: string; + databaseId: string; + databaseDocumentId: string; + }; + expectedSchemaRevision: string; + idempotencyKey: string; + title: string; + }; } export const parityEvalScenarios: ParityEvalScenario[] = [ + { + id: "database-create-property-preservation", + title: "Database create property preservation", + capabilityIds: ["database.rows"], + gateEnv: "CONTENT_PARITY_EVALS", + defaultState: "skipped", + requiresPrivateCredentials: false, + prompt: + "Create exactly one row in the already-verified fixture Content database. Do not rediscover or alter the target. Use target authorityScope { kind: personal, id: fixture-owner@example.test }, spaceId fixture_personal_space, databaseId fixture_feedback_database, and databaseDocumentId fixture_feedback_document; expectedSchemaRevision fixture_schema_revision; title [FIXTURE] preserve explicit properties; idempotencyKey fixture-create-property-preservation-v1. Set property fixture_status_property to status-cannot-verify and property fixture_evidence_property to Baseline fixture preserve-me. No Blocks value was requested. Call add-database-item once with every exact target constraint and both property entries, then report its result truthfully.", + successSignals: [ + "Uses add-database-item once for the exact fixture target.", + "Preserves both explicitly requested writable property values.", + "Does not invent a Blocks value or another property.", + "Reports an action failure rather than claiming a row was created if the fixture is unavailable.", + ], + expectedTools: ["add-database-item"], + expectedPropertyValues: { + fixture_status_property: "status-cannot-verify", + fixture_evidence_property: "Baseline fixture preserve-me", + }, + expectedCreateEnvelope: { + target: { + authorityScope: { + kind: "personal", + id: "fixture-owner@example.test", + }, + spaceId: "fixture_personal_space", + databaseId: "fixture_feedback_database", + databaseDocumentId: "fixture_feedback_document", + }, + expectedSchemaRevision: "fixture_schema_revision", + idempotencyKey: "fixture-create-property-preservation-v1", + title: "[FIXTURE] preserve explicit properties", + }, + }, { id: "database-bulk-row-reliability", title: "Bulk database row reliability", diff --git a/templates/content/parity/run-database-create-property-preservation.ts b/templates/content/parity/run-database-create-property-preservation.ts new file mode 100644 index 0000000000..1713a9fd52 --- /dev/null +++ b/templates/content/parity/run-database-create-property-preservation.ts @@ -0,0 +1,45 @@ +import { createAgentRunner, runEvals } from "@agent-native/core/eval"; + +import addDatabaseItem from "../actions/add-database-item.ts"; +import { parityEvalScenarios } from "./eval-scenarios.ts"; +import { scenarioToEval } from "./scenario-to-eval.ts"; + +const scenario = parityEvalScenarios.find( + (candidate) => candidate.id === "database-create-property-preservation", +); +if (!scenario) { + throw new Error("Missing database create property preservation scenario."); +} + +const evalCase = scenarioToEval(scenario); +evalCase.scorers = evalCase.scorers.filter( + (scorer) => + scorer.name === "expected_tools" || + scorer.name === "expected_property_values", +); + +const runner = await createAgentRunner({ + actions: { + "add-database-item": { + ...addDatabaseItem, + run: async (input) => ({ fixtureOnly: true, received: input }), + }, + }, + systemPrompt: + "You are Content's AI document assistant. Use the registered Content action and preserve exact user-supplied target constraints, property IDs, and property values. Never invent fields or claim an action succeeded when it failed.", +}); + +const report = await runEvals([evalCase], runner, { persist: false }); +console.log( + JSON.stringify( + { + engine: runner.engine.name, + model: runner.model, + report, + }, + null, + 2, + ), +); + +process.exitCode = report.failed === 0 ? 0 : 1; diff --git a/templates/content/parity/scenario-to-eval.ts b/templates/content/parity/scenario-to-eval.ts index 6385b1d8c2..6f7e50d378 100644 --- a/templates/content/parity/scenario-to-eval.ts +++ b/templates/content/parity/scenario-to-eval.ts @@ -30,6 +30,228 @@ function expectedToolScorer(expectedTools: string[]) { }); } +function hasExactKeys( + record: Record, + expectedKeys: readonly string[], +): boolean { + const actualKeys = Object.keys(record).sort(); + const sortedExpectedKeys = [...expectedKeys].sort(); + return ( + actualKeys.length === sortedExpectedKeys.length && + actualKeys.every((key, index) => key === sortedExpectedKeys[index]) + ); +} + +function analyzePropertyValues(input: unknown): { + received: Record; + invalid: string[]; +} { + if (!input || typeof input !== "object" || Array.isArray(input)) { + return { received: {}, invalid: ["tool input is not an object"] }; + } + const record = input as Record; + const hasEntries = record.propertyEntries !== undefined; + const hasValues = record.propertyValues !== undefined; + if (hasEntries && hasValues) { + return { + received: {}, + invalid: ["propertyEntries and propertyValues were both provided"], + }; + } + if (hasValues) { + if ( + !record.propertyValues || + typeof record.propertyValues !== "object" || + Array.isArray(record.propertyValues) + ) { + return { received: {}, invalid: ["propertyValues is not a record"] }; + } + return { + received: record.propertyValues as Record, + invalid: [], + }; + } + if (!hasEntries || !Array.isArray(record.propertyEntries)) { + return { received: {}, invalid: ["propertyEntries is not an array"] }; + } + + const received: Record = Object.create(null) as Record< + string, + unknown + >; + const invalid: string[] = []; + for (const entry of record.propertyEntries) { + if (!entry || typeof entry !== "object" || Array.isArray(entry)) { + invalid.push("propertyEntries contains a non-object entry"); + continue; + } + const { propertyId, value } = entry as Record; + if ( + !hasExactKeys(entry as Record, ["propertyId", "value"]) + ) { + invalid.push( + "propertyEntries contains an entry with unrecognized fields", + ); + continue; + } + if (typeof propertyId !== "string" || propertyId.length === 0) { + invalid.push("propertyEntries contains an invalid propertyId"); + continue; + } + if (Object.prototype.hasOwnProperty.call(received, propertyId)) { + invalid.push(`propertyEntries contains duplicate ID ${propertyId}`); + continue; + } + received[propertyId] = value; + } + return { received, invalid }; +} + +const databaseRowMutationTools = new Set([ + "add-database-item", + "update-database-item", + "upsert-database-item-by-key", + "duplicate-database-items", + "remove-database-items", +]); + +function matchesCreateEnvelope( + input: Record, + expected: NonNullable, +): boolean { + const target = input.target; + if (!target || typeof target !== "object" || Array.isArray(target)) { + return false; + } + const actualTarget = target as Record; + const authorityScope = actualTarget.authorityScope; + if ( + !authorityScope || + typeof authorityScope !== "object" || + Array.isArray(authorityScope) + ) { + return false; + } + const actualAuthority = authorityScope as Record; + return ( + hasExactKeys(input, [ + "target", + "expectedSchemaRevision", + "idempotencyKey", + "title", + input.propertyEntries === undefined + ? "propertyValues" + : "propertyEntries", + ]) && + hasExactKeys(actualTarget, [ + "authorityScope", + "spaceId", + "databaseId", + "databaseDocumentId", + ]) && + hasExactKeys(actualAuthority, ["kind", "id"]) && + actualAuthority.kind === expected.target.authorityScope.kind && + actualAuthority.id === expected.target.authorityScope.id && + actualTarget.spaceId === expected.target.spaceId && + actualTarget.databaseId === expected.target.databaseId && + actualTarget.databaseDocumentId === expected.target.databaseDocumentId && + input.expectedSchemaRevision === expected.expectedSchemaRevision && + input.idempotencyKey === expected.idempotencyKey && + input.title === expected.title + ); +} + +function expectedPropertyValuesScorer( + expected: Record, + expectedEnvelope?: ParityEvalScenario["expectedCreateEnvelope"], +) { + return createScorer< + AgentRunOutput, + { + received: Record; + missing: string[]; + unexpected: string[]; + invalid: string[]; + mutationCalls: string[]; + } + >({ + name: "expected_property_values", + analyze(run) { + const mutationCalls = (run.toolCallDetails ?? []) + .filter((call) => databaseRowMutationTools.has(call.name)) + .map((call) => call.name); + const createCalls = (run.toolCallDetails ?? []).filter( + (call) => call.name === "add-database-item", + ); + const analysis = analyzePropertyValues(createCalls[0]?.input); + const invalid = [...analysis.invalid]; + if (createCalls.length !== 1) { + invalid.push( + `expected exactly one add-database-item call, received ${createCalls.length}`, + ); + } + if (mutationCalls.length !== 1) { + invalid.push( + `expected exactly one row mutation, received ${mutationCalls.length}`, + ); + } + const createInput = createCalls[0]?.input; + if ( + expectedEnvelope && + (!createInput || + typeof createInput !== "object" || + Array.isArray(createInput) || + !matchesCreateEnvelope( + createInput as Record, + expectedEnvelope, + )) + ) { + invalid.push( + "create target, schema revision, idempotency key, or title did not match the fixture", + ); + } + if ( + !createCalls[0]?.completed || + createCalls[0]?.completedSideEffect !== true || + createCalls[0]?.isError + ) { + invalid.push("add-database-item did not complete successfully"); + } + if (!run.ok) { + invalid.push("agent run did not complete successfully"); + } + const received = analysis.received; + const missing = Object.entries(expected) + .filter(([propertyId, value]) => received[propertyId] !== value) + .map(([propertyId]) => propertyId); + const unexpected = Object.keys(received).filter( + (propertyId) => + !Object.prototype.hasOwnProperty.call(expected, propertyId), + ); + return { received, missing, unexpected, invalid, mutationCalls }; + }, + generateScore({ missing, unexpected, invalid }) { + return missing.length === 0 && + unexpected.length === 0 && + invalid.length === 0 + ? 1 + : 0; + }, + generateReason({ + analysis: { received, missing, unexpected, invalid, mutationCalls }, + }) { + if ( + missing.length === 0 && + unexpected.length === 0 && + invalid.length === 0 + ) { + return "Agent preserved every expected property ID and exact value without inventing another property."; + } + return `Received propertyValues ${JSON.stringify(received)}; mutations: ${mutationCalls.join(", ") || "none"}; missing or changed: ${missing.join(", ") || "none"}; unexpected: ${unexpected.join(", ") || "none"}; invalid: ${invalid.join("; ") || "none"}`; + }, + }); +} + export function scenarioToEval(scenario: ParityEvalScenario): Eval { const name = `content-parity:${scenario.id}`; @@ -52,6 +274,14 @@ export function scenarioToEval(scenario: ParityEvalScenario): Eval { ...(scenario.expectedTools?.length ? [expectedToolScorer(scenario.expectedTools)] : []), + ...(scenario.expectedPropertyValues + ? [ + expectedPropertyValuesScorer( + scenario.expectedPropertyValues, + scenario.expectedCreateEnvelope, + ), + ] + : []), ], }); }