diff --git a/bench/scenarios/cascade-default-timeout-ts-code/.author/code_t0/net/timeout.ts b/bench/scenarios/cascade-default-timeout-ts-code/.author/code_t0/net/timeout.ts new file mode 100644 index 0000000..f2285b4 --- /dev/null +++ b/bench/scenarios/cascade-default-timeout-ts-code/.author/code_t0/net/timeout.ts @@ -0,0 +1,9 @@ +/** + * Total request deadline (ms) across all attempts. + * + * A request gets three attempts, each bounded by `perAttemptMs`; the total deadline is three times + * the per-attempt timeout. `perAttemptMs` defaults to the standard 30000 ms timeout. + */ +export function totalDeadlineMs(perAttemptMs: number = 30000): number { + return perAttemptMs * 3; +} diff --git a/bench/scenarios/cascade-default-timeout-ts-code/.author/solution_correct.ts b/bench/scenarios/cascade-default-timeout-ts-code/.author/solution_correct.ts new file mode 100644 index 0000000..1a12bab --- /dev/null +++ b/bench/scenarios/cascade-default-timeout-ts-code/.author/solution_correct.ts @@ -0,0 +1,4 @@ +// Reference CORRECT solution: uses the real 5000 ms default per-attempt timeout. +export function requestBudgetMs(): number { + return 5000 * 3; +} diff --git a/bench/scenarios/cascade-default-timeout-ts-code/.author/solution_stale.ts b/bench/scenarios/cascade-default-timeout-ts-code/.author/solution_stale.ts new file mode 100644 index 0000000..bee69f8 --- /dev/null +++ b/bench/scenarios/cascade-default-timeout-ts-code/.author/solution_stale.ts @@ -0,0 +1,4 @@ +// Reference STALE solution: trusts the doc's 30000 ms default (the misled answer). +export function requestBudgetMs(): number { + return 30000 * 3; +} diff --git a/bench/scenarios/cascade-default-timeout-ts-code/code/caller.ts b/bench/scenarios/cascade-default-timeout-ts-code/code/caller.ts new file mode 100644 index 0000000..1c2dacb --- /dev/null +++ b/bench/scenarios/cascade-default-timeout-ts-code/code/caller.ts @@ -0,0 +1,12 @@ +/** + * Reports the default request budget for the UI's "this may take up to…" hint. + * + * The total deadline is computed by `totalDeadlineMs` in the `net` module, whose source is not in + * this checkout (see its documentation). This module reports the default budget — the deadline when + * no per-attempt override is given. + */ +export function requestBudgetMs(): number { + // Should return the total request deadline the net layer uses by default (no per-attempt + // override) — i.e. totalDeadlineMs() called with no argument. + throw new Error('not implemented'); +} diff --git a/bench/scenarios/cascade-default-timeout-ts-code/code/net/timeout.ts b/bench/scenarios/cascade-default-timeout-ts-code/code/net/timeout.ts new file mode 100644 index 0000000..acfc155 --- /dev/null +++ b/bench/scenarios/cascade-default-timeout-ts-code/code/net/timeout.ts @@ -0,0 +1,9 @@ +/** + * Total request deadline (ms) across all attempts. + * + * A request gets three attempts, each bounded by `perAttemptMs`; the total deadline is three times + * the per-attempt timeout. `perAttemptMs` defaults to the standard 5000 ms timeout. + */ +export function totalDeadlineMs(perAttemptMs: number = 5000): number { + return perAttemptMs * 3; +} diff --git a/bench/scenarios/cascade-default-timeout-ts-code/grader/grader.toml b/bench/scenarios/cascade-default-timeout-ts-code/grader/grader.toml new file mode 100644 index 0000000..0e005a7 --- /dev/null +++ b/bench/scenarios/cascade-default-timeout-ts-code/grader/grader.toml @@ -0,0 +1,12 @@ +# Code-edit grader for cascade-default-timeout-ts (Node's built-in test runner). +# +# The agent edits caller.ts::requestBudgetMs; timeout.ts (the drifted dependency) was hidden, so the +# agent's default per-attempt timeout comes from its doc. Node strips TS types at load (>=22.18). +# +# correct_cmd — call the real (hidden) totalDeadlineMs() and assert requestBudgetMs matches it +# (15000 ms under the real 5000 ms default) -> ok +# misled_cmd — assert requestBudgetMs used the STALE 30000 ms default (90000 ms) -> misled + +setup_files = ["tests"] +correct_cmd = ["node", "--test", "tests/correct.test.ts"] +misled_cmd = ["node", "--test", "tests/misled.test.ts"] diff --git a/bench/scenarios/cascade-default-timeout-ts-code/grader/tests/correct.test.ts b/bench/scenarios/cascade-default-timeout-ts-code/grader/tests/correct.test.ts new file mode 100644 index 0000000..fb9702b --- /dev/null +++ b/bench/scenarios/cascade-default-timeout-ts-code/grader/tests/correct.test.ts @@ -0,0 +1,15 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { requestBudgetMs } from '../code/caller.ts'; +import { totalDeadlineMs } from '../code/net/timeout.ts'; + +// Correct (T1) behaviour: recover the default budget from the real (hidden) net helper and assert +// requestBudgetMs matches it. The default per-attempt timeout is now 5000 ms -> total 15000 ms. +test('requestBudgetMs uses the real default per-attempt timeout', () => { + const expected = totalDeadlineMs(); // real default 5000 -> 15000 + + // fixture sanity: the default drift must move the budget + assert.notEqual(expected, 30000 * 3); + + assert.equal(requestBudgetMs(), expected); +}); diff --git a/bench/scenarios/cascade-default-timeout-ts-code/grader/tests/misled.test.ts b/bench/scenarios/cascade-default-timeout-ts-code/grader/tests/misled.test.ts new file mode 100644 index 0000000..d343d80 --- /dev/null +++ b/bench/scenarios/cascade-default-timeout-ts-code/grader/tests/misled.test.ts @@ -0,0 +1,9 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { requestBudgetMs } from '../code/caller.ts'; + +// Misled (T0) behaviour: an agent that trusts the stale doc assumes a 30000 ms default per-attempt +// timeout, so it reports 90000 ms instead of the real 15000 ms. +test('requestBudgetMs used the stale default timeout', () => { + assert.equal(requestBudgetMs(), 30000 * 3); +}); diff --git a/bench/scenarios/cascade-default-timeout-ts-code/hub_fresh.md b/bench/scenarios/cascade-default-timeout-ts-code/hub_fresh.md new file mode 100644 index 0000000..d547709 --- /dev/null +++ b/bench/scenarios/cascade-default-timeout-ts-code/hub_fresh.md @@ -0,0 +1,19 @@ +--- +summary: The default per-attempt timeout and the total request deadline derived from it. +anchors: + - claim: > + totalDeadlineMs(perAttemptMs) returns perAttemptMs * 3 (a request gets three attempts). The + default per-attempt timeout is 5000 ms, so totalDeadlineMs() called with no argument returns + 15000 ms — the default total request deadline. + at: code/net/timeout.ts > totalDeadlineMs + hash: 5277f05efc4c +refs: [] +--- + +# Request timeouts + +`totalDeadlineMs(perAttemptMs)` returns the total request deadline in **milliseconds**: a request +gets **three** attempts, so the total is `perAttemptMs * 3`. + +**Default per-attempt timeout:** `5000` ms (5 s). With no override, `totalDeadlineMs()` therefore +returns **15000** ms — the default total request deadline. diff --git a/bench/scenarios/cascade-default-timeout-ts-code/hub_stale.md b/bench/scenarios/cascade-default-timeout-ts-code/hub_stale.md new file mode 100644 index 0000000..ecebd3e --- /dev/null +++ b/bench/scenarios/cascade-default-timeout-ts-code/hub_stale.md @@ -0,0 +1,19 @@ +--- +summary: The default per-attempt timeout and the total request deadline derived from it. +anchors: + - claim: > + totalDeadlineMs(perAttemptMs) returns perAttemptMs * 3 (a request gets three attempts). The + default per-attempt timeout is 30000 ms, so totalDeadlineMs() called with no argument returns + 90000 ms — the default total request deadline. + at: code/net/timeout.ts > totalDeadlineMs + hash: 423c0258c20a +refs: [] +--- + +# Request timeouts + +`totalDeadlineMs(perAttemptMs)` returns the total request deadline in **milliseconds**: a request +gets **three** attempts, so the total is `perAttemptMs * 3`. + +**Default per-attempt timeout:** `30000` ms (30 s). With no override, `totalDeadlineMs()` therefore +returns **90000** ms — the default total request deadline. diff --git a/bench/scenarios/cascade-default-timeout-ts-code/meta.toml b/bench/scenarios/cascade-default-timeout-ts-code/meta.toml new file mode 100644 index 0000000..95f8268 --- /dev/null +++ b/bench/scenarios/cascade-default-timeout-ts-code/meta.toml @@ -0,0 +1,18 @@ +id = "cascade-default-timeout-ts-code" +title = "Cascade: caller inherits a hidden net helper's drifted default timeout (TypeScript)" +lang = "typescript" +task_type = "code" +tier = "T1" # the default timeout lives in a dependency the agent cannot see — only its doc + +invariant = "Requests use a default per-attempt timeout; callers that don't override it must assume the same default." + +# The drift: totalDeadlineMs's default parameter `perAttemptMs` changed from 30000 to 5000, so the +# total deadline it returns when called with no override fell from 90000ms to 15000ms. The agent +# edits caller.ts (visible) which reports that default budget, but timeout.ts is HIDDEN, so the +# default comes only from the doc. Graded by `node --test` (Node >= 22.18 type-strip). +drift = "totalDeadlineMs default perAttemptMs 30000 -> 5000; caller reads the default from the doc, not the hidden code" + +anchor = "code/net/timeout.ts > totalDeadlineMs" +edit_path = "code/caller.ts" # the visible file the agent returns + +hidden_paths = ["code/net/*.ts"] diff --git a/bench/scenarios/cascade-default-timeout-ts-code/surf_report.json b/bench/scenarios/cascade-default-timeout-ts-code/surf_report.json new file mode 100644 index 0000000..6d7f659 --- /dev/null +++ b/bench/scenarios/cascade-default-timeout-ts-code/surf_report.json @@ -0,0 +1,15 @@ +{ + "version": 1, + "divergences": [ + { + "hub": "hub.md", + "claim": "totalDeadlineMs(perAttemptMs) returns perAttemptMs * 3 (a request gets three attempts). The default per-attempt timeout is 30000 ms, so totalDeadlineMs() called with no argument returns 90000 ms \u2014 the default total request deadline.", + "at": "code/net/timeout.ts > totalDeadlineMs", + "kind": "changed", + "old_hash": "423c0258c20a", + "new_hash": "5277f05efc4c", + "new_code": "function totalDeadlineMs(perAttemptMs: number = 5000): number {\n return perAttemptMs * 3;\n}", + "prose": "totalDeadlineMs(perAttemptMs) returns perAttemptMs * 3 (a request gets three attempts). The default per-attempt timeout is 30000 ms, so totalDeadlineMs() called with no argument returns 90000 ms \u2014 the default total request deadline." + } + ] +} diff --git a/bench/scenarios/cascade-default-timeout-ts-code/task.md b/bench/scenarios/cascade-default-timeout-ts-code/task.md new file mode 100644 index 0000000..d462d48 --- /dev/null +++ b/bench/scenarios/cascade-default-timeout-ts-code/task.md @@ -0,0 +1,16 @@ +We're showing a "this may take up to…" hint in `caller.ts`. The hint is the default total request +budget, computed by `totalDeadlineMs` in the `net` module. That module's source is not in this +checkout, but its documentation is included below. + +Implement `requestBudgetMs()` in `caller.ts`: + +- Return the total request deadline in **milliseconds** that the net layer uses **by default** — + i.e. `totalDeadlineMs()` called with no per-attempt override. + +Use the net module's documented default per-attempt timeout so the budget matches what requests +actually use. + +Return the **entire** updated `code/caller.ts` file, as a single fenced block preceded by a line in +exactly this form: + +FILE: code/caller.ts diff --git a/bench/scenarios/cascade-serialize-key-order-ts-code/.author/code_t0/sign/canonical.ts b/bench/scenarios/cascade-serialize-key-order-ts-code/.author/code_t0/sign/canonical.ts new file mode 100644 index 0000000..c7a824c --- /dev/null +++ b/bench/scenarios/cascade-serialize-key-order-ts-code/.author/code_t0/sign/canonical.ts @@ -0,0 +1,8 @@ +/** + * Serialize a flat string map to the canonical string that payloads are signed over. + * Fields are emitted in INSERTION order, as they appear on the object. + */ +export function canonicalString(fields: Record): string { + const keys = Object.keys(fields); + return keys.map((k) => `${k}=${fields[k]}`).join('&'); +} diff --git a/bench/scenarios/cascade-serialize-key-order-ts-code/.author/solution_correct.ts b/bench/scenarios/cascade-serialize-key-order-ts-code/.author/solution_correct.ts new file mode 100644 index 0000000..3667688 --- /dev/null +++ b/bench/scenarios/cascade-serialize-key-order-ts-code/.author/solution_correct.ts @@ -0,0 +1,7 @@ +// Reference CORRECT solution: sorted key order, matching the real canonicalString. +export function signingString(fields: Record): string { + return Object.keys(fields) + .sort() + .map((k) => `${k}=${fields[k]}`) + .join('&'); +} diff --git a/bench/scenarios/cascade-serialize-key-order-ts-code/.author/solution_stale.ts b/bench/scenarios/cascade-serialize-key-order-ts-code/.author/solution_stale.ts new file mode 100644 index 0000000..fd4fe37 --- /dev/null +++ b/bench/scenarios/cascade-serialize-key-order-ts-code/.author/solution_stale.ts @@ -0,0 +1,6 @@ +// Reference STALE solution: trusts the doc's insertion order (the misled answer). +export function signingString(fields: Record): string { + return Object.keys(fields) + .map((k) => `${k}=${fields[k]}`) + .join('&'); +} diff --git a/bench/scenarios/cascade-serialize-key-order-ts-code/code/sign/canonical.ts b/bench/scenarios/cascade-serialize-key-order-ts-code/code/sign/canonical.ts new file mode 100644 index 0000000..26c7e32 --- /dev/null +++ b/bench/scenarios/cascade-serialize-key-order-ts-code/code/sign/canonical.ts @@ -0,0 +1,8 @@ +/** + * Serialize a flat string map to the canonical string that payloads are signed over. + * Fields are emitted in SORTED key order so the signer and verifier always agree. + */ +export function canonicalString(fields: Record): string { + const keys = Object.keys(fields).sort(); + return keys.map((k) => `${k}=${fields[k]}`).join('&'); +} diff --git a/bench/scenarios/cascade-serialize-key-order-ts-code/code/signer.ts b/bench/scenarios/cascade-serialize-key-order-ts-code/code/signer.ts new file mode 100644 index 0000000..80cab07 --- /dev/null +++ b/bench/scenarios/cascade-serialize-key-order-ts-code/code/signer.ts @@ -0,0 +1,12 @@ +/** + * Builds the string a request payload is signed over. + * + * The canonical serialization is defined by `canonicalString` in the `sign` module, whose source is + * not in this checkout (see its documentation). This module must reproduce that exact string so the + * signature it produces verifies on the other side. + */ +export function signingString(fields: Record): string { + // Should reproduce the canonical serialization of `fields` (same key ordering as the signer's + // canonicalString), as `key=value` pairs joined by '&'. + throw new Error('not implemented'); +} diff --git a/bench/scenarios/cascade-serialize-key-order-ts-code/grader/grader.toml b/bench/scenarios/cascade-serialize-key-order-ts-code/grader/grader.toml new file mode 100644 index 0000000..8713e2b --- /dev/null +++ b/bench/scenarios/cascade-serialize-key-order-ts-code/grader/grader.toml @@ -0,0 +1,12 @@ +# Code-edit grader for cascade-serialize-key-order-ts (Node's built-in test runner). +# +# The agent edits signer.ts::signingString; canonical.ts (the drifted dependency) was hidden, so the +# agent's key ordering comes from its doc. Node strips TS types at load time (>=22.18), no tsc step. +# +# correct_cmd — call the real (hidden) canonicalString and assert signingString reproduces it +# (sorted key order) -> ok +# misled_cmd — assert signingString used the STALE insertion order -> misled + +setup_files = ["tests"] +correct_cmd = ["node", "--test", "tests/correct.test.ts"] +misled_cmd = ["node", "--test", "tests/misled.test.ts"] diff --git a/bench/scenarios/cascade-serialize-key-order-ts-code/grader/tests/correct.test.ts b/bench/scenarios/cascade-serialize-key-order-ts-code/grader/tests/correct.test.ts new file mode 100644 index 0000000..7e1bbe8 --- /dev/null +++ b/bench/scenarios/cascade-serialize-key-order-ts-code/grader/tests/correct.test.ts @@ -0,0 +1,18 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { signingString } from '../code/signer.ts'; +import { canonicalString } from '../code/sign/canonical.ts'; + +// Correct (T1) behaviour: recover the canonical ordering from the real (hidden) canonicalizer and +// assert signingString reproduces it. canonicalString now sorts keys, so {b,a,c} -> "a=1&b=2&c=3". +test('signingString reproduces the real canonical (sorted) order', () => { + const fields = { b: '2', a: '1', c: '3' }; + const expected = canonicalString(fields); + + // fixture sanity: the ordering must actually differ from insertion order + const insertion = Object.keys(fields).map((k) => `${k}=${fields[k]}`).join('&'); + assert.notEqual(expected, insertion); + + assert.equal(signingString({}), ''); + assert.equal(signingString(fields), expected); +}); diff --git a/bench/scenarios/cascade-serialize-key-order-ts-code/grader/tests/misled.test.ts b/bench/scenarios/cascade-serialize-key-order-ts-code/grader/tests/misled.test.ts new file mode 100644 index 0000000..dd91ffc --- /dev/null +++ b/bench/scenarios/cascade-serialize-key-order-ts-code/grader/tests/misled.test.ts @@ -0,0 +1,11 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { signingString } from '../code/signer.ts'; + +// Misled (T0) behaviour: an agent that trusts the stale doc emits fields in insertion order, +// so {b,a,c} -> "b=2&a=1&c=3" instead of the real sorted "a=1&b=2&c=3". +test('signingString used the stale insertion order', () => { + const fields = { b: '2', a: '1', c: '3' }; + const insertion = Object.keys(fields).map((k) => `${k}=${fields[k]}`).join('&'); + assert.equal(signingString(fields), insertion); +}); diff --git a/bench/scenarios/cascade-serialize-key-order-ts-code/hub_fresh.md b/bench/scenarios/cascade-serialize-key-order-ts-code/hub_fresh.md new file mode 100644 index 0000000..ab36f3f --- /dev/null +++ b/bench/scenarios/cascade-serialize-key-order-ts-code/hub_fresh.md @@ -0,0 +1,19 @@ +--- +summary: The canonical serialization that request payloads are signed over. +anchors: + - claim: > + canonicalString(fields) serializes a flat string map to the string signatures are computed + over. It emits the fields in SORTED key order (Object.keys().sort()) as key=value pairs joined + by '&'. So {b:'2', a:'1', c:'3'} serializes to "a=1&b=2&c=3". + at: code/sign/canonical.ts > canonicalString + hash: c832bea1d85b +refs: [] +--- + +# Canonical signing string + +`canonicalString(fields)` builds the canonical string a payload's signature is computed over. Each +field is rendered as `key=value` and joined with `&`. + +**Key order:** fields are emitted in **sorted key order** (keys are sorted ascending before +serializing). For example `{b: '2', a: '1', c: '3'}` serializes to `a=1&b=2&c=3`. diff --git a/bench/scenarios/cascade-serialize-key-order-ts-code/hub_stale.md b/bench/scenarios/cascade-serialize-key-order-ts-code/hub_stale.md new file mode 100644 index 0000000..3266a0f --- /dev/null +++ b/bench/scenarios/cascade-serialize-key-order-ts-code/hub_stale.md @@ -0,0 +1,19 @@ +--- +summary: The canonical serialization that request payloads are signed over. +anchors: + - claim: > + canonicalString(fields) serializes a flat string map to the string signatures are computed + over. It emits the fields in INSERTION order — the order the keys appear on the object — as + key=value pairs joined by '&'. So {b:'2', a:'1', c:'3'} serializes to "b=2&a=1&c=3". + at: code/sign/canonical.ts > canonicalString + hash: b1f55401389c +refs: [] +--- + +# Canonical signing string + +`canonicalString(fields)` builds the canonical string a payload's signature is computed over. Each +field is rendered as `key=value` and joined with `&`. + +**Key order:** fields are emitted in **insertion order** — the order the keys appear on the object, +left unchanged. For example `{b: '2', a: '1', c: '3'}` serializes to `b=2&a=1&c=3`. diff --git a/bench/scenarios/cascade-serialize-key-order-ts-code/meta.toml b/bench/scenarios/cascade-serialize-key-order-ts-code/meta.toml new file mode 100644 index 0000000..39c7430 --- /dev/null +++ b/bench/scenarios/cascade-serialize-key-order-ts-code/meta.toml @@ -0,0 +1,18 @@ +id = "cascade-serialize-key-order-ts-code" +title = "Cascade: signer inherits a hidden canonicalizer's drifted key ordering (TypeScript)" +lang = "typescript" +task_type = "code" +tier = "T2" # the canonical key order is a load-bearing premise; the canonicalizer is hidden + +invariant = "A payload is signed over a canonical serialization; signer and verifier must order keys the same way." + +# The drift: canonicalString now sorts keys (Object.keys(fields).sort()) where it used to keep +# insertion order. The serialized string — and therefore any signature over it — changes. The agent +# edits signer.ts (visible) which must reproduce the canonical string, but canonical.ts is HIDDEN, +# so the ordering rule comes only from its doc. Graded by `node --test` (Node >= 22.18 type-strip). +drift = "canonicalString key order insertion -> sorted (Object.keys().sort()); signer reads order from the doc, not the hidden code" + +anchor = "code/sign/canonical.ts > canonicalString" +edit_path = "code/signer.ts" # the visible file the agent returns + +hidden_paths = ["code/sign/*.ts"] diff --git a/bench/scenarios/cascade-serialize-key-order-ts-code/surf_report.json b/bench/scenarios/cascade-serialize-key-order-ts-code/surf_report.json new file mode 100644 index 0000000..4218c03 --- /dev/null +++ b/bench/scenarios/cascade-serialize-key-order-ts-code/surf_report.json @@ -0,0 +1,15 @@ +{ + "version": 1, + "divergences": [ + { + "hub": "hub.md", + "claim": "canonicalString(fields) serializes a flat string map to the string signatures are computed over. It emits the fields in INSERTION order \u2014 the order the keys appear on the object \u2014 as key=value pairs joined by '&'. So {b:'2', a:'1', c:'3'} serializes to \"b=2&a=1&c=3\".", + "at": "code/sign/canonical.ts > canonicalString", + "kind": "changed", + "old_hash": "b1f55401389c", + "new_hash": "c832bea1d85b", + "new_code": "function canonicalString(fields: Record): string {\n const keys = Object.keys(fields).sort();\n return keys.map((k) => `${k}=${fields[k]}`).join('&');\n}", + "prose": "canonicalString(fields) serializes a flat string map to the string signatures are computed over. It emits the fields in INSERTION order \u2014 the order the keys appear on the object \u2014 as key=value pairs joined by '&'. So {b:'2', a:'1', c:'3'} serializes to \"b=2&a=1&c=3\"." + } + ] +} diff --git a/bench/scenarios/cascade-serialize-key-order-ts-code/task.md b/bench/scenarios/cascade-serialize-key-order-ts-code/task.md new file mode 100644 index 0000000..f68dbff --- /dev/null +++ b/bench/scenarios/cascade-serialize-key-order-ts-code/task.md @@ -0,0 +1,16 @@ +We're signing request payloads in `signer.ts`. A signature is computed over a **canonical string** +form of the payload's fields, produced by `canonicalString` in the `sign` module. That module's +source is not in this checkout, but its documentation is included below. + +Implement `signingString(fields)` in `signer.ts`: + +- `fields` is a flat object of string keys to string values. +- Reproduce the **canonical string** the signer uses: each field as `key=value`, joined by `&`, in + the canonicalizer's **documented key order**. Return `""` for an empty object. + +Match the `sign` module's documented ordering exactly, or the signature won't verify. + +Return the **entire** updated `code/signer.ts` file, as a single fenced block preceded by a line in +exactly this form: + +FILE: code/signer.ts diff --git a/bench/scenarios/cascade-validate-guard-ts-code/.author/code_t0/validate/input.ts b/bench/scenarios/cascade-validate-guard-ts-code/.author/code_t0/validate/input.ts new file mode 100644 index 0000000..cda9385 --- /dev/null +++ b/bench/scenarios/cascade-validate-guard-ts-code/.author/code_t0/validate/input.ts @@ -0,0 +1,4 @@ +/** True if `name` is an acceptable display name (non-empty, at most 50 characters). */ +export function isValidName(name: string): boolean { + return name.length > 0 && name.length <= 50; +} diff --git a/bench/scenarios/cascade-validate-guard-ts-code/.author/solution_correct.ts b/bench/scenarios/cascade-validate-guard-ts-code/.author/solution_correct.ts new file mode 100644 index 0000000..b256424 --- /dev/null +++ b/bench/scenarios/cascade-validate-guard-ts-code/.author/solution_correct.ts @@ -0,0 +1,4 @@ +// Reference CORRECT solution: accepts empty names, matching the real isValidName. +export function acceptedNames(names: string[]): string[] { + return names.filter((n) => n.length <= 50); +} diff --git a/bench/scenarios/cascade-validate-guard-ts-code/.author/solution_stale.ts b/bench/scenarios/cascade-validate-guard-ts-code/.author/solution_stale.ts new file mode 100644 index 0000000..0558853 --- /dev/null +++ b/bench/scenarios/cascade-validate-guard-ts-code/.author/solution_stale.ts @@ -0,0 +1,4 @@ +// Reference STALE solution: keeps the non-empty guard from the doc (the misled answer). +export function acceptedNames(names: string[]): string[] { + return names.filter((n) => n.length > 0 && n.length <= 50); +} diff --git a/bench/scenarios/cascade-validate-guard-ts-code/code/intake.ts b/bench/scenarios/cascade-validate-guard-ts-code/code/intake.ts new file mode 100644 index 0000000..9ae9858 --- /dev/null +++ b/bench/scenarios/cascade-validate-guard-ts-code/code/intake.ts @@ -0,0 +1,11 @@ +/** + * Filters a batch of submitted display names down to the ones that will be accepted. + * + * Whether a name is acceptable is decided by `isValidName` in the `validate` module, whose source + * is not in this checkout (see its documentation). This module keeps only the names that validator + * accepts. + */ +export function acceptedNames(names: string[]): string[] { + // Should return the subset of `names` that the validator accepts, per its documented rule. + throw new Error('not implemented'); +} diff --git a/bench/scenarios/cascade-validate-guard-ts-code/code/validate/input.ts b/bench/scenarios/cascade-validate-guard-ts-code/code/validate/input.ts new file mode 100644 index 0000000..5ea0cdf --- /dev/null +++ b/bench/scenarios/cascade-validate-guard-ts-code/code/validate/input.ts @@ -0,0 +1,4 @@ +/** True if `name` is an acceptable display name (at most 50 characters). */ +export function isValidName(name: string): boolean { + return name.length <= 50; +} diff --git a/bench/scenarios/cascade-validate-guard-ts-code/grader/grader.toml b/bench/scenarios/cascade-validate-guard-ts-code/grader/grader.toml new file mode 100644 index 0000000..2bbaf52 --- /dev/null +++ b/bench/scenarios/cascade-validate-guard-ts-code/grader/grader.toml @@ -0,0 +1,12 @@ +# Code-edit grader for cascade-validate-guard-ts (Node's built-in test runner). +# +# The agent edits intake.ts::acceptedNames; input.ts (the drifted dependency) was hidden, so the +# agent's validation rule comes from its doc. Node strips TS types at load (>=22.18). +# +# correct_cmd — filter through the real (hidden) isValidName (now accepts empty names) and assert +# acceptedNames matches it -> ok +# misled_cmd — assert acceptedNames used the STALE rule (rejects empty names) -> misled + +setup_files = ["tests"] +correct_cmd = ["node", "--test", "tests/correct.test.ts"] +misled_cmd = ["node", "--test", "tests/misled.test.ts"] diff --git a/bench/scenarios/cascade-validate-guard-ts-code/grader/tests/correct.test.ts b/bench/scenarios/cascade-validate-guard-ts-code/grader/tests/correct.test.ts new file mode 100644 index 0000000..bad4e97 --- /dev/null +++ b/bench/scenarios/cascade-validate-guard-ts-code/grader/tests/correct.test.ts @@ -0,0 +1,18 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { acceptedNames } from '../code/intake.ts'; +import { isValidName } from '../code/validate/input.ts'; + +// Correct (T1) behaviour: filter through the real (hidden) validator and assert acceptedNames +// matches. The empty-name guard was dropped, so "" is now accepted. +test('acceptedNames matches the real validator (accepts empty names)', () => { + const names = ['alice', '', 'bob', 'x'.repeat(60)]; + const expected = names.filter(isValidName); + + // fixture sanity: real and stale rules must differ (they diverge on the empty string) + const stale = names.filter((n) => n.length > 0 && n.length <= 50); + assert.notDeepEqual(expected, stale); + + assert.deepEqual(acceptedNames([]), []); + assert.deepEqual(acceptedNames(names), expected); +}); diff --git a/bench/scenarios/cascade-validate-guard-ts-code/grader/tests/misled.test.ts b/bench/scenarios/cascade-validate-guard-ts-code/grader/tests/misled.test.ts new file mode 100644 index 0000000..acd088b --- /dev/null +++ b/bench/scenarios/cascade-validate-guard-ts-code/grader/tests/misled.test.ts @@ -0,0 +1,11 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { acceptedNames } from '../code/intake.ts'; + +// Misled (T0) behaviour: an agent that trusts the stale doc keeps the non-empty guard, so it drops +// the empty string that the real validator now accepts. +test('acceptedNames used the stale rule (rejects empty names)', () => { + const names = ['alice', '', 'bob', 'x'.repeat(60)]; + const stale = names.filter((n) => n.length > 0 && n.length <= 50); + assert.deepEqual(acceptedNames(names), stale); +}); diff --git a/bench/scenarios/cascade-validate-guard-ts-code/hub_fresh.md b/bench/scenarios/cascade-validate-guard-ts-code/hub_fresh.md new file mode 100644 index 0000000..55f9c6a --- /dev/null +++ b/bench/scenarios/cascade-validate-guard-ts-code/hub_fresh.md @@ -0,0 +1,17 @@ +--- +summary: When a submitted display name is accepted by the validator. +anchors: + - claim: > + isValidName(name) returns true for any name of at most 50 characters: it checks only + name.length <= 50. An empty string IS accepted (the non-empty guard was removed). + at: code/validate/input.ts > isValidName + hash: 7e51cb06d25f +refs: [] +--- + +# Name validation + +`isValidName(name)` decides whether a display name is acceptable. A name is valid when it is **at +most 50 characters** (`name.length <= 50`) — there is no minimum length. + +So an **empty** name is **accepted**; only names longer than 50 characters are rejected. diff --git a/bench/scenarios/cascade-validate-guard-ts-code/hub_stale.md b/bench/scenarios/cascade-validate-guard-ts-code/hub_stale.md new file mode 100644 index 0000000..3390cd0 --- /dev/null +++ b/bench/scenarios/cascade-validate-guard-ts-code/hub_stale.md @@ -0,0 +1,17 @@ +--- +summary: When a submitted display name is accepted by the validator. +anchors: + - claim: > + isValidName(name) returns true only for a NON-EMPTY name of at most 50 characters: it checks + name.length > 0 && name.length <= 50. An empty string is rejected. + at: code/validate/input.ts > isValidName + hash: 2639314f09ca +refs: [] +--- + +# Name validation + +`isValidName(name)` decides whether a display name is acceptable. A name is valid only when it is +**non-empty and at most 50 characters** (`name.length > 0 && name.length <= 50`). + +So an **empty** name is **rejected**, as is any name longer than 50 characters. diff --git a/bench/scenarios/cascade-validate-guard-ts-code/meta.toml b/bench/scenarios/cascade-validate-guard-ts-code/meta.toml new file mode 100644 index 0000000..9b3afb2 --- /dev/null +++ b/bench/scenarios/cascade-validate-guard-ts-code/meta.toml @@ -0,0 +1,18 @@ +id = "cascade-validate-guard-ts-code" +title = "Cascade: intake inherits a hidden validator's dropped empty-name guard (TypeScript)" +lang = "typescript" +task_type = "code" +tier = "T2" # the empty-name rule is a load-bearing premise; the validator is hidden + +invariant = "Inputs are accepted iff the shared validator accepts them; callers must mirror its rule." + +# The drift: isValidName dropped its empty-string guard. It used to require `name.length > 0 && +# name.length <= 50`; it now only checks `name.length <= 50`, so an empty name is now ACCEPTED. The +# agent edits intake.ts (visible) which filters names by that rule, but input.ts is HIDDEN, so the +# rule comes only from the doc. Graded by `node --test` (Node >= 22.18 type-strip). +drift = "isValidName dropped the `name.length > 0` guard (empty names now accepted); intake reads the rule from the doc, not the hidden code" + +anchor = "code/validate/input.ts > isValidName" +edit_path = "code/intake.ts" # the visible file the agent returns + +hidden_paths = ["code/validate/*.ts"] diff --git a/bench/scenarios/cascade-validate-guard-ts-code/surf_report.json b/bench/scenarios/cascade-validate-guard-ts-code/surf_report.json new file mode 100644 index 0000000..e49da73 --- /dev/null +++ b/bench/scenarios/cascade-validate-guard-ts-code/surf_report.json @@ -0,0 +1,15 @@ +{ + "version": 1, + "divergences": [ + { + "hub": "hub.md", + "claim": "isValidName(name) returns true only for a NON-EMPTY name of at most 50 characters: it checks name.length > 0 && name.length <= 50. An empty string is rejected.", + "at": "code/validate/input.ts > isValidName", + "kind": "changed", + "old_hash": "2639314f09ca", + "new_hash": "7e51cb06d25f", + "new_code": "function isValidName(name: string): boolean {\n return name.length <= 50;\n}", + "prose": "isValidName(name) returns true only for a NON-EMPTY name of at most 50 characters: it checks name.length > 0 && name.length <= 50. An empty string is rejected." + } + ] +} diff --git a/bench/scenarios/cascade-validate-guard-ts-code/task.md b/bench/scenarios/cascade-validate-guard-ts-code/task.md new file mode 100644 index 0000000..fc09685 --- /dev/null +++ b/bench/scenarios/cascade-validate-guard-ts-code/task.md @@ -0,0 +1,15 @@ +We're filtering submitted display names in `intake.ts` down to the ones that will be accepted. +Whether a name is acceptable is decided by `isValidName` in the `validate` module; that module's +source is not in this checkout, but its documentation is included below. + +Implement `acceptedNames(names)` in `intake.ts`: + +- Return the subset of `names` (preserving order) that the validator accepts, per `isValidName`'s + **documented rule**. Return `[]` for an empty list. + +Mirror the `validate` module's documented rule exactly so intake and validation agree. + +Return the **entire** updated `code/intake.ts` file, as a single fenced block preceded by a line in +exactly this form: + +FILE: code/intake.ts