feat(bench): 3 TypeScript cascade scenarios (#104) [re-targeted to main]

bench/scenarios/cascade-default-timeout-ts-code/.author/code_t0/net/timeout.ts

@@ -0,0 +1,9 @@
+/**
+ * Total request deadline (ms) across all attempts.
+ *
+ * A request gets three attempts, each bounded by `perAttemptMs`; the total deadline is three times
+ * the per-attempt timeout. `perAttemptMs` defaults to the standard 30000 ms timeout.
+ */
+export function totalDeadlineMs(perAttemptMs: number = 30000): number {
+  return perAttemptMs * 3;
+}

bench/scenarios/cascade-default-timeout-ts-code/.author/solution_correct.ts

@@ -0,0 +1,4 @@
+// Reference CORRECT solution: uses the real 5000 ms default per-attempt timeout.
+export function requestBudgetMs(): number {
+  return 5000 * 3;
+}

bench/scenarios/cascade-default-timeout-ts-code/.author/solution_stale.ts

@@ -0,0 +1,4 @@
+// Reference STALE solution: trusts the doc's 30000 ms default (the misled answer).
+export function requestBudgetMs(): number {
+  return 30000 * 3;
+}

bench/scenarios/cascade-default-timeout-ts-code/code/caller.ts

@@ -0,0 +1,12 @@
+/**
+ * Reports the default request budget for the UI's "this may take up to…" hint.
+ *
+ * The total deadline is computed by `totalDeadlineMs` in the `net` module, whose source is not in
+ * this checkout (see its documentation). This module reports the default budget — the deadline when
+ * no per-attempt override is given.
+ */
+export function requestBudgetMs(): number {
+  // Should return the total request deadline the net layer uses by default (no per-attempt
+  // override) — i.e. totalDeadlineMs() called with no argument.
+  throw new Error('not implemented');
+}

bench/scenarios/cascade-default-timeout-ts-code/code/net/timeout.ts

@@ -0,0 +1,9 @@
+/**
+ * Total request deadline (ms) across all attempts.
+ *
+ * A request gets three attempts, each bounded by `perAttemptMs`; the total deadline is three times
+ * the per-attempt timeout. `perAttemptMs` defaults to the standard 5000 ms timeout.
+ */
+export function totalDeadlineMs(perAttemptMs: number = 5000): number {
+  return perAttemptMs * 3;
+}

bench/scenarios/cascade-default-timeout-ts-code/grader/grader.toml

@@ -0,0 +1,12 @@
+# Code-edit grader for cascade-default-timeout-ts (Node's built-in test runner).
+#
+# The agent edits caller.ts::requestBudgetMs; timeout.ts (the drifted dependency) was hidden, so the
+# agent's default per-attempt timeout comes from its doc. Node strips TS types at load (>=22.18).
+#
+#   correct_cmd — call the real (hidden) totalDeadlineMs() and assert requestBudgetMs matches it
+#                 (15000 ms under the real 5000 ms default)                                     -> ok
+#   misled_cmd  — assert requestBudgetMs used the STALE 30000 ms default (90000 ms)              -> misled
+
+setup_files = ["tests"]
+correct_cmd = ["node", "--test", "tests/correct.test.ts"]
+misled_cmd = ["node", "--test", "tests/misled.test.ts"]

bench/scenarios/cascade-default-timeout-ts-code/grader/tests/correct.test.ts

@@ -0,0 +1,15 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { requestBudgetMs } from '../code/caller.ts';
+import { totalDeadlineMs } from '../code/net/timeout.ts';
+
+// Correct (T1) behaviour: recover the default budget from the real (hidden) net helper and assert
+// requestBudgetMs matches it. The default per-attempt timeout is now 5000 ms -> total 15000 ms.
+test('requestBudgetMs uses the real default per-attempt timeout', () => {
+  const expected = totalDeadlineMs(); // real default 5000 -> 15000
+
+  // fixture sanity: the default drift must move the budget
+  assert.notEqual(expected, 30000 * 3);
+
+  assert.equal(requestBudgetMs(), expected);
+});

bench/scenarios/cascade-default-timeout-ts-code/grader/tests/misled.test.ts

@@ -0,0 +1,9 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { requestBudgetMs } from '../code/caller.ts';
+
+// Misled (T0) behaviour: an agent that trusts the stale doc assumes a 30000 ms default per-attempt
+// timeout, so it reports 90000 ms instead of the real 15000 ms.
+test('requestBudgetMs used the stale default timeout', () => {
+  assert.equal(requestBudgetMs(), 30000 * 3);
+});

bench/scenarios/cascade-default-timeout-ts-code/hub_fresh.md

@@ -0,0 +1,19 @@
+---
+summary: The default per-attempt timeout and the total request deadline derived from it.
+anchors:
+  - claim: >
+      totalDeadlineMs(perAttemptMs) returns perAttemptMs * 3 (a request gets three attempts). The
+      default per-attempt timeout is 5000 ms, so totalDeadlineMs() called with no argument returns
+      15000 ms — the default total request deadline.
+    at: code/net/timeout.ts > totalDeadlineMs
+    hash: 5277f05efc4c
+refs: []
+---
+
+# Request timeouts
+
+`totalDeadlineMs(perAttemptMs)` returns the total request deadline in **milliseconds**: a request
+gets **three** attempts, so the total is `perAttemptMs * 3`.
+
+**Default per-attempt timeout:** `5000` ms (5 s). With no override, `totalDeadlineMs()` therefore
+returns **15000** ms — the default total request deadline.

bench/scenarios/cascade-default-timeout-ts-code/hub_stale.md

@@ -0,0 +1,19 @@
+---
+summary: The default per-attempt timeout and the total request deadline derived from it.
+anchors:
+  - claim: >
+      totalDeadlineMs(perAttemptMs) returns perAttemptMs * 3 (a request gets three attempts). The
+      default per-attempt timeout is 30000 ms, so totalDeadlineMs() called with no argument returns
+      90000 ms — the default total request deadline.
+    at: code/net/timeout.ts > totalDeadlineMs
+    hash: 423c0258c20a
+refs: []
+---
+
+# Request timeouts
+
+`totalDeadlineMs(perAttemptMs)` returns the total request deadline in **milliseconds**: a request
+gets **three** attempts, so the total is `perAttemptMs * 3`.
+
+**Default per-attempt timeout:** `30000` ms (30 s). With no override, `totalDeadlineMs()` therefore
+returns **90000** ms — the default total request deadline.

bench/scenarios/cascade-default-timeout-ts-code/meta.toml

@@ -0,0 +1,18 @@
+id = "cascade-default-timeout-ts-code"
+title = "Cascade: caller inherits a hidden net helper's drifted default timeout (TypeScript)"
+lang = "typescript"
+task_type = "code"
+tier = "T1"  # the default timeout lives in a dependency the agent cannot see — only its doc
+
+invariant = "Requests use a default per-attempt timeout; callers that don't override it must assume the same default."
+
+# The drift: totalDeadlineMs's default parameter `perAttemptMs` changed from 30000 to 5000, so the
+# total deadline it returns when called with no override fell from 90000ms to 15000ms. The agent
+# edits caller.ts (visible) which reports that default budget, but timeout.ts is HIDDEN, so the
+# default comes only from the doc. Graded by `node --test` (Node >= 22.18 type-strip).
+drift = "totalDeadlineMs default perAttemptMs 30000 -> 5000; caller reads the default from the doc, not the hidden code"
+
+anchor = "code/net/timeout.ts > totalDeadlineMs"
+edit_path = "code/caller.ts"  # the visible file the agent returns
+
+hidden_paths = ["code/net/*.ts"]

bench/scenarios/cascade-default-timeout-ts-code/surf_report.json

@@ -0,0 +1,15 @@
+{
+  "version": 1,
+  "divergences": [
+    {
+      "hub": "hub.md",
+      "claim": "totalDeadlineMs(perAttemptMs) returns perAttemptMs * 3 (a request gets three attempts). The default per-attempt timeout is 30000 ms, so totalDeadlineMs() called with no argument returns 90000 ms \u2014 the default total request deadline.",
+      "at": "code/net/timeout.ts > totalDeadlineMs",
+      "kind": "changed",
+      "old_hash": "423c0258c20a",
+      "new_hash": "5277f05efc4c",
+      "new_code": "function totalDeadlineMs(perAttemptMs: number = 5000): number {\n  return perAttemptMs * 3;\n}",
+      "prose": "totalDeadlineMs(perAttemptMs) returns perAttemptMs * 3 (a request gets three attempts). The default per-attempt timeout is 30000 ms, so totalDeadlineMs() called with no argument returns 90000 ms \u2014 the default total request deadline."
+    }
+  ]
+}

bench/scenarios/cascade-default-timeout-ts-code/task.md

@@ -0,0 +1,16 @@
+We're showing a "this may take up to…" hint in `caller.ts`. The hint is the default total request
+budget, computed by `totalDeadlineMs` in the `net` module. That module's source is not in this
+checkout, but its documentation is included below.
+
+Implement `requestBudgetMs()` in `caller.ts`:
+
+- Return the total request deadline in **milliseconds** that the net layer uses **by default** —
+  i.e. `totalDeadlineMs()` called with no per-attempt override.
+
+Use the net module's documented default per-attempt timeout so the budget matches what requests
+actually use.
+
+Return the **entire** updated `code/caller.ts` file, as a single fenced block preceded by a line in
+exactly this form:
+
+FILE: code/caller.ts

bench/scenarios/cascade-serialize-key-order-ts-code/.author/code_t0/sign/canonical.ts

@@ -0,0 +1,8 @@
+/**
+ * Serialize a flat string map to the canonical string that payloads are signed over.
+ * Fields are emitted in INSERTION order, as they appear on the object.
+ */
+export function canonicalString(fields: Record<string, string>): string {
+  const keys = Object.keys(fields);
+  return keys.map((k) => `${k}=${fields[k]}`).join('&');
+}

bench/scenarios/cascade-serialize-key-order-ts-code/.author/solution_correct.ts

@@ -0,0 +1,7 @@
+// Reference CORRECT solution: sorted key order, matching the real canonicalString.
+export function signingString(fields: Record<string, string>): string {
+  return Object.keys(fields)
+    .sort()
+    .map((k) => `${k}=${fields[k]}`)
+    .join('&');
+}

bench/scenarios/cascade-serialize-key-order-ts-code/.author/solution_stale.ts

@@ -0,0 +1,6 @@
+// Reference STALE solution: trusts the doc's insertion order (the misled answer).
+export function signingString(fields: Record<string, string>): string {
+  return Object.keys(fields)
+    .map((k) => `${k}=${fields[k]}`)
+    .join('&');
+}

bench/scenarios/cascade-serialize-key-order-ts-code/code/sign/canonical.ts

@@ -0,0 +1,8 @@
+/**
+ * Serialize a flat string map to the canonical string that payloads are signed over.
+ * Fields are emitted in SORTED key order so the signer and verifier always agree.
+ */
+export function canonicalString(fields: Record<string, string>): string {
+  const keys = Object.keys(fields).sort();
+  return keys.map((k) => `${k}=${fields[k]}`).join('&');
+}

bench/scenarios/cascade-serialize-key-order-ts-code/code/signer.ts

@@ -0,0 +1,12 @@
+/**
+ * Builds the string a request payload is signed over.
+ *
+ * The canonical serialization is defined by `canonicalString` in the `sign` module, whose source is
+ * not in this checkout (see its documentation). This module must reproduce that exact string so the
+ * signature it produces verifies on the other side.
+ */
+export function signingString(fields: Record<string, string>): string {
+  // Should reproduce the canonical serialization of `fields` (same key ordering as the signer's
+  // canonicalString), as `key=value` pairs joined by '&'.
+  throw new Error('not implemented');
+}

bench/scenarios/cascade-serialize-key-order-ts-code/grader/grader.toml

@@ -0,0 +1,12 @@
+# Code-edit grader for cascade-serialize-key-order-ts (Node's built-in test runner).
+#
+# The agent edits signer.ts::signingString; canonical.ts (the drifted dependency) was hidden, so the
+# agent's key ordering comes from its doc. Node strips TS types at load time (>=22.18), no tsc step.
+#
+#   correct_cmd — call the real (hidden) canonicalString and assert signingString reproduces it
+#                 (sorted key order)                                                            -> ok
+#   misled_cmd  — assert signingString used the STALE insertion order                            -> misled
+
+setup_files = ["tests"]
+correct_cmd = ["node", "--test", "tests/correct.test.ts"]
+misled_cmd = ["node", "--test", "tests/misled.test.ts"]

bench/scenarios/cascade-serialize-key-order-ts-code/grader/tests/correct.test.ts

@@ -0,0 +1,18 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { signingString } from '../code/signer.ts';
+import { canonicalString } from '../code/sign/canonical.ts';
+
+// Correct (T1) behaviour: recover the canonical ordering from the real (hidden) canonicalizer and
+// assert signingString reproduces it. canonicalString now sorts keys, so {b,a,c} -> "a=1&b=2&c=3".
+test('signingString reproduces the real canonical (sorted) order', () => {
+  const fields = { b: '2', a: '1', c: '3' };
+  const expected = canonicalString(fields);
+
+  // fixture sanity: the ordering must actually differ from insertion order
+  const insertion = Object.keys(fields).map((k) => `${k}=${fields[k]}`).join('&');
+  assert.notEqual(expected, insertion);
+
+  assert.equal(signingString({}), '');
+  assert.equal(signingString(fields), expected);
+});

bench/scenarios/cascade-serialize-key-order-ts-code/grader/tests/misled.test.ts

@@ -0,0 +1,11 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { signingString } from '../code/signer.ts';
+
+// Misled (T0) behaviour: an agent that trusts the stale doc emits fields in insertion order,
+// so {b,a,c} -> "b=2&a=1&c=3" instead of the real sorted "a=1&b=2&c=3".
+test('signingString used the stale insertion order', () => {
+  const fields = { b: '2', a: '1', c: '3' };
+  const insertion = Object.keys(fields).map((k) => `${k}=${fields[k]}`).join('&');
+  assert.equal(signingString(fields), insertion);
+});

bench/scenarios/cascade-serialize-key-order-ts-code/hub_fresh.md

@@ -0,0 +1,19 @@
+---
+summary: The canonical serialization that request payloads are signed over.
+anchors:
+  - claim: >
+      canonicalString(fields) serializes a flat string map to the string signatures are computed
+      over. It emits the fields in SORTED key order (Object.keys().sort()) as key=value pairs joined
+      by '&'. So {b:'2', a:'1', c:'3'} serializes to "a=1&b=2&c=3".
+    at: code/sign/canonical.ts > canonicalString
+    hash: c832bea1d85b
+refs: []
+---
+
+# Canonical signing string
+
+`canonicalString(fields)` builds the canonical string a payload's signature is computed over. Each
+field is rendered as `key=value` and joined with `&`.
+
+**Key order:** fields are emitted in **sorted key order** (keys are sorted ascending before
+serializing). For example `{b: '2', a: '1', c: '3'}` serializes to `a=1&b=2&c=3`.

bench/scenarios/cascade-serialize-key-order-ts-code/hub_stale.md

@@ -0,0 +1,19 @@
+---
+summary: The canonical serialization that request payloads are signed over.
+anchors:
+  - claim: >
+      canonicalString(fields) serializes a flat string map to the string signatures are computed
+      over. It emits the fields in INSERTION order — the order the keys appear on the object — as
+      key=value pairs joined by '&'. So {b:'2', a:'1', c:'3'} serializes to "b=2&a=1&c=3".
+    at: code/sign/canonical.ts > canonicalString
+    hash: b1f55401389c
+refs: []
+---
+
+# Canonical signing string
+
+`canonicalString(fields)` builds the canonical string a payload's signature is computed over. Each
+field is rendered as `key=value` and joined with `&`.
+
+**Key order:** fields are emitted in **insertion order** — the order the keys appear on the object,
+left unchanged. For example `{b: '2', a: '1', c: '3'}` serializes to `b=2&a=1&c=3`.

bench/scenarios/cascade-serialize-key-order-ts-code/meta.toml

@@ -0,0 +1,18 @@
+id = "cascade-serialize-key-order-ts-code"
+title = "Cascade: signer inherits a hidden canonicalizer's drifted key ordering (TypeScript)"
+lang = "typescript"
+task_type = "code"
+tier = "T2"  # the canonical key order is a load-bearing premise; the canonicalizer is hidden
+
+invariant = "A payload is signed over a canonical serialization; signer and verifier must order keys the same way."
+
+# The drift: canonicalString now sorts keys (Object.keys(fields).sort()) where it used to keep
+# insertion order. The serialized string — and therefore any signature over it — changes. The agent
+# edits signer.ts (visible) which must reproduce the canonical string, but canonical.ts is HIDDEN,
+# so the ordering rule comes only from its doc. Graded by `node --test` (Node >= 22.18 type-strip).
+drift = "canonicalString key order insertion -> sorted (Object.keys().sort()); signer reads order from the doc, not the hidden code"
+
+anchor = "code/sign/canonical.ts > canonicalString"
+edit_path = "code/signer.ts"  # the visible file the agent returns
+
+hidden_paths = ["code/sign/*.ts"]

bench/scenarios/cascade-serialize-key-order-ts-code/surf_report.json

@@ -0,0 +1,15 @@
+{
+  "version": 1,
+  "divergences": [
+    {
+      "hub": "hub.md",
+      "claim": "canonicalString(fields) serializes a flat string map to the string signatures are computed over. It emits the fields in INSERTION order \u2014 the order the keys appear on the object \u2014 as key=value pairs joined by '&'. So {b:'2', a:'1', c:'3'} serializes to \"b=2&a=1&c=3\".",
+      "at": "code/sign/canonical.ts > canonicalString",
+      "kind": "changed",
+      "old_hash": "b1f55401389c",
+      "new_hash": "c832bea1d85b",
+      "new_code": "function canonicalString(fields: Record<string, string>): string {\n  const keys = Object.keys(fields).sort();\n  return keys.map((k) => `${k}=${fields[k]}`).join('&');\n}",
+      "prose": "canonicalString(fields) serializes a flat string map to the string signatures are computed over. It emits the fields in INSERTION order \u2014 the order the keys appear on the object \u2014 as key=value pairs joined by '&'. So {b:'2', a:'1', c:'3'} serializes to \"b=2&a=1&c=3\"."
+    }
+  ]
+}

bench/scenarios/cascade-serialize-key-order-ts-code/task.md

@@ -0,0 +1,16 @@
+We're signing request payloads in `signer.ts`. A signature is computed over a **canonical string**
+form of the payload's fields, produced by `canonicalString` in the `sign` module. That module's
+source is not in this checkout, but its documentation is included below.
+
+Implement `signingString(fields)` in `signer.ts`:
+
+- `fields` is a flat object of string keys to string values.
+- Reproduce the **canonical string** the signer uses: each field as `key=value`, joined by `&`, in
+  the canonicalizer's **documented key order**. Return `""` for an empty object.
+
+Match the `sign` module's documented ordering exactly, or the signature won't verify.
+
+Return the **entire** updated `code/signer.ts` file, as a single fenced block preceded by a line in
+exactly this form:
+
+FILE: code/signer.ts

bench/scenarios/cascade-validate-guard-ts-code/.author/code_t0/validate/input.ts

@@ -0,0 +1,4 @@
+/** True if `name` is an acceptable display name (non-empty, at most 50 characters). */
+export function isValidName(name: string): boolean {
+  return name.length > 0 && name.length <= 50;
+}

bench/scenarios/cascade-validate-guard-ts-code/.author/solution_correct.ts

@@ -0,0 +1,4 @@
+// Reference CORRECT solution: accepts empty names, matching the real isValidName.
+export function acceptedNames(names: string[]): string[] {
+  return names.filter((n) => n.length <= 50);
+}

bench/scenarios/cascade-validate-guard-ts-code/.author/solution_stale.ts

@@ -0,0 +1,4 @@
+// Reference STALE solution: keeps the non-empty guard from the doc (the misled answer).
+export function acceptedNames(names: string[]): string[] {
+  return names.filter((n) => n.length > 0 && n.length <= 50);
+}

bench/scenarios/cascade-validate-guard-ts-code/code/intake.ts

@@ -0,0 +1,11 @@
+/**
+ * Filters a batch of submitted display names down to the ones that will be accepted.
+ *
+ * Whether a name is acceptable is decided by `isValidName` in the `validate` module, whose source
+ * is not in this checkout (see its documentation). This module keeps only the names that validator
+ * accepts.
+ */
+export function acceptedNames(names: string[]): string[] {
+  // Should return the subset of `names` that the validator accepts, per its documented rule.
+  throw new Error('not implemented');
+}

bench/scenarios/cascade-validate-guard-ts-code/code/validate/input.ts

@@ -0,0 +1,4 @@
+/** True if `name` is an acceptable display name (at most 50 characters). */
+export function isValidName(name: string): boolean {
+  return name.length <= 50;
+}