diff --git a/bench/scenarios/CHECKLIST.md b/bench/scenarios/CHECKLIST.md new file mode 100644 index 0000000..c7a5226 --- /dev/null +++ b/bench/scenarios/CHECKLIST.md @@ -0,0 +1,59 @@ +# Authoring a cascade scenario + +A *cascade* scenario: the agent edits a **visible** file whose correctness depends on a **hidden** +dependency (in `code/`, listed in `meta.toml` `hidden_paths`, filtered out of the prompt). The +dependency has drifted from what the **stale** doc says, so an agent that trusts the stale doc is +wrong. The graders probe the *real* hidden dependency for ground truth. Clone +`cascade-quota-batcher-code/` as the canonical template. + +## 0. Pick a load-bearing drift +- [ ] Choose a drift archetype **not already in the family** (avoid: boundary `<`→`<=`, plain + constant, allow→block, page-size constant — see existing `cascade-*`). +- [ ] Name a concrete input where **T0 and T1 produce different observable outputs**. If you can't, + the drift isn't load-bearing — stop and rethink. (The whole effect dies if T0 ≈ T1.) + +## 1. Directory skeleton (`scenarios//`) +- [ ] `meta.toml`: `id`, `title`, `lang`, `task_type`, `tier`, `anchor = "code/ > Symbol > + method"`, `invariant`, `drift`, `hidden_paths` (glob over the dependency), `edit_path` (the + visible file the agent returns — code scenarios only). +- [ ] `code/` — the **T1 (drifted)** source. Hidden dependency present; the visible file is a + **stub** (`raise NotImplementedError` / TODO). No leaked value anywhere in `code/`. +- [ ] `.author/code_t0/` — **only the files that changed**, in their pre-drift (T0) form. +- [ ] `hub_stale.md` — TOML front matter (`summary`, `anchors:` with `claim`/`at`/`hash`, `refs`) + + prose, describing **T0**. Use a placeholder `hash: 000000000000` (author.py seals it). +- [ ] `hub_fresh.md` — same `anchor`, describing **T1**. Placeholder hash. +- [ ] `task.md` — neutral (see §3). For code, ends with the exact `FILE: ` contract. +- [ ] Grader — + - code: `grader/grader.toml` (`setup_files=["tests"]`, `correct_cmd`, `misled_cmd`) + + `grader/tests/check_correct.*` + `check_misled.*`. + - qa: `grader/rubric.toml` (`type="verdict"`, `[fields.]` regex, `[correct]`, `[misled]`). +- [ ] `.author/solution_correct.` and `.author/solution_stale.` — reference solutions for + the polarization self-test (code: the visible file's body; qa: a `.txt` ending in the VERDICT). + +## 2. Grader rules +- [ ] `check_correct` **derives ground truth by importing/probing the real hidden dependency** — never + hardcode the T1 value. `check_misled` hardcodes the **stale** doc value and asserts the agent + used it. (See `cascade-quota-batcher-code`'s `true_capacity()` probe.) +- [ ] Both exit non-zero on a failed assertion (`assert` / `raise SystemExit`). +- [ ] Choose a probe input where stale vs correct give **different** results (e.g. 25 → `[10,10,5]` + stale vs `[11,11,3]` correct). + +## 3. Neutrality (the #113 lesson — leaks killed the effect once) +- [ ] `task.md` states the goal + output contract only. **No worked example that reveals the stale or + the fresh value** (an illustrative example with an *unrelated* placeholder number is fine). +- [ ] No "the doc may be wrong" hint, no precedence between doc and code. +- [ ] The stale value appears **only** in `hub_stale.md` — never in `task.md` or the visible code. + +## 4. Seal + validate (offline, no spend) +- [ ] `python tools/author.py scenarios/` — seals hub hashes, emits `surf_report.json`, asserts a + `"changed"` divergence (fails loudly if the drift isn't detectable). +- [ ] `python tools/validate_scenario.py scenarios/` — proves the graders discriminate + (`solution_correct` → ok & not misled; `solution_stale` → not ok & misled). + +## 5. Pipeline smoke +- [ ] `python -m surface_bench.run --models mock --scenarios ` — runs end to end, no API cost. +- [ ] `python -m surface_bench.oracle results/` — tripwires clear. + +## Language notes +- python / typescript runtimes are always available (python3 via `sys.executable`; `node --test`, + TS type-stripped on Node ≥ 22.18). Scenarios are Python + TS only. diff --git a/bench/scenarios/cascade-access-policy-code/meta.toml b/bench/scenarios/cascade-access-policy-code/meta.toml index 481d922..10e844b 100644 --- a/bench/scenarios/cascade-access-policy-code/meta.toml +++ b/bench/scenarios/cascade-access-policy-code/meta.toml @@ -13,5 +13,6 @@ invariant = "Private-doc team access is governed by one policy; precomputed view drift = "team check allow-list -> block-list; the visible precompute encodes the direction from the doc" anchor = "code/access.py > AccessPolicy > can_view" +edit_path = "code/preload.py" # the visible file the agent returns hidden_paths = ["code/access.py", "code/models.py"] diff --git a/bench/scenarios/cascade-backoff-offbyone-code/.author/code_t0/retry/backoff.py b/bench/scenarios/cascade-backoff-offbyone-code/.author/code_t0/retry/backoff.py new file mode 100644 index 0000000..7bd7b64 --- /dev/null +++ b/bench/scenarios/cascade-backoff-offbyone-code/.author/code_t0/retry/backoff.py @@ -0,0 +1,9 @@ +BASE_MS = 100 + + +def delay_for(attempt: int) -> int: + """Backoff delay in milliseconds for a retry attempt. + + Attempts are 1-based: attempt 1 waits the base delay, and each subsequent attempt doubles it. + """ + return BASE_MS << (attempt - 1) diff --git a/bench/scenarios/cascade-backoff-offbyone-code/.author/solution_correct.py b/bench/scenarios/cascade-backoff-offbyone-code/.author/solution_correct.py new file mode 100644 index 0000000..61fea64 --- /dev/null +++ b/bench/scenarios/cascade-backoff-offbyone-code/.author/solution_correct.py @@ -0,0 +1,7 @@ +"""Reference CORRECT solution: 0-based attempts, matching the real delay_for.""" + +BASE_MS = 100 + + +def delays(attempts: list[int]) -> list[int]: + return [BASE_MS << a for a in attempts] diff --git a/bench/scenarios/cascade-backoff-offbyone-code/.author/solution_stale.py b/bench/scenarios/cascade-backoff-offbyone-code/.author/solution_stale.py new file mode 100644 index 0000000..b4784da --- /dev/null +++ b/bench/scenarios/cascade-backoff-offbyone-code/.author/solution_stale.py @@ -0,0 +1,7 @@ +"""Reference STALE solution: trusts the doc's 1-based attempts (the misled answer).""" + +BASE_MS = 100 + + +def delays(attempts: list[int]) -> list[int]: + return [BASE_MS << (a - 1) for a in attempts] diff --git a/bench/scenarios/cascade-backoff-offbyone-code/code/poller.py b/bench/scenarios/cascade-backoff-offbyone-code/code/poller.py new file mode 100644 index 0000000..056a10d --- /dev/null +++ b/bench/scenarios/cascade-backoff-offbyone-code/code/poller.py @@ -0,0 +1,18 @@ +"""Builds a retry backoff schedule for a poller. + +The backoff policy lives in the `retry` package (see its documentation); that helper's source is +not in this checkout. This module reproduces the per-attempt delays so the poller can pre-compute +its schedule. +""" + +BASE_MS = 100 + + +def delays(attempts: list[int]) -> list[int]: + """For each retry attempt number in ``attempts``, return its backoff delay in milliseconds. + + Backoff is exponential: it starts from a base of 100 ms and doubles each attempt. Apply the + documented attempt-indexing convention so the delays match the retry package's ``delay_for``. + Returns ``[]`` for an empty list. + """ + raise NotImplementedError diff --git a/bench/scenarios/cascade-backoff-offbyone-code/code/retry/__init__.py b/bench/scenarios/cascade-backoff-offbyone-code/code/retry/__init__.py new file mode 100644 index 0000000..bf87d19 --- /dev/null +++ b/bench/scenarios/cascade-backoff-offbyone-code/code/retry/__init__.py @@ -0,0 +1,3 @@ +from .backoff import BASE_MS, delay_for + +__all__ = ["BASE_MS", "delay_for"] diff --git a/bench/scenarios/cascade-backoff-offbyone-code/code/retry/backoff.py b/bench/scenarios/cascade-backoff-offbyone-code/code/retry/backoff.py new file mode 100644 index 0000000..7620a63 --- /dev/null +++ b/bench/scenarios/cascade-backoff-offbyone-code/code/retry/backoff.py @@ -0,0 +1,9 @@ +BASE_MS = 100 + + +def delay_for(attempt: int) -> int: + """Backoff delay in milliseconds for a retry attempt. + + Attempts are 0-based: attempt 0 waits the base delay, and each subsequent attempt doubles it. + """ + return BASE_MS << attempt diff --git a/bench/scenarios/cascade-backoff-offbyone-code/grader/grader.toml b/bench/scenarios/cascade-backoff-offbyone-code/grader/grader.toml new file mode 100644 index 0000000..4563e2e --- /dev/null +++ b/bench/scenarios/cascade-backoff-offbyone-code/grader/grader.toml @@ -0,0 +1,11 @@ +# Code-edit grader for cascade-backoff-offbyone. +# +# The agent edits poller.py::delays; delay_for (the drifted dependency) was hidden, so the agent's +# attempt-indexing convention comes entirely from the doc. Both checks run against the applied patch: +# +# correct_cmd — probe the REAL delay_for (0-based) for each attempt and assert delays matches -> ok +# misled_cmd — assert delays used the STALE 1-based formula (100 * 2**(n-1)) -> misled + +setup_files = ["tests"] +correct_cmd = ["python3", "tests/check_correct.py"] +misled_cmd = ["python3", "tests/check_misled.py"] diff --git a/bench/scenarios/cascade-backoff-offbyone-code/grader/tests/check_correct.py b/bench/scenarios/cascade-backoff-offbyone-code/grader/tests/check_correct.py new file mode 100644 index 0000000..9d3f10e --- /dev/null +++ b/bench/scenarios/cascade-backoff-offbyone-code/grader/tests/check_correct.py @@ -0,0 +1,27 @@ +"""Correct (T1) behaviour: delays reproduces the REAL delay_for for each attempt number. + +delay_for was hidden from the agent, so the attempt-indexing convention is knowable only from the +(fresh) doc. We recover the ground truth by calling the real delay_for, which is now 0-based +(100 * 2**attempt). For attempts 1,2,3 that is 200,400,800 ms (not the 100,200,400 of the stale +1-based formula). +""" + +import sys + +sys.path.insert(0, "code") + +from poller import delays +from retry.backoff import delay_for + +attempts = [1, 2, 3, 4] +expected = [delay_for(a) for a in attempts] + +# fixture sanity: the off-by-one must actually move the delays +stale = [100 << (a - 1) for a in attempts] +assert expected != stale, f"fixture sanity: indexing change does not move the delays ({expected})" + +assert delays([]) == [], "delays([]) must be []" +got = delays(attempts) +assert got == expected, f"delays({attempts}) = {got}, want {expected} (real 0-based delay_for)" + +print(f"ok: delays reproduces the real 0-based backoff ({expected})") diff --git a/bench/scenarios/cascade-backoff-offbyone-code/grader/tests/check_misled.py b/bench/scenarios/cascade-backoff-offbyone-code/grader/tests/check_misled.py new file mode 100644 index 0000000..2d1264d --- /dev/null +++ b/bench/scenarios/cascade-backoff-offbyone-code/grader/tests/check_misled.py @@ -0,0 +1,19 @@ +"""Misled (T0) behaviour: the stale doc says attempts are 1-based (100 * 2**(attempt - 1)). + +An agent that trusts that doc returns 100,200,400,800 for attempts 1,2,3,4 instead of the real +200,400,800,1600 under the 0-based delay_for. +""" + +import sys + +sys.path.insert(0, "code") + +from poller import delays + +attempts = [1, 2, 3, 4] +stale = [100 << (a - 1) for a in attempts] + +got = delays(attempts) +assert got == stale, f"delays({attempts}) = {got} (stale 1-based delays {stale})" + +print(f"misled: delays used the stale 1-based backoff ({stale})") diff --git a/bench/scenarios/cascade-backoff-offbyone-code/hub_fresh.md b/bench/scenarios/cascade-backoff-offbyone-code/hub_fresh.md new file mode 100644 index 0000000..51aa860 --- /dev/null +++ b/bench/scenarios/cascade-backoff-offbyone-code/hub_fresh.md @@ -0,0 +1,19 @@ +--- +summary: How the retry backoff computes a per-attempt delay. +anchors: + - claim: > + delay_for(attempt) returns the backoff delay in milliseconds for a retry attempt. Attempts + are 0-based: the delay is BASE_MS * 2**attempt with BASE_MS = 100. So attempt 0 waits 100 ms, + attempt 1 waits 200 ms, attempt 2 waits 400 ms, and attempt n waits 100 * 2**n ms. + at: code/retry/backoff.py > delay_for + hash: 0038327b28fb +refs: [] +--- + +# Retry backoff + +`delay_for(attempt)` returns the backoff delay, in **milliseconds**, for a retry attempt. The base +delay is `BASE_MS = 100` and the delay doubles each attempt. + +**Attempt indexing:** attempts are **0-based** — `delay_for(0)` is the base 100 ms, and in general +`delay_for(n) = 100 * 2**n` ms. So attempts 1, 2, 3 wait 200, 400, 800 ms. diff --git a/bench/scenarios/cascade-backoff-offbyone-code/hub_stale.md b/bench/scenarios/cascade-backoff-offbyone-code/hub_stale.md new file mode 100644 index 0000000..bd7724d --- /dev/null +++ b/bench/scenarios/cascade-backoff-offbyone-code/hub_stale.md @@ -0,0 +1,19 @@ +--- +summary: How the retry backoff computes a per-attempt delay. +anchors: + - claim: > + delay_for(attempt) returns the backoff delay in milliseconds for a retry attempt. Attempts + are 1-based: the delay is BASE_MS * 2**(attempt - 1) with BASE_MS = 100. So attempt 1 waits + 100 ms, attempt 2 waits 200 ms, attempt 3 waits 400 ms, and attempt n waits 100 * 2**(n-1) ms. + at: code/retry/backoff.py > delay_for + hash: 93b18e377590 +refs: [] +--- + +# Retry backoff + +`delay_for(attempt)` returns the backoff delay, in **milliseconds**, for a retry attempt. The base +delay is `BASE_MS = 100` and the delay doubles each attempt. + +**Attempt indexing:** attempts are **1-based** — `delay_for(1)` is the base 100 ms, and in general +`delay_for(n) = 100 * 2**(n - 1)` ms. So attempts 1, 2, 3 wait 100, 200, 400 ms. diff --git a/bench/scenarios/cascade-backoff-offbyone-code/meta.toml b/bench/scenarios/cascade-backoff-offbyone-code/meta.toml new file mode 100644 index 0000000..78bf58b --- /dev/null +++ b/bench/scenarios/cascade-backoff-offbyone-code/meta.toml @@ -0,0 +1,18 @@ +id = "cascade-backoff-offbyone-code" +title = "Cascade: poller inherits a hidden backoff helper's drifted attempt indexing (off-by-one)" +lang = "python" +task_type = "code" +tier = "T2" # the attempt-indexing convention is a load-bearing premise; the helper is hidden + +invariant = "Exponential backoff doubles each attempt from a base delay; callers must index attempts the same way." + +# The drift: retry/backoff.py::delay_for changed its attempt indexing from 1-based to 0-based. The +# exponent went from `attempt - 1` to `attempt`, so for the SAME attempt number the real delay is +# now double the documented one. The agent edits poller.py (visible) whose schedule reproduces the +# delays, but backoff.py is HIDDEN, so the indexing convention comes only from the doc. +drift = "delay_for exponent `attempt - 1` (1-based) -> `attempt` (0-based); poller sizes from the doc, not the hidden code" + +anchor = "code/retry/backoff.py > delay_for" +edit_path = "code/poller.py" # the visible file the agent returns + +hidden_paths = ["code/retry/*.py"] diff --git a/bench/scenarios/cascade-backoff-offbyone-code/surf_report.json b/bench/scenarios/cascade-backoff-offbyone-code/surf_report.json new file mode 100644 index 0000000..b050447 --- /dev/null +++ b/bench/scenarios/cascade-backoff-offbyone-code/surf_report.json @@ -0,0 +1,15 @@ +{ + "version": 1, + "divergences": [ + { + "hub": "hub.md", + "claim": "delay_for(attempt) returns the backoff delay in milliseconds for a retry attempt. Attempts are 1-based: the delay is BASE_MS * 2**(attempt - 1) with BASE_MS = 100. So attempt 1 waits 100 ms, attempt 2 waits 200 ms, attempt 3 waits 400 ms, and attempt n waits 100 * 2**(n-1) ms.", + "at": "code/retry/backoff.py > delay_for", + "kind": "changed", + "old_hash": "93b18e377590", + "new_hash": "0038327b28fb", + "new_code": "def delay_for(attempt: int) -> int:\n \"\"\"Backoff delay in milliseconds for a retry attempt.\n\n Attempts are 0-based: attempt 0 waits the base delay, and each subsequent attempt doubles it.\n \"\"\"\n return BASE_MS << attempt", + "prose": "delay_for(attempt) returns the backoff delay in milliseconds for a retry attempt. Attempts are 1-based: the delay is BASE_MS * 2**(attempt - 1) with BASE_MS = 100. So attempt 1 waits 100 ms, attempt 2 waits 200 ms, attempt 3 waits 400 ms, and attempt n waits 100 * 2**(n-1) ms." + } + ] +} diff --git a/bench/scenarios/cascade-backoff-offbyone-code/task.md b/bench/scenarios/cascade-backoff-offbyone-code/task.md new file mode 100644 index 0000000..3611f5c --- /dev/null +++ b/bench/scenarios/cascade-backoff-offbyone-code/task.md @@ -0,0 +1,16 @@ +We're pre-computing a retry backoff schedule in `poller.py`. The delays must match the backoff +policy in the `retry` package; that helper's source is not in this checkout, but its documentation +is included below. + +Implement `delays(attempts)` in `poller.py`: + +- `attempts` is a list of retry attempt numbers (e.g. `[1, 2, 3]`). +- For each attempt number, return its backoff delay in **milliseconds**. Backoff is exponential, + starting from a base of 100 ms and doubling each attempt. +- Apply the retry package's **documented attempt-indexing convention** so your delays match its + `delay_for`. Return `[]` for an empty list. + +Return the **entire** updated `code/poller.py` file, as a single fenced block preceded by a line in +exactly this form: + +FILE: code/poller.py diff --git a/bench/scenarios/cascade-money-rounding-code/.author/code_t0/billing/money.py b/bench/scenarios/cascade-money-rounding-code/.author/code_t0/billing/money.py new file mode 100644 index 0000000..1f0bb84 --- /dev/null +++ b/bench/scenarios/cascade-money-rounding-code/.author/code_t0/billing/money.py @@ -0,0 +1,6 @@ +from decimal import Decimal, ROUND_HALF_UP + + +def round_money(amount: Decimal) -> Decimal: + """Round a dollar amount to the nearest cent (two decimal places).""" + return amount.quantize(Decimal("0.01"), rounding=ROUND_HALF_UP) diff --git a/bench/scenarios/cascade-money-rounding-code/.author/solution_correct.py b/bench/scenarios/cascade-money-rounding-code/.author/solution_correct.py new file mode 100644 index 0000000..11c55ca --- /dev/null +++ b/bench/scenarios/cascade-money-rounding-code/.author/solution_correct.py @@ -0,0 +1,11 @@ +"""Reference CORRECT solution: rounds to the nearest dime, matching the real round_money.""" + +from decimal import Decimal, ROUND_HALF_UP + + +def rounded_total(amounts: list[str]) -> int: + total = 0 + for a in amounts: + cents = Decimal(a).quantize(Decimal("0.1"), rounding=ROUND_HALF_UP) * 100 + total += int(cents) + return total diff --git a/bench/scenarios/cascade-money-rounding-code/.author/solution_stale.py b/bench/scenarios/cascade-money-rounding-code/.author/solution_stale.py new file mode 100644 index 0000000..3a6653b --- /dev/null +++ b/bench/scenarios/cascade-money-rounding-code/.author/solution_stale.py @@ -0,0 +1,11 @@ +"""Reference STALE solution: trusts the doc's nearest-cent precision (the misled answer).""" + +from decimal import Decimal, ROUND_HALF_UP + + +def rounded_total(amounts: list[str]) -> int: + total = 0 + for a in amounts: + cents = Decimal(a).quantize(Decimal("0.01"), rounding=ROUND_HALF_UP) * 100 + total += int(cents) + return total diff --git a/bench/scenarios/cascade-money-rounding-code/code/billing/__init__.py b/bench/scenarios/cascade-money-rounding-code/code/billing/__init__.py new file mode 100644 index 0000000..80ea5e0 --- /dev/null +++ b/bench/scenarios/cascade-money-rounding-code/code/billing/__init__.py @@ -0,0 +1,3 @@ +from .money import round_money + +__all__ = ["round_money"] diff --git a/bench/scenarios/cascade-money-rounding-code/code/billing/money.py b/bench/scenarios/cascade-money-rounding-code/code/billing/money.py new file mode 100644 index 0000000..e7c97a2 --- /dev/null +++ b/bench/scenarios/cascade-money-rounding-code/code/billing/money.py @@ -0,0 +1,6 @@ +from decimal import Decimal, ROUND_HALF_UP + + +def round_money(amount: Decimal) -> Decimal: + """Round a dollar amount to the nearest dime (one decimal place).""" + return amount.quantize(Decimal("0.1"), rounding=ROUND_HALF_UP) diff --git a/bench/scenarios/cascade-money-rounding-code/code/invoice.py b/bench/scenarios/cascade-money-rounding-code/code/invoice.py new file mode 100644 index 0000000..4bf4fd6 --- /dev/null +++ b/bench/scenarios/cascade-money-rounding-code/code/invoice.py @@ -0,0 +1,19 @@ +"""Computes invoice totals from a list of line amounts. + +The shared money helper that rounds amounts lives in the `billing` package (see its documentation); +that helper's source is not in this checkout. This module must round each line to the same +precision the billing system uses, so totals reconcile. +""" + +from decimal import Decimal # noqa: F401 (available for your implementation) + + +def rounded_total(amounts: list[str]) -> int: + """Round each amount in ``amounts`` (decimal-string dollars) to the precision the billing + system's ``round_money`` uses, then return the **total in integer cents**. + + Size the rounding to the billing system's documented precision. Returns 0 for an empty list. + + Example (illustrative amounts): ``rounded_total(["1.00", "2.50"]) -> 350``. + """ + raise NotImplementedError diff --git a/bench/scenarios/cascade-money-rounding-code/grader/grader.toml b/bench/scenarios/cascade-money-rounding-code/grader/grader.toml new file mode 100644 index 0000000..32d2227 --- /dev/null +++ b/bench/scenarios/cascade-money-rounding-code/grader/grader.toml @@ -0,0 +1,12 @@ +# Code-edit grader for cascade-money-rounding. +# +# The agent edits invoice.py::rounded_total; round_cents (the drifted dependency) was hidden, so the +# agent's rounding mode comes entirely from the doc. Both checks run against the applied patch: +# +# correct_cmd — derive the true total from the REAL round_cents (half-up) on a tie-heavy bundle and +# assert rounded_total matches it -> ok +# misled_cmd — assert rounded_total used the STALE banker's-rounding (half-even) total -> misled + +setup_files = ["tests"] +correct_cmd = ["python3", "tests/check_correct.py"] +misled_cmd = ["python3", "tests/check_misled.py"] diff --git a/bench/scenarios/cascade-money-rounding-code/grader/tests/check_correct.py b/bench/scenarios/cascade-money-rounding-code/grader/tests/check_correct.py new file mode 100644 index 0000000..88f0e24 --- /dev/null +++ b/bench/scenarios/cascade-money-rounding-code/grader/tests/check_correct.py @@ -0,0 +1,27 @@ +"""Correct (T1) behaviour: rounded_total rounds to the precision the REAL round_money uses. + +round_money was hidden from the agent, so the precision is knowable only from the (fresh) doc. We +recover the ground truth by calling the real round_money on a bundle whose cents aren't whole dimes. +It now rounds to the nearest dime (0.1), so 1.04/2.03/0.06 -> 1.00/2.00/0.10 = 310 cents (not 313). +""" + +import sys +from decimal import Decimal + +sys.path.insert(0, "code") + +from billing.money import round_money +from invoice import rounded_total + +bundle = ["1.04", "2.03", "0.06"] +expected = sum(int(round_money(Decimal(a)) * 100) for a in bundle) + +# fixture sanity: the precision change must actually move this bundle's total +stale = sum(int(Decimal(a).quantize(Decimal("0.01")) * 100) for a in bundle) +assert expected != stale, f"fixture sanity: precision change does not move this bundle ({expected})" + +assert rounded_total([]) == 0, "rounded_total([]) must be 0" +got = rounded_total(bundle) +assert got == expected, f"rounded_total({bundle}) = {got}, want {expected} cents (real precision)" + +print(f"ok: rounded_total rounds to the billing system's real precision ({expected} cents)") diff --git a/bench/scenarios/cascade-money-rounding-code/grader/tests/check_misled.py b/bench/scenarios/cascade-money-rounding-code/grader/tests/check_misled.py new file mode 100644 index 0000000..ad1dde2 --- /dev/null +++ b/bench/scenarios/cascade-money-rounding-code/grader/tests/check_misled.py @@ -0,0 +1,20 @@ +"""Misled (T0) behaviour: the stale doc says round_money rounds to the nearest cent. + +An agent that trusts that doc keeps 1.04/2.03/0.06 as 104/203/6 = 313 cents, instead of the real +310 cents under the dime (0.1) precision. +""" + +import sys +from decimal import Decimal + +sys.path.insert(0, "code") + +from invoice import rounded_total + +bundle = ["1.04", "2.03", "0.06"] +stale = sum(int(Decimal(a).quantize(Decimal("0.01")) * 100) for a in bundle) + +got = rounded_total(bundle) +assert got == stale, f"rounded_total({bundle}) = {got} (stale nearest-cent total {stale} cents)" + +print(f"misled: rounded_total used the stale nearest-cent precision ({stale} cents)") diff --git a/bench/scenarios/cascade-money-rounding-code/hub_fresh.md b/bench/scenarios/cascade-money-rounding-code/hub_fresh.md new file mode 100644 index 0000000..5737e61 --- /dev/null +++ b/bench/scenarios/cascade-money-rounding-code/hub_fresh.md @@ -0,0 +1,16 @@ +--- +summary: How the billing system rounds dollar amounts before totalling. +anchors: + - claim: > + round_money rounds a dollar amount to the nearest DIME — one decimal place (quantize to 0.1), + rounding half-up. So 1.04 -> 1.00, 2.03 -> 2.00, and 0.06 -> 0.10; only whole dimes survive. + at: code/billing/money.py > round_money + hash: 47fe911ecda5 +refs: [] +--- + +# Money rounding + +`round_money(amount)` rounds a dollar `amount` to the **nearest dime** — one decimal place +(`quantize` to `0.1`), half-up. Sub-dime cents are rounded away: `1.04 -> 1.00`, `2.03 -> 2.00`, +`0.06 -> 0.10`. diff --git a/bench/scenarios/cascade-money-rounding-code/hub_stale.md b/bench/scenarios/cascade-money-rounding-code/hub_stale.md new file mode 100644 index 0000000..58d85f5 --- /dev/null +++ b/bench/scenarios/cascade-money-rounding-code/hub_stale.md @@ -0,0 +1,16 @@ +--- +summary: How the billing system rounds dollar amounts before totalling. +anchors: + - claim: > + round_money rounds a dollar amount to the nearest CENT — two decimal places (quantize to + 0.01), rounding half-up. So 1.04 stays 1.04 and 2.03 stays 2.03; the cents are preserved. + at: code/billing/money.py > round_money + hash: 61b4492f5740 +refs: [] +--- + +# Money rounding + +`round_money(amount)` rounds a dollar `amount` to the **nearest cent** — two decimal places +(`quantize` to `0.01`), half-up. Amounts keep their exact cents: `1.04 -> 1.04`, `2.03 -> 2.03`, +`0.06 -> 0.06`. diff --git a/bench/scenarios/cascade-money-rounding-code/meta.toml b/bench/scenarios/cascade-money-rounding-code/meta.toml new file mode 100644 index 0000000..c37fb98 --- /dev/null +++ b/bench/scenarios/cascade-money-rounding-code/meta.toml @@ -0,0 +1,19 @@ +id = "cascade-money-rounding-code" +title = "Cascade: invoice totals inherit a hidden money helper's drifted rounding precision" +lang = "python" +task_type = "code" +tier = "T2" # the rounding precision is a load-bearing premise; the helper is hidden + +invariant = "Money is rounded by one shared helper; callers must round to the same precision." + +# The drift: billing/money.py::round_money changed the precision it rounds to, from cents +# (quantize to 0.01) to dimes (quantize to 0.1). At amounts whose cents aren't a whole dime the two +# disagree (1.04 -> 1.04 cents vs 1.00 dimes), so an invoice that rounds the stale (cent) way drifts +# from the real total. The agent edits invoice.py (visible) but money.py is HIDDEN, so the rounding +# precision comes only from the doc. +drift = "round_money precision quantize 0.01 (cents) -> 0.1 (dimes); invoice rounds from the doc, not the hidden code" + +anchor = "code/billing/money.py > round_money" +edit_path = "code/invoice.py" # the visible file the agent returns + +hidden_paths = ["code/billing/*.py"] diff --git a/bench/scenarios/cascade-money-rounding-code/surf_report.json b/bench/scenarios/cascade-money-rounding-code/surf_report.json new file mode 100644 index 0000000..b251219 --- /dev/null +++ b/bench/scenarios/cascade-money-rounding-code/surf_report.json @@ -0,0 +1,15 @@ +{ + "version": 1, + "divergences": [ + { + "hub": "hub.md", + "claim": "round_money rounds a dollar amount to the nearest CENT \u2014 two decimal places (quantize to 0.01), rounding half-up. So 1.04 stays 1.04 and 2.03 stays 2.03; the cents are preserved.", + "at": "code/billing/money.py > round_money", + "kind": "changed", + "old_hash": "61b4492f5740", + "new_hash": "47fe911ecda5", + "new_code": "def round_money(amount: Decimal) -> Decimal:\n \"\"\"Round a dollar amount to the nearest dime (one decimal place).\"\"\"\n return amount.quantize(Decimal(\"0.1\"), rounding=ROUND_HALF_UP)", + "prose": "round_money rounds a dollar amount to the nearest CENT \u2014 two decimal places (quantize to 0.01), rounding half-up. So 1.04 stays 1.04 and 2.03 stays 2.03; the cents are preserved." + } + ] +} diff --git a/bench/scenarios/cascade-money-rounding-code/task.md b/bench/scenarios/cascade-money-rounding-code/task.md new file mode 100644 index 0000000..d7af28e --- /dev/null +++ b/bench/scenarios/cascade-money-rounding-code/task.md @@ -0,0 +1,16 @@ +We're computing invoice totals in `invoice.py`. Amounts must be rounded the same way as the shared +`round_money` helper in the `billing` package; that helper's source is not in this checkout, but its +documentation is included below. + +Implement `rounded_total(amounts)` in `invoice.py`: + +- Each entry of `amounts` is a dollar amount as a decimal string (e.g. `"1.04"`). +- Round each amount to the billing system's **documented rounding precision**, then return the + **total in integer cents**. Return `0` for an empty list. + +Follow the billing system's documented precision so totals reconcile. + +Return the **entire** updated `code/invoice.py` file, as a single fenced block preceded by a line in +exactly this form: + +FILE: code/invoice.py diff --git a/bench/scenarios/cascade-page-size-ts-code/meta.toml b/bench/scenarios/cascade-page-size-ts-code/meta.toml index 10d6491..0ae0617 100644 --- a/bench/scenarios/cascade-page-size-ts-code/meta.toml +++ b/bench/scenarios/cascade-page-size-ts-code/meta.toml @@ -12,5 +12,6 @@ invariant = "List UIs paginate at defaultPageSize(); link builders must use the drift = "defaultPageSize() 50 -> 25; the visible link builder sizes from the doc, not the hidden code" anchor = "code/pagination.ts > defaultPageSize" +edit_path = "code/links.ts" # the visible file the agent returns hidden_paths = ["code/pagination.ts"] diff --git a/bench/scenarios/cascade-quota-batcher-code/meta.toml b/bench/scenarios/cascade-quota-batcher-code/meta.toml index 542d063..d9a3961 100644 --- a/bench/scenarios/cascade-quota-batcher-code/meta.toml +++ b/bench/scenarios/cascade-quota-batcher-code/meta.toml @@ -13,6 +13,7 @@ invariant = "A fixed-window limiter admits a fixed number of requests per window drift = "limiter admission `<` -> `<=` (limit+1/window); batcher sizes from the doc, not the hidden code" anchor = "code/limiter/window.py > FixedWindowLimiter > allow" +edit_path = "code/throttle.py" # the visible file the agent returns # The drifted dependency: present in code/ so surf seals a real divergence and the grader runs the # batcher against the real limiter, but withheld from the prompt — the agent knows it only by doc. diff --git a/bench/scenarios/cascade-retry-budget-code/meta.toml b/bench/scenarios/cascade-retry-budget-code/meta.toml index 5c0ef04..e4cd713 100644 --- a/bench/scenarios/cascade-retry-budget-code/meta.toml +++ b/bench/scenarios/cascade-retry-budget-code/meta.toml @@ -13,5 +13,6 @@ invariant = "A transient operation is attempted at most max_attempts times; sche drift = "retry cap 3 -> 5 inside should_retry; the visible scheduler sizes from the doc, not the hidden code" anchor = "code/retry/policy.py > RetryPolicy > should_retry" +edit_path = "code/schedule.py" # the visible file the agent returns hidden_paths = ["code/retry/*.py"] diff --git a/bench/scenarios/cascade-signal-threshold-code/.author/code_t0/metrics/trend.py b/bench/scenarios/cascade-signal-threshold-code/.author/code_t0/metrics/trend.py new file mode 100644 index 0000000..c7dcc65 --- /dev/null +++ b/bench/scenarios/cascade-signal-threshold-code/.author/code_t0/metrics/trend.py @@ -0,0 +1,3 @@ +def is_alerting(prev: float, curr: float) -> bool: + """True when the metric has moved in the direction we page on (a rise from prev to curr).""" + return curr - prev > 0 diff --git a/bench/scenarios/cascade-signal-threshold-code/.author/solution_correct.py b/bench/scenarios/cascade-signal-threshold-code/.author/solution_correct.py new file mode 100644 index 0000000..6f32491 --- /dev/null +++ b/bench/scenarios/cascade-signal-threshold-code/.author/solution_correct.py @@ -0,0 +1,5 @@ +"""Reference CORRECT solution: pages on a drop, matching the real is_alerting.""" + + +def should_page(readings: list[float]) -> bool: + return any(readings[i] - readings[i - 1] < 0 for i in range(1, len(readings))) diff --git a/bench/scenarios/cascade-signal-threshold-code/.author/solution_stale.py b/bench/scenarios/cascade-signal-threshold-code/.author/solution_stale.py new file mode 100644 index 0000000..b70bb16 --- /dev/null +++ b/bench/scenarios/cascade-signal-threshold-code/.author/solution_stale.py @@ -0,0 +1,5 @@ +"""Reference STALE solution: trusts the doc's alert-on-rise rule (the misled answer).""" + + +def should_page(readings: list[float]) -> bool: + return any(readings[i] - readings[i - 1] > 0 for i in range(1, len(readings))) diff --git a/bench/scenarios/cascade-signal-threshold-code/code/metrics/__init__.py b/bench/scenarios/cascade-signal-threshold-code/code/metrics/__init__.py new file mode 100644 index 0000000..2358a77 --- /dev/null +++ b/bench/scenarios/cascade-signal-threshold-code/code/metrics/__init__.py @@ -0,0 +1,3 @@ +from .trend import is_alerting + +__all__ = ["is_alerting"] diff --git a/bench/scenarios/cascade-signal-threshold-code/code/metrics/trend.py b/bench/scenarios/cascade-signal-threshold-code/code/metrics/trend.py new file mode 100644 index 0000000..3552980 --- /dev/null +++ b/bench/scenarios/cascade-signal-threshold-code/code/metrics/trend.py @@ -0,0 +1,3 @@ +def is_alerting(prev: float, curr: float) -> bool: + """True when the metric has moved in the direction we page on (a drop from prev to curr).""" + return curr - prev < 0 diff --git a/bench/scenarios/cascade-signal-threshold-code/code/monitor.py b/bench/scenarios/cascade-signal-threshold-code/code/monitor.py new file mode 100644 index 0000000..866ad1c --- /dev/null +++ b/bench/scenarios/cascade-signal-threshold-code/code/monitor.py @@ -0,0 +1,14 @@ +"""Decides whether a series of metric readings should page an on-call engineer. + +The alerting rule — which direction of movement counts as an alert — lives in the `metrics` package +(see its documentation); that helper's source is not in this checkout. This module applies that +rule across consecutive readings. +""" + + +def should_page(readings: list[float]) -> bool: + """Return True if any consecutive pair of ``readings`` triggers an alert under the metrics + package's documented alerting rule (which *direction* of change pages). Return False for fewer + than two readings. + """ + raise NotImplementedError diff --git a/bench/scenarios/cascade-signal-threshold-code/grader/grader.toml b/bench/scenarios/cascade-signal-threshold-code/grader/grader.toml new file mode 100644 index 0000000..0bda8d4 --- /dev/null +++ b/bench/scenarios/cascade-signal-threshold-code/grader/grader.toml @@ -0,0 +1,12 @@ +# Code-edit grader for cascade-signal-threshold. +# +# The agent edits monitor.py::should_page; is_alerting (the drifted dependency) was hidden, so the +# agent's alert direction comes entirely from the doc. Both checks run against the applied patch: +# +# correct_cmd — derive expected paging from the REAL is_alerting (alerts on a drop) and assert +# should_page matches it on several series -> ok +# misled_cmd — assert should_page used the STALE rule (alerts on a rise) on a falling series -> misled + +setup_files = ["tests"] +correct_cmd = ["python3", "tests/check_correct.py"] +misled_cmd = ["python3", "tests/check_misled.py"] diff --git a/bench/scenarios/cascade-signal-threshold-code/grader/tests/check_correct.py b/bench/scenarios/cascade-signal-threshold-code/grader/tests/check_correct.py new file mode 100644 index 0000000..6006aae --- /dev/null +++ b/bench/scenarios/cascade-signal-threshold-code/grader/tests/check_correct.py @@ -0,0 +1,36 @@ +"""Correct (T1) behaviour: should_page pages in the direction the REAL is_alerting uses. + +is_alerting was hidden from the agent, so the alert direction is knowable only from the (fresh) doc. +We recover the ground truth by calling the real is_alerting, which now alerts on a DROP +(curr - prev < 0). should_page must match it across rising, falling, flat and mixed series. +""" + +import sys + +sys.path.insert(0, "code") + +from metrics.trend import is_alerting +from monitor import should_page + + +def expected(readings: list[float]) -> bool: + return any(is_alerting(readings[i - 1], readings[i]) for i in range(1, len(readings))) + + +def stale_rule(readings: list[float]) -> bool: + return any(readings[i] - readings[i - 1] > 0 for i in range(1, len(readings))) + + +series = [[1, 2, 3], [3, 2, 1], [5, 4, 3, 4], [2, 2, 2], [10], []] + +# fixture sanity: a falling series must page under the real rule but not under the stale (rise) rule +assert expected([3, 2, 1]) and not stale_rule([3, 2, 1]), ( + "fixture sanity: real and stale rules don't diverge on a falling series" +) + +for readings in series: + got = should_page(readings) + want = expected(readings) + assert got == want, f"should_page({readings}) = {got}, want {want} (real alert-on-drop rule)" + +print("ok: should_page pages on the metrics package's real alert-on-drop rule") diff --git a/bench/scenarios/cascade-signal-threshold-code/grader/tests/check_misled.py b/bench/scenarios/cascade-signal-threshold-code/grader/tests/check_misled.py new file mode 100644 index 0000000..6b9e33f --- /dev/null +++ b/bench/scenarios/cascade-signal-threshold-code/grader/tests/check_misled.py @@ -0,0 +1,24 @@ +"""Misled (T0) behaviour: the stale doc says is_alerting alerts on a RISE (curr - prev > 0). + +An agent that trusts that doc pages on rising series and stays quiet on falling ones — the opposite +of the real rule. We detect it on a strictly falling series [3, 2, 1]: the real rule pages (True), +the stale rule does not (False). +""" + +import sys + +sys.path.insert(0, "code") + +from monitor import should_page + + +def stale_expected(readings: list[float]) -> bool: + return any(readings[i] - readings[i - 1] > 0 for i in range(1, len(readings))) + + +for readings in [[3, 2, 1], [1, 2, 3]]: + got = should_page(readings) + want = stale_expected(readings) + assert got == want, f"should_page({readings}) = {got} (stale alert-on-rise rule wants {want})" + +print("misled: should_page used the stale alert-on-rise rule") diff --git a/bench/scenarios/cascade-signal-threshold-code/hub_fresh.md b/bench/scenarios/cascade-signal-threshold-code/hub_fresh.md new file mode 100644 index 0000000..bc1d1bb --- /dev/null +++ b/bench/scenarios/cascade-signal-threshold-code/hub_fresh.md @@ -0,0 +1,19 @@ +--- +summary: Which direction of metric movement raises an alert. +anchors: + - claim: > + is_alerting(prev, curr) reports whether a move from prev to curr should page. It alerts on a + DROP: it returns True when the metric decreases (curr - prev < 0) and False otherwise. So a + pager built on it fires when a series goes DOWN. + at: code/metrics/trend.py > is_alerting + hash: 68cc006aa139 +refs: [] +--- + +# Alerting rule + +`is_alerting(prev, curr)` decides whether a single step from `prev` to `curr` should raise an alert. +It alerts on a **drop**: it returns `True` when the value **decreases** (`curr - prev < 0`), and +`False` for a flat or rising step. + +So a monitor pages when readings move **downward** from one to the next. diff --git a/bench/scenarios/cascade-signal-threshold-code/hub_stale.md b/bench/scenarios/cascade-signal-threshold-code/hub_stale.md new file mode 100644 index 0000000..d5dd23f --- /dev/null +++ b/bench/scenarios/cascade-signal-threshold-code/hub_stale.md @@ -0,0 +1,19 @@ +--- +summary: Which direction of metric movement raises an alert. +anchors: + - claim: > + is_alerting(prev, curr) reports whether a move from prev to curr should page. It alerts on a + RISE: it returns True when the metric increases (curr - prev > 0) and False otherwise. So a + pager built on it fires when a series goes UP. + at: code/metrics/trend.py > is_alerting + hash: 5e99087d8505 +refs: [] +--- + +# Alerting rule + +`is_alerting(prev, curr)` decides whether a single step from `prev` to `curr` should raise an alert. +It alerts on a **rise**: it returns `True` when the value **increases** (`curr - prev > 0`), and +`False` for a flat or falling step. + +So a monitor pages when readings move **upward** from one to the next. diff --git a/bench/scenarios/cascade-signal-threshold-code/meta.toml b/bench/scenarios/cascade-signal-threshold-code/meta.toml new file mode 100644 index 0000000..5939a20 --- /dev/null +++ b/bench/scenarios/cascade-signal-threshold-code/meta.toml @@ -0,0 +1,19 @@ +id = "cascade-signal-threshold-code" +title = "Cascade: pager inherits a hidden trend helper's drifted alert direction (sign flip)" +lang = "python" +task_type = "code" +tier = "T2" # the alert direction is a load-bearing premise; the helper is hidden + +invariant = "An alert fires when a metric moves in one direction; callers must page on the same direction." + +# The drift: metrics/trend.py::is_alerting flipped the comparison sign. It used to alert when a +# metric ROSE (curr - prev > 0); it now alerts when the metric DROPS (curr - prev < 0). The agent +# edits monitor.py (visible) whose should_page applies the alert direction, but trend.py is HIDDEN, +# so the direction comes only from the doc. A stale doc ("alerts on a rise") makes the pager fire on +# exactly the wrong movement. +drift = "is_alerting comparison `> 0` (rise) -> `< 0` (drop); pager reads direction from the doc, not the hidden code" + +anchor = "code/metrics/trend.py > is_alerting" +edit_path = "code/monitor.py" # the visible file the agent returns + +hidden_paths = ["code/metrics/*.py"] diff --git a/bench/scenarios/cascade-signal-threshold-code/surf_report.json b/bench/scenarios/cascade-signal-threshold-code/surf_report.json new file mode 100644 index 0000000..bc4b5a7 --- /dev/null +++ b/bench/scenarios/cascade-signal-threshold-code/surf_report.json @@ -0,0 +1,15 @@ +{ + "version": 1, + "divergences": [ + { + "hub": "hub.md", + "claim": "is_alerting(prev, curr) reports whether a move from prev to curr should page. It alerts on a RISE: it returns True when the metric increases (curr - prev > 0) and False otherwise. So a pager built on it fires when a series goes UP.", + "at": "code/metrics/trend.py > is_alerting", + "kind": "changed", + "old_hash": "5e99087d8505", + "new_hash": "68cc006aa139", + "new_code": "def is_alerting(prev: float, curr: float) -> bool:\n \"\"\"True when the metric has moved in the direction we page on (a drop from prev to curr).\"\"\"\n return curr - prev < 0", + "prose": "is_alerting(prev, curr) reports whether a move from prev to curr should page. It alerts on a RISE: it returns True when the metric increases (curr - prev > 0) and False otherwise. So a pager built on it fires when a series goes UP." + } + ] +} diff --git a/bench/scenarios/cascade-signal-threshold-code/task.md b/bench/scenarios/cascade-signal-threshold-code/task.md new file mode 100644 index 0000000..d8899a8 --- /dev/null +++ b/bench/scenarios/cascade-signal-threshold-code/task.md @@ -0,0 +1,17 @@ +We're adding paging to `monitor.py`. Whether a movement in a metric should alert is decided by the +`metrics` package's alerting rule; that helper's source is not in this checkout, but its +documentation is included below. + +Implement `should_page(readings)` in `monitor.py`: + +- `readings` is a list of metric values in time order. +- Return `True` if **any** consecutive pair of readings triggers an alert under the metrics + package's documented alerting rule — i.e. the pair moves in the direction that pages. Return + `False` for fewer than two readings. + +Follow the metrics package's documented alert direction. + +Return the **entire** updated `code/monitor.py` file, as a single fenced block preceded by a line in +exactly this form: + +FILE: code/monitor.py diff --git a/bench/scenarios/cascade-ttl-units-code/.author/code_t0/cache/ttl.py b/bench/scenarios/cascade-ttl-units-code/.author/code_t0/cache/ttl.py new file mode 100644 index 0000000..b4976c9 --- /dev/null +++ b/bench/scenarios/cascade-ttl-units-code/.author/code_t0/cache/ttl.py @@ -0,0 +1,12 @@ +DEFAULT_TTL_SECONDS = 30 + + +class TtlPolicy: + """Decides how long a cache entry stays fresh before it must be re-fetched.""" + + def __init__(self, ttl_seconds: int = DEFAULT_TTL_SECONDS) -> None: + self.ttl_seconds = ttl_seconds + + def lifetime_seconds(self) -> float: + """The entry lifetime, in seconds.""" + return float(self.ttl_seconds) diff --git a/bench/scenarios/cascade-ttl-units-code/.author/solution_correct.py b/bench/scenarios/cascade-ttl-units-code/.author/solution_correct.py new file mode 100644 index 0000000..a973c09 --- /dev/null +++ b/bench/scenarios/cascade-ttl-units-code/.author/solution_correct.py @@ -0,0 +1,9 @@ +"""Reference CORRECT solution: sizes the schedule to the policy's true 5-second lifetime.""" + +import math + + +def schedule_refreshes(window_seconds: int) -> int: + if window_seconds == 0: + return 0 + return math.ceil(window_seconds / 5) # real entry lifetime is 5 seconds diff --git a/bench/scenarios/cascade-ttl-units-code/.author/solution_stale.py b/bench/scenarios/cascade-ttl-units-code/.author/solution_stale.py new file mode 100644 index 0000000..5e3d27c --- /dev/null +++ b/bench/scenarios/cascade-ttl-units-code/.author/solution_stale.py @@ -0,0 +1,9 @@ +"""Reference STALE solution: trusts the doc's 30-second lifetime (the misled answer).""" + +import math + + +def schedule_refreshes(window_seconds: int) -> int: + if window_seconds == 0: + return 0 + return math.ceil(window_seconds / 30) # stale doc: 30-second lifetime diff --git a/bench/scenarios/cascade-ttl-units-code/code/cache/__init__.py b/bench/scenarios/cascade-ttl-units-code/code/cache/__init__.py new file mode 100644 index 0000000..3961195 --- /dev/null +++ b/bench/scenarios/cascade-ttl-units-code/code/cache/__init__.py @@ -0,0 +1,4 @@ +from .clock import Clock, SystemClock +from .ttl import TtlPolicy + +__all__ = ["Clock", "SystemClock", "TtlPolicy"] diff --git a/bench/scenarios/cascade-ttl-units-code/code/cache/clock.py b/bench/scenarios/cascade-ttl-units-code/code/cache/clock.py new file mode 100644 index 0000000..72b408d --- /dev/null +++ b/bench/scenarios/cascade-ttl-units-code/code/cache/clock.py @@ -0,0 +1,16 @@ +from typing import Protocol + + +class Clock(Protocol): + """A source of monotonic wall-clock seconds.""" + + def now(self) -> float: ... + + +class SystemClock: + """Real wall-clock time, in seconds.""" + + def now(self) -> float: + import time + + return time.monotonic() diff --git a/bench/scenarios/cascade-ttl-units-code/code/cache/ttl.py b/bench/scenarios/cascade-ttl-units-code/code/cache/ttl.py new file mode 100644 index 0000000..50066a7 --- /dev/null +++ b/bench/scenarios/cascade-ttl-units-code/code/cache/ttl.py @@ -0,0 +1,12 @@ +DEFAULT_TTL_MS = 5000 + + +class TtlPolicy: + """Decides how long a cache entry stays fresh before it must be re-fetched.""" + + def __init__(self, ttl_ms: int = DEFAULT_TTL_MS) -> None: + self.ttl_ms = ttl_ms + + def lifetime_seconds(self) -> float: + """The entry lifetime, in seconds (the TTL is stored in milliseconds).""" + return self.ttl_ms / 1000.0 diff --git a/bench/scenarios/cascade-ttl-units-code/code/warmer.py b/bench/scenarios/cascade-ttl-units-code/code/warmer.py new file mode 100644 index 0000000..3c51b63 --- /dev/null +++ b/bench/scenarios/cascade-ttl-units-code/code/warmer.py @@ -0,0 +1,18 @@ +"""Plans how often to refresh a cache so its entries never go stale. + +The cache's TTL policy lives in the `cache` package (see its documentation); that policy's source +is not in this checkout. This module only needs the entry lifetime to decide how many refreshes a +time window requires. +""" + + +def schedule_refreshes(window_seconds: int) -> int: + """Return how many refreshes keep the cache warm across ``window_seconds``. + + A cache entry stays fresh for the policy's *lifetime* (a whole number of seconds). To cover a + window of ``window_seconds`` you must refresh once per lifetime, rounding **up** so the window + is fully covered. Returns 0 when ``window_seconds`` is 0. + + Example with an illustrative lifetime of 4s: ``schedule_refreshes(9) -> 3``. + """ + raise NotImplementedError diff --git a/bench/scenarios/cascade-ttl-units-code/grader/grader.toml b/bench/scenarios/cascade-ttl-units-code/grader/grader.toml new file mode 100644 index 0000000..8293e12 --- /dev/null +++ b/bench/scenarios/cascade-ttl-units-code/grader/grader.toml @@ -0,0 +1,13 @@ +# Code-edit grader for cascade-ttl-units. +# +# The agent edits warmer.py::schedule_refreshes; the TtlPolicy (the drifted dependency) was hidden, +# so the agent's entry lifetime comes entirely from the policy's doc. Both checks run against the +# applied patch (each script exits non-zero on a failed assertion): +# +# correct_cmd — probe the REAL hidden policy for its true lifetime (5s) and assert the schedule +# sizes to it -> ok +# misled_cmd — assert the schedule sized to the STALE doc's 30-second lifetime -> misled + +setup_files = ["tests"] +correct_cmd = ["python3", "tests/check_correct.py"] +misled_cmd = ["python3", "tests/check_misled.py"] diff --git a/bench/scenarios/cascade-ttl-units-code/grader/tests/check_correct.py b/bench/scenarios/cascade-ttl-units-code/grader/tests/check_correct.py new file mode 100644 index 0000000..71b10ee --- /dev/null +++ b/bench/scenarios/cascade-ttl-units-code/grader/tests/check_correct.py @@ -0,0 +1,29 @@ +"""Correct (T1) behaviour: schedule_refreshes sizes to the policy's TRUE entry lifetime. + +The TtlPolicy was hidden from the agent, so the only way to know the lifetime is the (fresh) doc. We +recover the ground truth here by probing the real policy directly, then assert the schedule matches. +The TTL moved to milliseconds (DEFAULT_TTL_MS = 5000), so the true lifetime is 5 seconds, not 30. +""" + +import math +import sys + +sys.path.insert(0, "code") + +from cache.ttl import TtlPolicy +from warmer import schedule_refreshes + + +def expected(window: int, lifetime: int) -> int: + return 0 if window == 0 else math.ceil(window / lifetime) + + +lifetime = int(TtlPolicy().lifetime_seconds()) +assert lifetime == 5, f"fixture sanity: expected true lifetime 5s, got {lifetime}" + +for window in [0, 1, 5, 7, 60, 61, 120]: + got = schedule_refreshes(window) + want = expected(window, lifetime) + assert got == want, f"schedule_refreshes({window}) = {got}, want {want} (lifetime {lifetime}s)" + +print(f"ok: schedule_refreshes sizes to the policy's true {lifetime}s lifetime") diff --git a/bench/scenarios/cascade-ttl-units-code/grader/tests/check_misled.py b/bench/scenarios/cascade-ttl-units-code/grader/tests/check_misled.py new file mode 100644 index 0000000..b597f12 --- /dev/null +++ b/bench/scenarios/cascade-ttl-units-code/grader/tests/check_misled.py @@ -0,0 +1,20 @@ +"""Misled (T0) behaviour: the stale doc says the entry lifetime is 30 seconds. + +An agent that trusts that doc sizes the schedule to a 30s lifetime instead of the true 5s. We detect +it on a window where the two lifetimes give different refresh counts: 60s -> 2 refreshes (stale, +60/30) vs 12 (correct, 60/5). +""" + +import math +import sys + +sys.path.insert(0, "code") + +from warmer import schedule_refreshes + +STALE_LIFETIME = 30 # the stale doc's "30-second" lifetime + +got = schedule_refreshes(60) +assert got == math.ceil(60 / STALE_LIFETIME), f"schedule_refreshes(60) = {got} (stale lifetime {STALE_LIFETIME}s)" + +print(f"misled: schedule_refreshes sized to the stale {STALE_LIFETIME}s lifetime") diff --git a/bench/scenarios/cascade-ttl-units-code/hub_fresh.md b/bench/scenarios/cascade-ttl-units-code/hub_fresh.md new file mode 100644 index 0000000..89da617 --- /dev/null +++ b/bench/scenarios/cascade-ttl-units-code/hub_fresh.md @@ -0,0 +1,20 @@ +--- +summary: How long a cache entry stays fresh under the default TTL policy. +anchors: + - claim: > + TtlPolicy.lifetime_seconds returns the entry lifetime in seconds, computed from a + millisecond TTL (ttl_ms / 1000). The default policy uses DEFAULT_TTL_MS = 5000, so + lifetime_seconds() returns 5.0: a cached entry is fresh for 5 seconds, and a warmer must + refresh at least once every 5 seconds. + at: code/cache/ttl.py > TtlPolicy > lifetime_seconds + hash: 0ab45f8ba46b +refs: [] +--- + +# Cache TTL policy + +`TtlPolicy.lifetime_seconds()` returns how long a cache entry stays fresh, **in seconds**. The TTL +is stored in **milliseconds** (`DEFAULT_TTL_MS = 5000`) and converted with `ttl_ms / 1000`. + +**Entry lifetime:** with the default `DEFAULT_TTL_MS = 5000`, a freshly written entry is valid for +**5 seconds**; a cache warmer must refresh each entry at least once every 5-second lifetime. diff --git a/bench/scenarios/cascade-ttl-units-code/hub_stale.md b/bench/scenarios/cascade-ttl-units-code/hub_stale.md new file mode 100644 index 0000000..0ef3841 --- /dev/null +++ b/bench/scenarios/cascade-ttl-units-code/hub_stale.md @@ -0,0 +1,20 @@ +--- +summary: How long a cache entry stays fresh under the default TTL policy. +anchors: + - claim: > + TtlPolicy.lifetime_seconds returns the entry lifetime in seconds. The default policy uses a + 30-second TTL, so lifetime_seconds() returns 30: a cached entry is considered fresh for 30 + seconds after it is written, and a warmer must refresh at least once every 30 seconds. + at: code/cache/ttl.py > TtlPolicy > lifetime_seconds + hash: 2a59ab2d0359 +refs: [] +--- + +# Cache TTL policy + +`TtlPolicy.lifetime_seconds()` returns how long a cache entry stays fresh, **in seconds**. The +default policy is configured with a **30-second** TTL. + +**Entry lifetime:** a freshly written entry is valid for **30 seconds**; after that it is stale and +must be re-fetched. A cache warmer therefore has to refresh each entry at least once per 30-second +lifetime to prevent a miss. diff --git a/bench/scenarios/cascade-ttl-units-code/meta.toml b/bench/scenarios/cascade-ttl-units-code/meta.toml new file mode 100644 index 0000000..c518d5c --- /dev/null +++ b/bench/scenarios/cascade-ttl-units-code/meta.toml @@ -0,0 +1,22 @@ +id = "cascade-ttl-units-code" +title = "Cascade: warmer inherits a hidden cache policy's drifted lifetime (seconds vs ms)" +lang = "python" +task_type = "code" +tier = "T1" # the real lifetime lives in a dependency the agent cannot see — only its doc + +invariant = "A cache entry has a fixed lifetime; a warmer must refresh at least once per lifetime." + +# The drift: the TTL policy was refactored from seconds to milliseconds. The default moved from +# `DEFAULT_TTL_SECONDS = 30` to `DEFAULT_TTL_MS = 5000`, and `lifetime_seconds()` now divides by +# 1000 — so the true entry lifetime fell from 30s to 5s. The doc still says 30s. The agent edits +# warmer.py (visible) whose schedule sizes from that lifetime, but ttl.py is HIDDEN, so the only +# source of the lifetime is the policy's doc. A stale doc ("30 seconds") makes the warmer refresh +# 6x too rarely. +drift = "TTL units seconds -> milliseconds (true lifetime 30s -> 5s); warmer sizes from the doc, not the hidden code" + +anchor = "code/cache/ttl.py > TtlPolicy > lifetime_seconds" +edit_path = "code/warmer.py" # the visible file the agent returns + +# The drifted dependency: present in code/ so surf seals a real divergence and the grader probes the +# real policy, but withheld from the prompt — the agent knows it only by doc. +hidden_paths = ["code/cache/*.py"] diff --git a/bench/scenarios/cascade-ttl-units-code/surf_report.json b/bench/scenarios/cascade-ttl-units-code/surf_report.json new file mode 100644 index 0000000..47c4fbc --- /dev/null +++ b/bench/scenarios/cascade-ttl-units-code/surf_report.json @@ -0,0 +1,15 @@ +{ + "version": 1, + "divergences": [ + { + "hub": "hub.md", + "claim": "TtlPolicy.lifetime_seconds returns the entry lifetime in seconds. The default policy uses a 30-second TTL, so lifetime_seconds() returns 30: a cached entry is considered fresh for 30 seconds after it is written, and a warmer must refresh at least once every 30 seconds.", + "at": "code/cache/ttl.py > TtlPolicy > lifetime_seconds", + "kind": "changed", + "old_hash": "2a59ab2d0359", + "new_hash": "0ab45f8ba46b", + "new_code": "def lifetime_seconds(self) -> float:\n \"\"\"The entry lifetime, in seconds (the TTL is stored in milliseconds).\"\"\"\n return self.ttl_ms / 1000.0", + "prose": "TtlPolicy.lifetime_seconds returns the entry lifetime in seconds. The default policy uses a 30-second TTL, so lifetime_seconds() returns 30: a cached entry is considered fresh for 30 seconds after it is written, and a warmer must refresh at least once every 30 seconds." + } + ] +} diff --git a/bench/scenarios/cascade-ttl-units-code/task.md b/bench/scenarios/cascade-ttl-units-code/task.md new file mode 100644 index 0000000..b557982 --- /dev/null +++ b/bench/scenarios/cascade-ttl-units-code/task.md @@ -0,0 +1,17 @@ +We're adding cache warming to `warmer.py`. The warmer keeps a cache hot by re-fetching entries +before they expire. Entry lifetime is governed by the `TtlPolicy` in the `cache` package; that +policy's source is not in this checkout, but its documentation is included below. + +Implement `schedule_refreshes(window_seconds)` in `warmer.py`: + +- A cache entry stays fresh for the policy's **lifetime** (a whole number of seconds). +- Return how many refreshes are needed to keep the cache warm across a window of `window_seconds`: + one refresh per lifetime, rounding **up** so the whole window is covered. Return `0` when + `window_seconds` is `0`. + +Determine the policy's lifetime in seconds and size the schedule to it. + +Return the **entire** updated `code/warmer.py` file, as a single fenced block preceded by a line in +exactly this form: + +FILE: code/warmer.py diff --git a/bench/surface_bench/scenarios.py b/bench/surface_bench/scenarios.py index acca381..c8a19d6 100644 --- a/bench/surface_bench/scenarios.py +++ b/bench/surface_bench/scenarios.py @@ -35,6 +35,9 @@ class Scenario: # code/ paths (or globs) present for grading but withheld from the prompt — the hidden # dependency in a cascade scenario, which the agent knows only through its doc. hidden_paths: list[str] = field(default_factory=list) + # code scenarios: the visible file the agent returns (e.g. "code/throttle.py"). Lets the + # grader-polarization self-test (tools/validate_scenario.py) build the `FILE:` block. + edit_path: str = "" @property def grader_dir(self) -> Path: @@ -78,6 +81,7 @@ def load_scenario(path: str | Path) -> Scenario: surf_report=(root / "surf_report.json").read_text(), code=_read_code(root / "code"), hidden_paths=meta.get("hidden_paths", []), + edit_path=meta.get("edit_path", ""), ) diff --git a/bench/tests/test_scenarios.py b/bench/tests/test_scenarios.py new file mode 100644 index 0000000..3966cbf --- /dev/null +++ b/bench/tests/test_scenarios.py @@ -0,0 +1,39 @@ +"""Guard the committed scenario set: every scenario shipping reference solutions must have +correctly-polarized graders (correct -> ok & not misled; stale -> not ok & misled). Runs the live +graders via the validate_scenario tool — offline, no API.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +BENCH_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(BENCH_ROOT / "tools")) + +import validate_scenario # noqa: E402 + +from surface_bench.scenarios import load_scenario # noqa: E402 + + +def _scenarios_with_solutions() -> list[Path]: + dirs = sorted(p.parent for p in (BENCH_ROOT / "scenarios").glob("*/meta.toml")) + return [d for d in dirs if list((d / ".author").glob("solution_correct.*"))] + + +@pytest.mark.parametrize("scenario_dir", _scenarios_with_solutions(), ids=lambda d: d.name) +def test_grader_polarization(scenario_dir: Path) -> None: + fails = validate_scenario.validate(load_scenario(scenario_dir)) + assert not fails, "; ".join(fails) + + +def test_at_least_the_new_cascades_are_covered() -> None: + covered = {d.name for d in _scenarios_with_solutions()} + expected = { + "cascade-ttl-units-code", + "cascade-money-rounding-code", + "cascade-backoff-offbyone-code", + "cascade-signal-threshold-code", + } + assert expected <= covered, f"missing reference solutions for {expected - covered}" diff --git a/bench/tools/validate_scenario.py b/bench/tools/validate_scenario.py new file mode 100644 index 0000000..b890b58 --- /dev/null +++ b/bench/tools/validate_scenario.py @@ -0,0 +1,107 @@ +"""Offline grader-polarization self-test for a scenario (no API, no spend). + +`tools/author.py` seals hubs and proves the *drift* is real; it does NOT prove the *graders* +discriminate. The biggest risk across many fixtures is a mis-polarized grader — `check_correct` and +`check_misled` both passing, both failing, or swapped — which only the post-run oracle would catch, +and only after spending tokens. This tool closes that gap before any spend. + +Each scenario ships two reference solutions in `.author/`: + * solution_correct. — implements the CURRENT (T1) behaviour (or, for QA, ends in the correct + VERDICT line). Must grade ok=True, misled=False. + * solution_stale. — implements the STALE (T0) doc's value (or the misled VERDICT). Must + grade ok=False, misled=True. +We feed each through the LIVE graders (`grade_code.grade` / `grade_qa.grade`), so the self-test +exercises the exact path the real run uses. A scenario passes only if both polarities hold. + + python tools/validate_scenario.py scenarios/ + python tools/validate_scenario.py --all +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +BENCH_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(BENCH_ROOT)) + +from surface_bench import grade_code, grade_qa # noqa: E402 +from surface_bench.scenarios import Scenario, load_scenario # noqa: E402 + +_FENCE = {"python": "python", "typescript": "typescript", "tsx": "tsx", "javascript": "javascript"} + + +def _solution(scenario: Scenario, which: str) -> Path: + hits = sorted((scenario.root / ".author").glob(f"solution_{which}.*")) + if not hits: + raise SystemExit(f"{scenario.id}: missing .author/solution_{which}.* reference solution") + if len(hits) > 1: + raise SystemExit(f"{scenario.id}: multiple .author/solution_{which}.* files: {hits}") + return hits[0] + + +def _code_output(scenario: Scenario, body: str) -> str: + if not scenario.edit_path: + raise SystemExit(f"{scenario.id}: code scenario needs `edit_path` in meta.toml") + fence = _FENCE.get(scenario.lang, "") + return f"FILE: {scenario.edit_path}\n```{fence}\n{body.rstrip()}\n```\n" + + +def _grade(scenario: Scenario, which: str) -> dict: + body = _solution(scenario, which).read_text() + if scenario.task_type == "code": + return grade_code.grade(scenario, _code_output(scenario, body)) + return grade_qa.grade(scenario, body) # QA: the .txt is the agent's verbatim answer + + +def validate(scenario: Scenario) -> list[str]: + """Return a list of polarization failures (empty == passes).""" + fails: list[str] = [] + correct = _grade(scenario, "correct") + if not (correct["ok"] and not correct["misled"]): + fails.append( + f"solution_correct graded ok={correct['ok']} misled={correct['misled']} " + f"(want ok=True, misled=False) — {correct.get('detail')}" + ) + stale = _grade(scenario, "stale") + if not (not stale["ok"] and stale["misled"]): + fails.append( + f"solution_stale graded ok={stale['ok']} misled={stale['misled']} " + f"(want ok=False, misled=True) — {stale.get('detail')}" + ) + return fails + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("scenario", nargs="?", help="path to a scenario dir") + ap.add_argument("--all", action="store_true", help="validate every scenario with reference solutions") + args = ap.parse_args() + + if args.all: + dirs = sorted(p.parent for p in (BENCH_ROOT / "scenarios").glob("*/meta.toml")) + targets = [d for d in dirs if list((d / ".author").glob("solution_correct.*"))] + elif args.scenario: + targets = [Path(args.scenario).resolve()] + else: + ap.error("pass a scenario dir or --all") + + failed = 0 + for d in targets: + scenario = load_scenario(d) + fails = validate(scenario) + if fails: + failed += 1 + print(f"FAIL {scenario.id}") + for f in fails: + print(f" - {f}") + else: + print(f"ok {scenario.id}") + if failed: + sys.exit(f"\n{failed} scenario(s) failed grader polarization") + print(f"\nall {len(targets)} scenario(s) pass grader polarization") + + +if __name__ == "__main__": + main()