From ca3ad31fcdce9adca327622a8a1f95a9545eb786 Mon Sep 17 00:00:00 2001 From: Connorrmcd6 Date: Sun, 14 Jun 2026 22:04:10 +0200 Subject: [PATCH] feat(bench): 3 QA cascade scenarios (#104) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR-C of the cascade expansion: three verdict-graded QA cascades. The agent reasons about a HIDDEN dependency from its doc and ends with a VERDICT line; graded deterministically by rubric (no code execution). Each drift is detectable by surf (structural/literal, not an identifier swap). - cascade-idempotency-window-qa (T2, payments) — IdempotencyStore.is_duplicate body `key in self._processed` -> `return False` (dedup guard removed; a retried payment can double-charge). - cascade-ratelimit-burst-qa (T2, rate-limiting) — RatePolicy.admission_mode "fixed_window" -> "token_bucket" (idle clients can now burst above the rate). - cascade-session-expiry-tz-qa (T2, dates/tz) — is_expired threshold `> 7200` (2h, local+grace) -> `> 3600` (1h, UTC); a 90-min session flips valid->expired. Each: author.py seals a genuine "changed" divergence, the rubric encodes the T1 truth as [correct] and the stale claim as [misled], and validate_scenario proves polarity via .author/solution_{correct,stale}.txt. Note: all three are single-anchor. A genuine T3 "multi-claim" scenario needs author.py to seal more than the first anchor's hash (it currently seals only the first) — deferred as a small tooling follow-up. Verified offline: author.py seals all three, validate_scenario green, a mock run is error-free, uv run pytest passes (38). Co-Authored-By: Claude Opus 4.8 --- .../.author/code_t0/payments/idempotency.py | 12 +++++++++ .../.author/solution_correct.txt | 5 ++++ .../.author/solution_stale.txt | 4 +++ .../code/payments/__init__.py | 3 +++ .../code/payments/idempotency.py | 12 +++++++++ .../grader/rubric.toml | 25 +++++++++++++++++++ .../hub_fresh.md | 21 ++++++++++++++++ .../hub_stale.md | 21 ++++++++++++++++ .../cascade-idempotency-window-qa/meta.toml | 17 +++++++++++++ .../surf_report.json | 15 +++++++++++ .../cascade-idempotency-window-qa/task.md | 22 ++++++++++++++++ .../.author/code_t0/limiter/policy.py | 8 ++++++ .../.author/solution_correct.txt | 5 ++++ .../.author/solution_stale.txt | 5 ++++ .../code/limiter/__init__.py | 3 +++ .../code/limiter/policy.py | 8 ++++++ .../grader/rubric.toml | 25 +++++++++++++++++++ .../cascade-ratelimit-burst-qa/hub_fresh.md | 22 ++++++++++++++++ .../cascade-ratelimit-burst-qa/hub_stale.md | 21 ++++++++++++++++ .../cascade-ratelimit-burst-qa/meta.toml | 18 +++++++++++++ .../surf_report.json | 15 +++++++++++ .../cascade-ratelimit-burst-qa/task.md | 20 +++++++++++++++ .../.author/code_t0/auth/session.py | 6 +++++ .../.author/solution_correct.txt | 5 ++++ .../.author/solution_stale.txt | 4 +++ .../code/auth/__init__.py | 3 +++ .../code/auth/session.py | 6 +++++ .../grader/rubric.toml | 25 +++++++++++++++++++ .../cascade-session-expiry-tz-qa/hub_fresh.md | 19 ++++++++++++++ .../cascade-session-expiry-tz-qa/hub_stale.md | 19 ++++++++++++++ .../cascade-session-expiry-tz-qa/meta.toml | 18 +++++++++++++ .../surf_report.json | 15 +++++++++++ .../cascade-session-expiry-tz-qa/task.md | 19 ++++++++++++++ 33 files changed, 446 insertions(+) create mode 100644 bench/scenarios/cascade-idempotency-window-qa/.author/code_t0/payments/idempotency.py create mode 100644 bench/scenarios/cascade-idempotency-window-qa/.author/solution_correct.txt create mode 100644 bench/scenarios/cascade-idempotency-window-qa/.author/solution_stale.txt create mode 100644 bench/scenarios/cascade-idempotency-window-qa/code/payments/__init__.py create mode 100644 bench/scenarios/cascade-idempotency-window-qa/code/payments/idempotency.py create mode 100644 bench/scenarios/cascade-idempotency-window-qa/grader/rubric.toml create mode 100644 bench/scenarios/cascade-idempotency-window-qa/hub_fresh.md create mode 100644 bench/scenarios/cascade-idempotency-window-qa/hub_stale.md create mode 100644 bench/scenarios/cascade-idempotency-window-qa/meta.toml create mode 100644 bench/scenarios/cascade-idempotency-window-qa/surf_report.json create mode 100644 bench/scenarios/cascade-idempotency-window-qa/task.md create mode 100644 bench/scenarios/cascade-ratelimit-burst-qa/.author/code_t0/limiter/policy.py create mode 100644 bench/scenarios/cascade-ratelimit-burst-qa/.author/solution_correct.txt create mode 100644 bench/scenarios/cascade-ratelimit-burst-qa/.author/solution_stale.txt create mode 100644 bench/scenarios/cascade-ratelimit-burst-qa/code/limiter/__init__.py create mode 100644 bench/scenarios/cascade-ratelimit-burst-qa/code/limiter/policy.py create mode 100644 bench/scenarios/cascade-ratelimit-burst-qa/grader/rubric.toml create mode 100644 bench/scenarios/cascade-ratelimit-burst-qa/hub_fresh.md create mode 100644 bench/scenarios/cascade-ratelimit-burst-qa/hub_stale.md create mode 100644 bench/scenarios/cascade-ratelimit-burst-qa/meta.toml create mode 100644 bench/scenarios/cascade-ratelimit-burst-qa/surf_report.json create mode 100644 bench/scenarios/cascade-ratelimit-burst-qa/task.md create mode 100644 bench/scenarios/cascade-session-expiry-tz-qa/.author/code_t0/auth/session.py create mode 100644 bench/scenarios/cascade-session-expiry-tz-qa/.author/solution_correct.txt create mode 100644 bench/scenarios/cascade-session-expiry-tz-qa/.author/solution_stale.txt create mode 100644 bench/scenarios/cascade-session-expiry-tz-qa/code/auth/__init__.py create mode 100644 bench/scenarios/cascade-session-expiry-tz-qa/code/auth/session.py create mode 100644 bench/scenarios/cascade-session-expiry-tz-qa/grader/rubric.toml create mode 100644 bench/scenarios/cascade-session-expiry-tz-qa/hub_fresh.md create mode 100644 bench/scenarios/cascade-session-expiry-tz-qa/hub_stale.md create mode 100644 bench/scenarios/cascade-session-expiry-tz-qa/meta.toml create mode 100644 bench/scenarios/cascade-session-expiry-tz-qa/surf_report.json create mode 100644 bench/scenarios/cascade-session-expiry-tz-qa/task.md diff --git a/bench/scenarios/cascade-idempotency-window-qa/.author/code_t0/payments/idempotency.py b/bench/scenarios/cascade-idempotency-window-qa/.author/code_t0/payments/idempotency.py new file mode 100644 index 0000000..0762747 --- /dev/null +++ b/bench/scenarios/cascade-idempotency-window-qa/.author/code_t0/payments/idempotency.py @@ -0,0 +1,12 @@ +class IdempotencyStore: + """Tracks processed payment keys so a retried request is never charged twice.""" + + def __init__(self) -> None: + self._processed: set[str] = set() + + def record(self, key: str) -> None: + self._processed.add(key) + + def is_duplicate(self, key: str) -> bool: + """Return True if a payment with this idempotency key was already processed.""" + return key in self._processed diff --git a/bench/scenarios/cascade-idempotency-window-qa/.author/solution_correct.txt b/bench/scenarios/cascade-idempotency-window-qa/.author/solution_correct.txt new file mode 100644 index 0000000..9f35833 --- /dev/null +++ b/bench/scenarios/cascade-idempotency-window-qa/.author/solution_correct.txt @@ -0,0 +1,5 @@ +The retry is not caught. `is_duplicate` always returns False now, so the second request with key +`pay_123` is not recognized as a duplicate and the payment is processed again — the customer can be +charged twice. + +VERDICT: treated_as_duplicate=no; double_charge_possible=yes diff --git a/bench/scenarios/cascade-idempotency-window-qa/.author/solution_stale.txt b/bench/scenarios/cascade-idempotency-window-qa/.author/solution_stale.txt new file mode 100644 index 0000000..eb2b1ce --- /dev/null +++ b/bench/scenarios/cascade-idempotency-window-qa/.author/solution_stale.txt @@ -0,0 +1,4 @@ +The store recognizes the retry: `is_duplicate("pay_123")` returns True because the key is already in +the processed set, so the second charge is skipped and the customer is charged at most once. + +VERDICT: treated_as_duplicate=yes; double_charge_possible=no diff --git a/bench/scenarios/cascade-idempotency-window-qa/code/payments/__init__.py b/bench/scenarios/cascade-idempotency-window-qa/code/payments/__init__.py new file mode 100644 index 0000000..3d8961f --- /dev/null +++ b/bench/scenarios/cascade-idempotency-window-qa/code/payments/__init__.py @@ -0,0 +1,3 @@ +from .idempotency import IdempotencyStore + +__all__ = ["IdempotencyStore"] diff --git a/bench/scenarios/cascade-idempotency-window-qa/code/payments/idempotency.py b/bench/scenarios/cascade-idempotency-window-qa/code/payments/idempotency.py new file mode 100644 index 0000000..05e1f03 --- /dev/null +++ b/bench/scenarios/cascade-idempotency-window-qa/code/payments/idempotency.py @@ -0,0 +1,12 @@ +class IdempotencyStore: + """Tracks processed payment keys so a retried request is never charged twice.""" + + def __init__(self) -> None: + self._processed: set[str] = set() + + def record(self, key: str) -> None: + self._processed.add(key) + + def is_duplicate(self, key: str) -> bool: + """Return True if a payment with this idempotency key was already processed.""" + return False diff --git a/bench/scenarios/cascade-idempotency-window-qa/grader/rubric.toml b/bench/scenarios/cascade-idempotency-window-qa/grader/rubric.toml new file mode 100644 index 0000000..94e79b7 --- /dev/null +++ b/bench/scenarios/cascade-idempotency-window-qa/grader/rubric.toml @@ -0,0 +1,25 @@ +# QA grader for cascade-idempotency-window. +# +# The agent ends its answer with: +# VERDICT: treated_as_duplicate=; double_charge_possible= +# +# - correct (current/T1 code): treated_as_duplicate=no AND double_charge_possible=yes +# (is_duplicate now always returns False, so the retry is not caught -> double charge) +# - misled (stale/T0 claim): treated_as_duplicate=yes OR double_charge_possible=no +# (parroting the doc's "recognizes duplicates, charged at most once") + +type = "verdict" + +[fields.treated_as_duplicate] +pattern = "treated_as_duplicate\\s*=\\s*(yes|no)" + +[fields.double_charge_possible] +pattern = "double_charge_possible\\s*=\\s*(yes|no)" + +[correct] +treated_as_duplicate = "no" +double_charge_possible = "yes" + +[misled] +treated_as_duplicate = "yes" +double_charge_possible = "no" diff --git a/bench/scenarios/cascade-idempotency-window-qa/hub_fresh.md b/bench/scenarios/cascade-idempotency-window-qa/hub_fresh.md new file mode 100644 index 0000000..3cb696e --- /dev/null +++ b/bench/scenarios/cascade-idempotency-window-qa/hub_fresh.md @@ -0,0 +1,21 @@ +--- +summary: How the payments idempotency store recognizes a retried request. +anchors: + - claim: > + IdempotencyStore.is_duplicate(key) always returns False — the duplicate check was removed and + the processed-key set is no longer consulted (the body is just `return False`). A retry with + the same key is therefore NOT recognized as a duplicate, so the payment is processed again and + a customer can be charged twice. + at: code/payments/idempotency.py > IdempotencyStore > is_duplicate + hash: c3faa17370af +refs: [] +--- + +# Payment idempotency + +`IdempotencyStore.is_duplicate(key)` no longer guards against double-charging. Its body is now just +`return False`: the processed-key set is never consulted, so **every** request — including a retry +with an already-seen key — is reported as **not** a duplicate. + +So when a client **retries** a request with the same key, the store does **not** recognize it, the +payment service charges again, and the customer can be **charged twice**. diff --git a/bench/scenarios/cascade-idempotency-window-qa/hub_stale.md b/bench/scenarios/cascade-idempotency-window-qa/hub_stale.md new file mode 100644 index 0000000..50223e1 --- /dev/null +++ b/bench/scenarios/cascade-idempotency-window-qa/hub_stale.md @@ -0,0 +1,21 @@ +--- +summary: How the payments idempotency store recognizes a retried request. +anchors: + - claim: > + IdempotencyStore.is_duplicate(key) returns True when a payment with that idempotency key was + already processed — it checks the key against the processed-key set (key in self._processed). + A retry with the same key is therefore recognized as a duplicate and the charge is skipped, so + a payment is never charged twice. + at: code/payments/idempotency.py > IdempotencyStore > is_duplicate + hash: f01d54cf0aef +refs: [] +--- + +# Payment idempotency + +`IdempotencyStore.is_duplicate(key)` guards against double-charging. Every processed payment records +its idempotency key; `is_duplicate` returns **True** when the key is already in the processed set +(`key in self._processed`). + +So when a client **retries** a request with the same key, the store recognizes it as a **duplicate** +and the payment service skips the second charge — a key is charged **at most once**. diff --git a/bench/scenarios/cascade-idempotency-window-qa/meta.toml b/bench/scenarios/cascade-idempotency-window-qa/meta.toml new file mode 100644 index 0000000..0fd6365 --- /dev/null +++ b/bench/scenarios/cascade-idempotency-window-qa/meta.toml @@ -0,0 +1,17 @@ +id = "cascade-idempotency-window-qa" +title = "Payments: idempotency duplicate-check silently became a no-op" +lang = "python" +task_type = "qa" +tier = "T2" # premise: the dedup invariant is load-bearing for a double-charge conclusion + +invariant = "A retried payment with the same idempotency key must be recognized as a duplicate and not charged twice." + +# The drift: IdempotencyStore.is_duplicate stopped consulting the processed-key set — its body +# changed from `return key in self._processed` to `return False`, so it now reports every request as +# new. The duplicate guard is gone. The store's source is HIDDEN, so an agent knows the behaviour +# only from its doc; a stale doc ("recognizes duplicates") hides a live double-charge risk. +drift = "is_duplicate body `key in self._processed` -> `return False` (duplicate guard removed)" + +anchor = "code/payments/idempotency.py > IdempotencyStore > is_duplicate" + +hidden_paths = ["code/payments/*.py"] diff --git a/bench/scenarios/cascade-idempotency-window-qa/surf_report.json b/bench/scenarios/cascade-idempotency-window-qa/surf_report.json new file mode 100644 index 0000000..bdd7350 --- /dev/null +++ b/bench/scenarios/cascade-idempotency-window-qa/surf_report.json @@ -0,0 +1,15 @@ +{ + "version": 1, + "divergences": [ + { + "hub": "hub.md", + "claim": "IdempotencyStore.is_duplicate(key) returns True when a payment with that idempotency key was already processed \u2014 it checks the key against the processed-key set (key in self._processed). A retry with the same key is therefore recognized as a duplicate and the charge is skipped, so a payment is never charged twice.", + "at": "code/payments/idempotency.py > IdempotencyStore > is_duplicate", + "kind": "changed", + "old_hash": "f01d54cf0aef", + "new_hash": "c3faa17370af", + "new_code": "def is_duplicate(self, key: str) -> bool:\n \"\"\"Return True if a payment with this idempotency key was already processed.\"\"\"\n return False", + "prose": "IdempotencyStore.is_duplicate(key) returns True when a payment with that idempotency key was already processed \u2014 it checks the key against the processed-key set (key in self._processed). A retry with the same key is therefore recognized as a duplicate and the charge is skipped, so a payment is never charged twice." + } + ] +} diff --git a/bench/scenarios/cascade-idempotency-window-qa/task.md b/bench/scenarios/cascade-idempotency-window-qa/task.md new file mode 100644 index 0000000..cbb092b --- /dev/null +++ b/bench/scenarios/cascade-idempotency-window-qa/task.md @@ -0,0 +1,22 @@ +A teammate is reviewing the payments flow and asks you to confirm a specific behaviour of the +idempotency layer. The payment service calls `IdempotencyStore.is_duplicate(key)` before charging: +if it returns True the charge is skipped. That store's source is not in this checkout; its +documentation is included below. + +Scenario: + +- A client submits a payment with idempotency key `pay_123`. It succeeds and the key is recorded. +- The client's connection drops before it sees the response, so it **retries the exact same request** + with the same key `pay_123`. + +Based strictly on the documented behaviour of `is_duplicate`: + +1. When the retry is checked with `is_duplicate("pay_123")`, is it **treated as a duplicate** (and + the second charge skipped)? +2. Could this retry result in the customer being **charged twice**? + +Give a direct answer to each part, then one sentence of justification. + +Finally, end your response with a single line in exactly this format (no extra text on it): + +`VERDICT: treated_as_duplicate=; double_charge_possible=` diff --git a/bench/scenarios/cascade-ratelimit-burst-qa/.author/code_t0/limiter/policy.py b/bench/scenarios/cascade-ratelimit-burst-qa/.author/code_t0/limiter/policy.py new file mode 100644 index 0000000..332dba2 --- /dev/null +++ b/bench/scenarios/cascade-ratelimit-burst-qa/.author/code_t0/limiter/policy.py @@ -0,0 +1,8 @@ +class RatePolicy: + """The rate-limiting policy for the public API.""" + + PER_MINUTE_LIMIT = 100 + + def admission_mode(self) -> str: + """Which admission algorithm the limiter uses to decide whether to admit a request.""" + return "fixed_window" diff --git a/bench/scenarios/cascade-ratelimit-burst-qa/.author/solution_correct.txt b/bench/scenarios/cascade-ratelimit-burst-qa/.author/solution_correct.txt new file mode 100644 index 0000000..c3d609b --- /dev/null +++ b/bench/scenarios/cascade-ratelimit-burst-qa/.author/solution_correct.txt @@ -0,0 +1,5 @@ +The limiter uses a token-bucket admission mode. Because an idle client accrues tokens up to the +bucket capacity, it can spend them in a short burst that briefly exceeds the per-minute rate, so the +burst is admitted (only the long-run average is capped). + +VERDICT: limiter_mode=token_bucket; burst_allowed=yes diff --git a/bench/scenarios/cascade-ratelimit-burst-qa/.author/solution_stale.txt b/bench/scenarios/cascade-ratelimit-burst-qa/.author/solution_stale.txt new file mode 100644 index 0000000..5fe86c6 --- /dev/null +++ b/bench/scenarios/cascade-ratelimit-burst-qa/.author/solution_stale.txt @@ -0,0 +1,5 @@ +The limiter uses a fixed-window admission mode. It counts requests per 60-second window and rejects +anything beyond the per-minute limit, with no token accrual, so the client's burst cannot exceed the +rate — the excess requests are rejected. + +VERDICT: limiter_mode=fixed_window; burst_allowed=no diff --git a/bench/scenarios/cascade-ratelimit-burst-qa/code/limiter/__init__.py b/bench/scenarios/cascade-ratelimit-burst-qa/code/limiter/__init__.py new file mode 100644 index 0000000..e19d87d --- /dev/null +++ b/bench/scenarios/cascade-ratelimit-burst-qa/code/limiter/__init__.py @@ -0,0 +1,3 @@ +from .policy import RatePolicy + +__all__ = ["RatePolicy"] diff --git a/bench/scenarios/cascade-ratelimit-burst-qa/code/limiter/policy.py b/bench/scenarios/cascade-ratelimit-burst-qa/code/limiter/policy.py new file mode 100644 index 0000000..9ddb72b --- /dev/null +++ b/bench/scenarios/cascade-ratelimit-burst-qa/code/limiter/policy.py @@ -0,0 +1,8 @@ +class RatePolicy: + """The rate-limiting policy for the public API.""" + + PER_MINUTE_LIMIT = 100 + + def admission_mode(self) -> str: + """Which admission algorithm the limiter uses to decide whether to admit a request.""" + return "token_bucket" diff --git a/bench/scenarios/cascade-ratelimit-burst-qa/grader/rubric.toml b/bench/scenarios/cascade-ratelimit-burst-qa/grader/rubric.toml new file mode 100644 index 0000000..939f0ef --- /dev/null +++ b/bench/scenarios/cascade-ratelimit-burst-qa/grader/rubric.toml @@ -0,0 +1,25 @@ +# QA grader for cascade-ratelimit-burst. +# +# The agent ends its answer with: +# VERDICT: limiter_mode=; burst_allowed= +# +# - correct (current/T1 code): limiter_mode=token_bucket AND burst_allowed=yes +# (token bucket lets an idle client spend accrued tokens in a burst) +# - misled (stale/T0 claim): limiter_mode=fixed_window OR burst_allowed=no +# (parroting the doc's fixed-window, no-burst model) + +type = "verdict" + +[fields.limiter_mode] +pattern = "limiter_mode\\s*=\\s*(fixed_window|token_bucket)" + +[fields.burst_allowed] +pattern = "burst_allowed\\s*=\\s*(yes|no)" + +[correct] +limiter_mode = "token_bucket" +burst_allowed = "yes" + +[misled] +limiter_mode = "fixed_window" +burst_allowed = "no" diff --git a/bench/scenarios/cascade-ratelimit-burst-qa/hub_fresh.md b/bench/scenarios/cascade-ratelimit-burst-qa/hub_fresh.md new file mode 100644 index 0000000..a8bac4d --- /dev/null +++ b/bench/scenarios/cascade-ratelimit-burst-qa/hub_fresh.md @@ -0,0 +1,22 @@ +--- +summary: How the public API limiter admits requests and whether bursts can exceed the rate. +anchors: + - claim: > + RatePolicy.admission_mode returns "token_bucket". The limiter refills tokens continuously up + to a bucket capacity; an idle client accrues tokens and can spend them in a short burst, so it + CAN briefly exceed the per-minute rate (up to the bucket size) even though the long-run + average stays capped at PER_MINUTE_LIMIT. + at: code/limiter/policy.py > RatePolicy > admission_mode + hash: 90c822614537 +refs: [] +--- + +# Rate limiting + +The public API limiter's admission algorithm is reported by `RatePolicy.admission_mode()`, which +returns **`"token_bucket"`**. + +A **token bucket** refills tokens continuously up to a bucket capacity. A client that has been idle +accumulates tokens and can spend them all at once, so it **can** briefly send a burst that exceeds +the per-minute rate (up to the bucket size) — only the long-run average is held to +`PER_MINUTE_LIMIT`. diff --git a/bench/scenarios/cascade-ratelimit-burst-qa/hub_stale.md b/bench/scenarios/cascade-ratelimit-burst-qa/hub_stale.md new file mode 100644 index 0000000..6311fb6 --- /dev/null +++ b/bench/scenarios/cascade-ratelimit-burst-qa/hub_stale.md @@ -0,0 +1,21 @@ +--- +summary: How the public API limiter admits requests and whether bursts can exceed the rate. +anchors: + - claim: > + RatePolicy.admission_mode returns "fixed_window". The limiter counts requests in fixed + 60-second windows and rejects anything beyond PER_MINUTE_LIMIT in the current window. There is + no token accrual, so a client cannot exceed the per-minute limit even briefly — bursts above + the limit are not possible. + at: code/limiter/policy.py > RatePolicy > admission_mode + hash: b3055e5a988e +refs: [] +--- + +# Rate limiting + +The public API limiter's admission algorithm is reported by `RatePolicy.admission_mode()`, which +returns **`"fixed_window"`**. + +A **fixed window** counts requests in discrete 60-second windows and rejects any request beyond +`PER_MINUTE_LIMIT` in the current window. There is no carry-over or token accrual, so a client +**cannot** briefly exceed the per-minute rate — a burst above the limit is simply rejected. diff --git a/bench/scenarios/cascade-ratelimit-burst-qa/meta.toml b/bench/scenarios/cascade-ratelimit-burst-qa/meta.toml new file mode 100644 index 0000000..fbb8ab3 --- /dev/null +++ b/bench/scenarios/cascade-ratelimit-burst-qa/meta.toml @@ -0,0 +1,18 @@ +id = "cascade-ratelimit-burst-qa" +title = "Rate limiting: admission mode switched from fixed-window to token-bucket (bursts now possible)" +lang = "python" +task_type = "qa" +tier = "T2" # premise: the admission mode is load-bearing for whether bursts can exceed the limit + +invariant = "Whether a client can briefly exceed the per-minute rate depends on the limiter's admission mode." + +# The drift: RatePolicy.admission_mode changed its returned mode string from "fixed_window" to +# "token_bucket". A fixed window strictly caps each 60s window (no bursts); a token bucket lets an +# idle client accrue tokens and spend them in a short burst above the rate. The policy's source is +# HIDDEN, so an agent knows the mode only from its doc; a stale doc ("fixed_window") wrongly implies +# bursts are impossible. +drift = "admission_mode return string `fixed_window` -> `token_bucket` (bursts above the limit now possible)" + +anchor = "code/limiter/policy.py > RatePolicy > admission_mode" + +hidden_paths = ["code/limiter/*.py"] diff --git a/bench/scenarios/cascade-ratelimit-burst-qa/surf_report.json b/bench/scenarios/cascade-ratelimit-burst-qa/surf_report.json new file mode 100644 index 0000000..9af5e9f --- /dev/null +++ b/bench/scenarios/cascade-ratelimit-burst-qa/surf_report.json @@ -0,0 +1,15 @@ +{ + "version": 1, + "divergences": [ + { + "hub": "hub.md", + "claim": "RatePolicy.admission_mode returns \"fixed_window\". The limiter counts requests in fixed 60-second windows and rejects anything beyond PER_MINUTE_LIMIT in the current window. There is no token accrual, so a client cannot exceed the per-minute limit even briefly \u2014 bursts above the limit are not possible.", + "at": "code/limiter/policy.py > RatePolicy > admission_mode", + "kind": "changed", + "old_hash": "b3055e5a988e", + "new_hash": "90c822614537", + "new_code": "def admission_mode(self) -> str:\n \"\"\"Which admission algorithm the limiter uses to decide whether to admit a request.\"\"\"\n return \"token_bucket\"", + "prose": "RatePolicy.admission_mode returns \"fixed_window\". The limiter counts requests in fixed 60-second windows and rejects anything beyond PER_MINUTE_LIMIT in the current window. There is no token accrual, so a client cannot exceed the per-minute limit even briefly \u2014 bursts above the limit are not possible." + } + ] +} diff --git a/bench/scenarios/cascade-ratelimit-burst-qa/task.md b/bench/scenarios/cascade-ratelimit-burst-qa/task.md new file mode 100644 index 0000000..9503b33 --- /dev/null +++ b/bench/scenarios/cascade-ratelimit-burst-qa/task.md @@ -0,0 +1,20 @@ +A teammate is sizing a client's request pattern against the public API's rate limiter and asks you +to confirm its behaviour. Admission decisions are governed by `RatePolicy` in the `limiter` module; +that module's source is not in this checkout, but its documentation is included below. + +Scenario: + +- A client has been idle for a while and then sends a quick burst of requests — more requests in a + few seconds than the per-minute limit would allow on average. + +Based strictly on the documented behaviour of `RatePolicy`: + +1. Which **admission mode** does the limiter use? +2. Can the idle client's short **burst** briefly exceed the per-minute rate, or will the excess + requests be rejected? + +Give a direct answer to each part, then one sentence of justification. + +Finally, end your response with a single line in exactly this format (no extra text on it): + +`VERDICT: limiter_mode=; burst_allowed=` diff --git a/bench/scenarios/cascade-session-expiry-tz-qa/.author/code_t0/auth/session.py b/bench/scenarios/cascade-session-expiry-tz-qa/.author/code_t0/auth/session.py new file mode 100644 index 0000000..03e59ee --- /dev/null +++ b/bench/scenarios/cascade-session-expiry-tz-qa/.author/code_t0/auth/session.py @@ -0,0 +1,6 @@ +def is_expired(age_seconds: int) -> bool: + """Return True if a session of this age (in seconds) has expired. + + Expiry uses the base session lifetime plus a one-hour timezone-offset grace, in local time. + """ + return age_seconds > 7200 diff --git a/bench/scenarios/cascade-session-expiry-tz-qa/.author/solution_correct.txt b/bench/scenarios/cascade-session-expiry-tz-qa/.author/solution_correct.txt new file mode 100644 index 0000000..57d1334 --- /dev/null +++ b/bench/scenarios/cascade-session-expiry-tz-qa/.author/solution_correct.txt @@ -0,0 +1,5 @@ +The effective expiry window is 60 minutes: the timezone grace was removed and expiry is measured in +UTC against the 1-hour base lifetime. A 90-minute-old session is past that window, so it has expired +and is no longer valid. + +VERDICT: expiry_minutes=60; valid_at_90min=no diff --git a/bench/scenarios/cascade-session-expiry-tz-qa/.author/solution_stale.txt b/bench/scenarios/cascade-session-expiry-tz-qa/.author/solution_stale.txt new file mode 100644 index 0000000..948c521 --- /dev/null +++ b/bench/scenarios/cascade-session-expiry-tz-qa/.author/solution_stale.txt @@ -0,0 +1,4 @@ +The effective expiry window is 120 minutes — the 1-hour base lifetime plus a 1-hour timezone grace. +A 90-minute-old session is under that 2-hour window, so it is still valid. + +VERDICT: expiry_minutes=120; valid_at_90min=yes diff --git a/bench/scenarios/cascade-session-expiry-tz-qa/code/auth/__init__.py b/bench/scenarios/cascade-session-expiry-tz-qa/code/auth/__init__.py new file mode 100644 index 0000000..d9b65d8 --- /dev/null +++ b/bench/scenarios/cascade-session-expiry-tz-qa/code/auth/__init__.py @@ -0,0 +1,3 @@ +from .session import is_expired + +__all__ = ["is_expired"] diff --git a/bench/scenarios/cascade-session-expiry-tz-qa/code/auth/session.py b/bench/scenarios/cascade-session-expiry-tz-qa/code/auth/session.py new file mode 100644 index 0000000..ab79c9d --- /dev/null +++ b/bench/scenarios/cascade-session-expiry-tz-qa/code/auth/session.py @@ -0,0 +1,6 @@ +def is_expired(age_seconds: int) -> bool: + """Return True if a session of this age (in seconds) has expired. + + Expiry is measured in UTC against the base session lifetime. + """ + return age_seconds > 3600 diff --git a/bench/scenarios/cascade-session-expiry-tz-qa/grader/rubric.toml b/bench/scenarios/cascade-session-expiry-tz-qa/grader/rubric.toml new file mode 100644 index 0000000..3262fc0 --- /dev/null +++ b/bench/scenarios/cascade-session-expiry-tz-qa/grader/rubric.toml @@ -0,0 +1,25 @@ +# QA grader for cascade-session-expiry-tz. +# +# The agent ends its answer with: +# VERDICT: expiry_minutes=; valid_at_90min= +# +# - correct (current/T1 code): expiry_minutes=60 AND valid_at_90min=no +# (UTC, grace removed: a 90-minute-old session is past the 60-minute window) +# - misled (stale/T0 claim): expiry_minutes=120 OR valid_at_90min=yes +# (parroting the doc's 2-hour window with the timezone grace) + +type = "verdict" + +[fields.expiry_minutes] +pattern = "expiry_minutes\\s*=\\s*(\\d+)" + +[fields.valid_at_90min] +pattern = "valid_at_90min\\s*=\\s*(yes|no)" + +[correct] +expiry_minutes = "60" +valid_at_90min = "no" + +[misled] +expiry_minutes = "120" +valid_at_90min = "yes" diff --git a/bench/scenarios/cascade-session-expiry-tz-qa/hub_fresh.md b/bench/scenarios/cascade-session-expiry-tz-qa/hub_fresh.md new file mode 100644 index 0000000..a7e662c --- /dev/null +++ b/bench/scenarios/cascade-session-expiry-tz-qa/hub_fresh.md @@ -0,0 +1,19 @@ +--- +summary: How long a session stays valid before is_expired reports it expired. +anchors: + - claim: > + is_expired(age_seconds) returns True once a session is older than 3600 seconds — 60 minutes + (1 hour). The timezone-offset grace was removed and expiry is now measured in UTC, so the + effective window is just the 1-hour base lifetime. A session over 1 hour old is expired. + at: code/auth/session.py > is_expired + hash: 5a7ca998bba9 +refs: [] +--- + +# Session expiry + +`is_expired(age_seconds)` reports whether a session has expired. A session expires once its age +exceeds **3600 seconds — 60 minutes (1 hour)**. + +The previous 1-hour timezone-offset grace was **removed**, and expiry is now measured in **UTC**, so +the effective window is just the **1-hour** base lifetime. A session older than 1 hour is expired. diff --git a/bench/scenarios/cascade-session-expiry-tz-qa/hub_stale.md b/bench/scenarios/cascade-session-expiry-tz-qa/hub_stale.md new file mode 100644 index 0000000..c5102cf --- /dev/null +++ b/bench/scenarios/cascade-session-expiry-tz-qa/hub_stale.md @@ -0,0 +1,19 @@ +--- +summary: How long a session stays valid before is_expired reports it expired. +anchors: + - claim: > + is_expired(age_seconds) returns True once a session is older than 7200 seconds — 120 minutes + (2 hours). The window is the 1-hour base lifetime plus a 1-hour timezone-offset grace, measured + in local time. A session under 2 hours old is still valid. + at: code/auth/session.py > is_expired + hash: 44c1968c2e04 +refs: [] +--- + +# Session expiry + +`is_expired(age_seconds)` reports whether a session has expired. A session expires once its age +exceeds **7200 seconds — 120 minutes (2 hours)**. + +That window is the **1-hour base lifetime plus a 1-hour timezone-offset grace**, measured in local +time. So a session is valid until it is **2 hours** old; anything younger than that is still valid. diff --git a/bench/scenarios/cascade-session-expiry-tz-qa/meta.toml b/bench/scenarios/cascade-session-expiry-tz-qa/meta.toml new file mode 100644 index 0000000..93086bc --- /dev/null +++ b/bench/scenarios/cascade-session-expiry-tz-qa/meta.toml @@ -0,0 +1,18 @@ +id = "cascade-session-expiry-tz-qa" +title = "Auth: session expiry lost its timezone grace (effective window 2h -> 1h)" +lang = "python" +task_type = "qa" +tier = "T2" # premise: the effective expiry window is load-bearing for a validity conclusion + +invariant = "A session is valid until its age exceeds the effective expiry window." + +# The drift: is_expired's threshold dropped from 7200s to 3600s. The old 2-hour window was the +# 1-hour base lifetime plus a 1-hour timezone-offset grace measured in local time; the grace was +# removed and expiry is now measured in UTC, so the effective window is just 1 hour. The function's +# source is HIDDEN, so an agent knows the window only from its doc; a stale doc ("2 hours") wrongly +# treats a 90-minute-old session as still valid. +drift = "is_expired threshold `> 7200` (2h, local + grace) -> `> 3600` (1h, UTC)" + +anchor = "code/auth/session.py > is_expired" + +hidden_paths = ["code/auth/*.py"] diff --git a/bench/scenarios/cascade-session-expiry-tz-qa/surf_report.json b/bench/scenarios/cascade-session-expiry-tz-qa/surf_report.json new file mode 100644 index 0000000..f5464b6 --- /dev/null +++ b/bench/scenarios/cascade-session-expiry-tz-qa/surf_report.json @@ -0,0 +1,15 @@ +{ + "version": 1, + "divergences": [ + { + "hub": "hub.md", + "claim": "is_expired(age_seconds) returns True once a session is older than 7200 seconds \u2014 120 minutes (2 hours). The window is the 1-hour base lifetime plus a 1-hour timezone-offset grace, measured in local time. A session under 2 hours old is still valid.", + "at": "code/auth/session.py > is_expired", + "kind": "changed", + "old_hash": "44c1968c2e04", + "new_hash": "5a7ca998bba9", + "new_code": "def is_expired(age_seconds: int) -> bool:\n \"\"\"Return True if a session of this age (in seconds) has expired.\n\n Expiry is measured in UTC against the base session lifetime.\n \"\"\"\n return age_seconds > 3600", + "prose": "is_expired(age_seconds) returns True once a session is older than 7200 seconds \u2014 120 minutes (2 hours). The window is the 1-hour base lifetime plus a 1-hour timezone-offset grace, measured in local time. A session under 2 hours old is still valid." + } + ] +} diff --git a/bench/scenarios/cascade-session-expiry-tz-qa/task.md b/bench/scenarios/cascade-session-expiry-tz-qa/task.md new file mode 100644 index 0000000..81b063f --- /dev/null +++ b/bench/scenarios/cascade-session-expiry-tz-qa/task.md @@ -0,0 +1,19 @@ +A teammate is debugging why some users get logged out sooner than expected and asks you to confirm +the session expiry behaviour. Whether a session has expired is decided by `is_expired(age_seconds)` +in the `auth` module; that module's source is not in this checkout, but its documentation is +included below. + +Scenario: + +- A user's session is currently **90 minutes** old (5400 seconds) and has not been refreshed. + +Based strictly on the documented behaviour of `is_expired`: + +1. What is the **effective expiry window**, in minutes? +2. Is this 90-minute-old session **still valid**, or has it expired? + +Give a direct answer to each part, then one sentence of justification. + +Finally, end your response with a single line in exactly this format (no extra text on it): + +`VERDICT: expiry_minutes=; valid_at_90min=`