feat(bench): 3 QA cascade scenarios (#104)

bench/scenarios/cascade-idempotency-window-qa/.author/code_t0/payments/idempotency.py

@@ -0,0 +1,12 @@
+class IdempotencyStore:
+    """Tracks processed payment keys so a retried request is never charged twice."""
+
+    def __init__(self) -> None:
+        self._processed: set[str] = set()
+
+    def record(self, key: str) -> None:
+        self._processed.add(key)
+
+    def is_duplicate(self, key: str) -> bool:
+        """Return True if a payment with this idempotency key was already processed."""
+        return key in self._processed

bench/scenarios/cascade-idempotency-window-qa/.author/solution_correct.txt

@@ -0,0 +1,5 @@
+The retry is not caught. `is_duplicate` always returns False now, so the second request with key
+`pay_123` is not recognized as a duplicate and the payment is processed again — the customer can be
+charged twice.
+
+VERDICT: treated_as_duplicate=no; double_charge_possible=yes

bench/scenarios/cascade-idempotency-window-qa/.author/solution_stale.txt

@@ -0,0 +1,4 @@
+The store recognizes the retry: `is_duplicate("pay_123")` returns True because the key is already in
+the processed set, so the second charge is skipped and the customer is charged at most once.
+
+VERDICT: treated_as_duplicate=yes; double_charge_possible=no

bench/scenarios/cascade-idempotency-window-qa/code/payments/__init__.py

@@ -0,0 +1,3 @@
+from .idempotency import IdempotencyStore
+
+__all__ = ["IdempotencyStore"]

bench/scenarios/cascade-idempotency-window-qa/code/payments/idempotency.py

@@ -0,0 +1,12 @@
+class IdempotencyStore:
+    """Tracks processed payment keys so a retried request is never charged twice."""
+
+    def __init__(self) -> None:
+        self._processed: set[str] = set()
+
+    def record(self, key: str) -> None:
+        self._processed.add(key)
+
+    def is_duplicate(self, key: str) -> bool:
+        """Return True if a payment with this idempotency key was already processed."""
+        return False

bench/scenarios/cascade-idempotency-window-qa/grader/rubric.toml

@@ -0,0 +1,25 @@
+# QA grader for cascade-idempotency-window.
+#
+# The agent ends its answer with:
+#   VERDICT: treated_as_duplicate=<yes|no>; double_charge_possible=<yes|no>
+#
+#   - correct (current/T1 code): treated_as_duplicate=no AND double_charge_possible=yes
+#       (is_duplicate now always returns False, so the retry is not caught -> double charge)
+#   - misled  (stale/T0 claim):  treated_as_duplicate=yes OR double_charge_possible=no
+#       (parroting the doc's "recognizes duplicates, charged at most once")
+
+type = "verdict"
+
+[fields.treated_as_duplicate]
+pattern = "treated_as_duplicate\\s*=\\s*(yes|no)"
+
+[fields.double_charge_possible]
+pattern = "double_charge_possible\\s*=\\s*(yes|no)"
+
+[correct]
+treated_as_duplicate = "no"
+double_charge_possible = "yes"
+
+[misled]
+treated_as_duplicate = "yes"
+double_charge_possible = "no"

bench/scenarios/cascade-idempotency-window-qa/hub_fresh.md

@@ -0,0 +1,21 @@
+---
+summary: How the payments idempotency store recognizes a retried request.
+anchors:
+  - claim: >
+      IdempotencyStore.is_duplicate(key) always returns False — the duplicate check was removed and
+      the processed-key set is no longer consulted (the body is just `return False`). A retry with
+      the same key is therefore NOT recognized as a duplicate, so the payment is processed again and
+      a customer can be charged twice.
+    at: code/payments/idempotency.py > IdempotencyStore > is_duplicate
+    hash: c3faa17370af
+refs: []
+---
+
+# Payment idempotency
+
+`IdempotencyStore.is_duplicate(key)` no longer guards against double-charging. Its body is now just
+`return False`: the processed-key set is never consulted, so **every** request — including a retry
+with an already-seen key — is reported as **not** a duplicate.
+
+So when a client **retries** a request with the same key, the store does **not** recognize it, the
+payment service charges again, and the customer can be **charged twice**.

bench/scenarios/cascade-idempotency-window-qa/hub_stale.md

@@ -0,0 +1,21 @@
+---
+summary: How the payments idempotency store recognizes a retried request.
+anchors:
+  - claim: >
+      IdempotencyStore.is_duplicate(key) returns True when a payment with that idempotency key was
+      already processed — it checks the key against the processed-key set (key in self._processed).
+      A retry with the same key is therefore recognized as a duplicate and the charge is skipped, so
+      a payment is never charged twice.
+    at: code/payments/idempotency.py > IdempotencyStore > is_duplicate
+    hash: f01d54cf0aef
+refs: []
+---
+
+# Payment idempotency
+
+`IdempotencyStore.is_duplicate(key)` guards against double-charging. Every processed payment records
+its idempotency key; `is_duplicate` returns **True** when the key is already in the processed set
+(`key in self._processed`).
+
+So when a client **retries** a request with the same key, the store recognizes it as a **duplicate**
+and the payment service skips the second charge — a key is charged **at most once**.

bench/scenarios/cascade-idempotency-window-qa/meta.toml

@@ -0,0 +1,17 @@
+id = "cascade-idempotency-window-qa"
+title = "Payments: idempotency duplicate-check silently became a no-op"
+lang = "python"
+task_type = "qa"
+tier = "T2"  # premise: the dedup invariant is load-bearing for a double-charge conclusion
+
+invariant = "A retried payment with the same idempotency key must be recognized as a duplicate and not charged twice."
+
+# The drift: IdempotencyStore.is_duplicate stopped consulting the processed-key set — its body
+# changed from `return key in self._processed` to `return False`, so it now reports every request as
+# new. The duplicate guard is gone. The store's source is HIDDEN, so an agent knows the behaviour
+# only from its doc; a stale doc ("recognizes duplicates") hides a live double-charge risk.
+drift = "is_duplicate body `key in self._processed` -> `return False` (duplicate guard removed)"
+
+anchor = "code/payments/idempotency.py > IdempotencyStore > is_duplicate"
+
+hidden_paths = ["code/payments/*.py"]

bench/scenarios/cascade-idempotency-window-qa/surf_report.json

@@ -0,0 +1,15 @@
+{
+  "version": 1,
+  "divergences": [
+    {
+      "hub": "hub.md",
+      "claim": "IdempotencyStore.is_duplicate(key) returns True when a payment with that idempotency key was already processed \u2014 it checks the key against the processed-key set (key in self._processed). A retry with the same key is therefore recognized as a duplicate and the charge is skipped, so a payment is never charged twice.",
+      "at": "code/payments/idempotency.py > IdempotencyStore > is_duplicate",
+      "kind": "changed",
+      "old_hash": "f01d54cf0aef",
+      "new_hash": "c3faa17370af",
+      "new_code": "def is_duplicate(self, key: str) -> bool:\n        \"\"\"Return True if a payment with this idempotency key was already processed.\"\"\"\n        return False",
+      "prose": "IdempotencyStore.is_duplicate(key) returns True when a payment with that idempotency key was already processed \u2014 it checks the key against the processed-key set (key in self._processed). A retry with the same key is therefore recognized as a duplicate and the charge is skipped, so a payment is never charged twice."
+    }
+  ]
+}

bench/scenarios/cascade-idempotency-window-qa/task.md

@@ -0,0 +1,22 @@
+A teammate is reviewing the payments flow and asks you to confirm a specific behaviour of the
+idempotency layer. The payment service calls `IdempotencyStore.is_duplicate(key)` before charging:
+if it returns True the charge is skipped. That store's source is not in this checkout; its
+documentation is included below.
+
+Scenario:
+
+- A client submits a payment with idempotency key `pay_123`. It succeeds and the key is recorded.
+- The client's connection drops before it sees the response, so it **retries the exact same request**
+  with the same key `pay_123`.
+
+Based strictly on the documented behaviour of `is_duplicate`:
+
+1. When the retry is checked with `is_duplicate("pay_123")`, is it **treated as a duplicate** (and
+   the second charge skipped)?
+2. Could this retry result in the customer being **charged twice**?
+
+Give a direct answer to each part, then one sentence of justification.
+
+Finally, end your response with a single line in exactly this format (no extra text on it):
+
+`VERDICT: treated_as_duplicate=<yes|no>; double_charge_possible=<yes|no>`

bench/scenarios/cascade-ratelimit-burst-qa/.author/code_t0/limiter/policy.py

@@ -0,0 +1,8 @@
+class RatePolicy:
+    """The rate-limiting policy for the public API."""
+
+    PER_MINUTE_LIMIT = 100
+
+    def admission_mode(self) -> str:
+        """Which admission algorithm the limiter uses to decide whether to admit a request."""
+        return "fixed_window"

bench/scenarios/cascade-ratelimit-burst-qa/.author/solution_correct.txt

@@ -0,0 +1,5 @@
+The limiter uses a token-bucket admission mode. Because an idle client accrues tokens up to the
+bucket capacity, it can spend them in a short burst that briefly exceeds the per-minute rate, so the
+burst is admitted (only the long-run average is capped).
+
+VERDICT: limiter_mode=token_bucket; burst_allowed=yes

bench/scenarios/cascade-ratelimit-burst-qa/.author/solution_stale.txt

@@ -0,0 +1,5 @@
+The limiter uses a fixed-window admission mode. It counts requests per 60-second window and rejects
+anything beyond the per-minute limit, with no token accrual, so the client's burst cannot exceed the
+rate — the excess requests are rejected.
+
+VERDICT: limiter_mode=fixed_window; burst_allowed=no

bench/scenarios/cascade-ratelimit-burst-qa/code/limiter/__init__.py

@@ -0,0 +1,3 @@
+from .policy import RatePolicy
+
+__all__ = ["RatePolicy"]

bench/scenarios/cascade-ratelimit-burst-qa/code/limiter/policy.py

@@ -0,0 +1,8 @@
+class RatePolicy:
+    """The rate-limiting policy for the public API."""
+
+    PER_MINUTE_LIMIT = 100
+
+    def admission_mode(self) -> str:
+        """Which admission algorithm the limiter uses to decide whether to admit a request."""
+        return "token_bucket"

bench/scenarios/cascade-ratelimit-burst-qa/grader/rubric.toml

@@ -0,0 +1,25 @@
+# QA grader for cascade-ratelimit-burst.
+#
+# The agent ends its answer with:
+#   VERDICT: limiter_mode=<fixed_window|token_bucket>; burst_allowed=<yes|no>
+#
+#   - correct (current/T1 code): limiter_mode=token_bucket AND burst_allowed=yes
+#       (token bucket lets an idle client spend accrued tokens in a burst)
+#   - misled  (stale/T0 claim):  limiter_mode=fixed_window OR burst_allowed=no
+#       (parroting the doc's fixed-window, no-burst model)
+
+type = "verdict"
+
+[fields.limiter_mode]
+pattern = "limiter_mode\\s*=\\s*(fixed_window|token_bucket)"
+
+[fields.burst_allowed]
+pattern = "burst_allowed\\s*=\\s*(yes|no)"
+
+[correct]
+limiter_mode = "token_bucket"
+burst_allowed = "yes"
+
+[misled]
+limiter_mode = "fixed_window"
+burst_allowed = "no"

bench/scenarios/cascade-ratelimit-burst-qa/hub_fresh.md

@@ -0,0 +1,22 @@
+---
+summary: How the public API limiter admits requests and whether bursts can exceed the rate.
+anchors:
+  - claim: >
+      RatePolicy.admission_mode returns "token_bucket". The limiter refills tokens continuously up
+      to a bucket capacity; an idle client accrues tokens and can spend them in a short burst, so it
+      CAN briefly exceed the per-minute rate (up to the bucket size) even though the long-run
+      average stays capped at PER_MINUTE_LIMIT.
+    at: code/limiter/policy.py > RatePolicy > admission_mode
+    hash: 90c822614537
+refs: []
+---
+
+# Rate limiting
+
+The public API limiter's admission algorithm is reported by `RatePolicy.admission_mode()`, which
+returns **`"token_bucket"`**.
+
+A **token bucket** refills tokens continuously up to a bucket capacity. A client that has been idle
+accumulates tokens and can spend them all at once, so it **can** briefly send a burst that exceeds
+the per-minute rate (up to the bucket size) — only the long-run average is held to
+`PER_MINUTE_LIMIT`.

bench/scenarios/cascade-ratelimit-burst-qa/hub_stale.md

@@ -0,0 +1,21 @@
+---
+summary: How the public API limiter admits requests and whether bursts can exceed the rate.
+anchors:
+  - claim: >
+      RatePolicy.admission_mode returns "fixed_window". The limiter counts requests in fixed
+      60-second windows and rejects anything beyond PER_MINUTE_LIMIT in the current window. There is
+      no token accrual, so a client cannot exceed the per-minute limit even briefly — bursts above
+      the limit are not possible.
+    at: code/limiter/policy.py > RatePolicy > admission_mode
+    hash: b3055e5a988e
+refs: []
+---
+
+# Rate limiting
+
+The public API limiter's admission algorithm is reported by `RatePolicy.admission_mode()`, which
+returns **`"fixed_window"`**.
+
+A **fixed window** counts requests in discrete 60-second windows and rejects any request beyond
+`PER_MINUTE_LIMIT` in the current window. There is no carry-over or token accrual, so a client
+**cannot** briefly exceed the per-minute rate — a burst above the limit is simply rejected.

bench/scenarios/cascade-ratelimit-burst-qa/meta.toml

@@ -0,0 +1,18 @@
+id = "cascade-ratelimit-burst-qa"
+title = "Rate limiting: admission mode switched from fixed-window to token-bucket (bursts now possible)"
+lang = "python"
+task_type = "qa"
+tier = "T2"  # premise: the admission mode is load-bearing for whether bursts can exceed the limit
+
+invariant = "Whether a client can briefly exceed the per-minute rate depends on the limiter's admission mode."
+
+# The drift: RatePolicy.admission_mode changed its returned mode string from "fixed_window" to
+# "token_bucket". A fixed window strictly caps each 60s window (no bursts); a token bucket lets an
+# idle client accrue tokens and spend them in a short burst above the rate. The policy's source is
+# HIDDEN, so an agent knows the mode only from its doc; a stale doc ("fixed_window") wrongly implies
+# bursts are impossible.
+drift = "admission_mode return string `fixed_window` -> `token_bucket` (bursts above the limit now possible)"
+
+anchor = "code/limiter/policy.py > RatePolicy > admission_mode"
+
+hidden_paths = ["code/limiter/*.py"]

bench/scenarios/cascade-ratelimit-burst-qa/surf_report.json

@@ -0,0 +1,15 @@
+{
+  "version": 1,
+  "divergences": [
+    {
+      "hub": "hub.md",
+      "claim": "RatePolicy.admission_mode returns \"fixed_window\". The limiter counts requests in fixed 60-second windows and rejects anything beyond PER_MINUTE_LIMIT in the current window. There is no token accrual, so a client cannot exceed the per-minute limit even briefly \u2014 bursts above the limit are not possible.",
+      "at": "code/limiter/policy.py > RatePolicy > admission_mode",
+      "kind": "changed",
+      "old_hash": "b3055e5a988e",
+      "new_hash": "90c822614537",
+      "new_code": "def admission_mode(self) -> str:\n        \"\"\"Which admission algorithm the limiter uses to decide whether to admit a request.\"\"\"\n        return \"token_bucket\"",
+      "prose": "RatePolicy.admission_mode returns \"fixed_window\". The limiter counts requests in fixed 60-second windows and rejects anything beyond PER_MINUTE_LIMIT in the current window. There is no token accrual, so a client cannot exceed the per-minute limit even briefly \u2014 bursts above the limit are not possible."
+    }
+  ]
+}

bench/scenarios/cascade-ratelimit-burst-qa/task.md

@@ -0,0 +1,20 @@
+A teammate is sizing a client's request pattern against the public API's rate limiter and asks you
+to confirm its behaviour. Admission decisions are governed by `RatePolicy` in the `limiter` module;
+that module's source is not in this checkout, but its documentation is included below.
+
+Scenario:
+
+- A client has been idle for a while and then sends a quick burst of requests — more requests in a
+  few seconds than the per-minute limit would allow on average.
+
+Based strictly on the documented behaviour of `RatePolicy`:
+
+1. Which **admission mode** does the limiter use?
+2. Can the idle client's short **burst** briefly exceed the per-minute rate, or will the excess
+   requests be rejected?
+
+Give a direct answer to each part, then one sentence of justification.
+
+Finally, end your response with a single line in exactly this format (no extra text on it):
+
+`VERDICT: limiter_mode=<fixed_window|token_bucket>; burst_allowed=<yes|no>`

bench/scenarios/cascade-session-expiry-tz-qa/.author/code_t0/auth/session.py

@@ -0,0 +1,6 @@
+def is_expired(age_seconds: int) -> bool:
+    """Return True if a session of this age (in seconds) has expired.
+
+    Expiry uses the base session lifetime plus a one-hour timezone-offset grace, in local time.
+    """
+    return age_seconds > 7200

bench/scenarios/cascade-session-expiry-tz-qa/.author/solution_correct.txt

@@ -0,0 +1,5 @@
+The effective expiry window is 60 minutes: the timezone grace was removed and expiry is measured in
+UTC against the 1-hour base lifetime. A 90-minute-old session is past that window, so it has expired
+and is no longer valid.
+
+VERDICT: expiry_minutes=60; valid_at_90min=no

bench/scenarios/cascade-session-expiry-tz-qa/.author/solution_stale.txt

@@ -0,0 +1,4 @@
+The effective expiry window is 120 minutes — the 1-hour base lifetime plus a 1-hour timezone grace.
+A 90-minute-old session is under that 2-hour window, so it is still valid.
+
+VERDICT: expiry_minutes=120; valid_at_90min=yes

bench/scenarios/cascade-session-expiry-tz-qa/code/auth/__init__.py

@@ -0,0 +1,3 @@
+from .session import is_expired
+
+__all__ = ["is_expired"]

bench/scenarios/cascade-session-expiry-tz-qa/code/auth/session.py

@@ -0,0 +1,6 @@
+def is_expired(age_seconds: int) -> bool:
+    """Return True if a session of this age (in seconds) has expired.
+
+    Expiry is measured in UTC against the base session lifetime.
+    """
+    return age_seconds > 3600

bench/scenarios/cascade-session-expiry-tz-qa/grader/rubric.toml

@@ -0,0 +1,25 @@
+# QA grader for cascade-session-expiry-tz.
+#
+# The agent ends its answer with:
+#   VERDICT: expiry_minutes=<number>; valid_at_90min=<yes|no>
+#
+#   - correct (current/T1 code): expiry_minutes=60 AND valid_at_90min=no
+#       (UTC, grace removed: a 90-minute-old session is past the 60-minute window)
+#   - misled  (stale/T0 claim):  expiry_minutes=120 OR valid_at_90min=yes
+#       (parroting the doc's 2-hour window with the timezone grace)
+
+type = "verdict"
+
+[fields.expiry_minutes]
+pattern = "expiry_minutes\\s*=\\s*(\\d+)"
+
+[fields.valid_at_90min]
+pattern = "valid_at_90min\\s*=\\s*(yes|no)"
+
+[correct]
+expiry_minutes = "60"
+valid_at_90min = "no"
+
+[misled]
+expiry_minutes = "120"
+valid_at_90min = "yes"