From 64c2a0eebe3461cf4b40967490d2092a2d7debfb Mon Sep 17 00:00:00 2001 From: iamsiddhesh-dev Date: Fri, 4 Sep 2026 12:07:05 +0530 Subject: [PATCH 1/5] fix: stop reconsidering a payment already handed to a human --- recoup/agent/executor.py | 14 ++++++++++++-- recoup/eval/runner.py | 7 ++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/recoup/agent/executor.py b/recoup/agent/executor.py index 7c604e8..26b5c88 100644 --- a/recoup/agent/executor.py +++ b/recoup/agent/executor.py @@ -50,6 +50,13 @@ class Execution: contacted: bool = False detail: str = "" + # Whether this action ends the agent's involvement. Distinct from `succeeded`, + # and the distinction matters: handing a case to a human did not recover the + # money, but it is also not a failed attempt to be tried again. Without this + # the caller can only see "did not succeed" and will quite reasonably + # reschedule — which is exactly what happened. See FAILURES.md. + terminal: bool = False + class Executor: def __init__( @@ -124,7 +131,7 @@ def record(kind: EventKind, **data) -> None: if action is ActionKind.STOP: record(EventKind.STOPPED, reason=decision.reason) - return Execution(action=action, at=at, succeeded=False), events + return Execution(action=action, at=at, succeeded=False, terminal=True), events if action is ActionKind.ESCALATE_HUMAN: # Humans are outside the system. The agent hands the case over and @@ -138,8 +145,11 @@ def record(kind: EventKind, **data) -> None: detail="handed to human review", ) record(EventKind.STOPPED, reason="escalated to human review") + self._context.note_escalation(context.payment.id) return ( - Execution(action=action, at=at, succeeded=False, cost_paise=cost), + Execution( + action=action, at=at, succeeded=False, cost_paise=cost, terminal=True + ), events, ) diff --git a/recoup/eval/runner.py b/recoup/eval/runner.py index 2ba35ac..57fb1bb 100644 --- a/recoup/eval/runner.py +++ b/recoup/eval/runner.py @@ -328,7 +328,12 @@ def _perform( # Failed, still open, still time: reconsider. The policy will propose the # next attempt or stop. - if not executed.succeeded and when < horizon: + # + # `terminal` is the important half of that condition. Escalation does not + # succeed — the money is not recovered — but it is not an open case + # either, and reading only `succeeded` put every escalated payment back on + # the timeline a minute later to be escalated again. + if not executed.succeeded and not executed.terminal and when < horizon: timeline.schedule( min(when + timedelta(minutes=1), horizon), Scheduled(task=Task.RECONSIDER, payment=context.payment), From bc789a4e18137dcf3f6eb3cbb92ce885f3fe0066 Mon Sep 17 00:00:00 2001 From: iamsiddhesh-dev Date: Fri, 4 Sep 2026 12:07:05 +0530 Subject: [PATCH 2/5] feat: cap human review at one per payment, not just per run --- config/compliance.yaml | 6 ++++++ recoup/agent/compliance.py | 12 ++++++++++++ recoup/agent/config.py | 1 + recoup/agent/context.py | 12 ++++++++++++ 4 files changed, 31 insertions(+) diff --git a/config/compliance.yaml b/config/compliance.yaml index 041e516..8621da2 100644 --- a/config/compliance.yaml +++ b/config/compliance.yaml @@ -130,6 +130,12 @@ escalation: # cap being hit as an exception rather than silently dropping cases. max_escalations_per_run: 50 + # One payment, one human. A second reviewer does not make the first one's + # answer different, and the run cap is a shared pool: four payments taking + # twelve slots each consumed 48 of 50 and starved every other case that + # needed one. See FAILURES.md. + max_escalations_per_payment: 1 + # --------------------------------------------------------------------------- # Execution safety # --------------------------------------------------------------------------- diff --git a/recoup/agent/compliance.py b/recoup/agent/compliance.py index 7e1773a..c4a134c 100644 --- a/recoup/agent/compliance.py +++ b/recoup/agent/compliance.py @@ -212,6 +212,18 @@ def _check_escalation( ), ) + if context.escalations >= rules.max_escalations_per_payment: + return Veto( + rule="escalation:already_escalated", + action=candidate.action, + why=( + f"already handed to human review " + f"{context.escalations} time(s). A second reviewer does not " + f"make the first one's answer different, and each slot spent " + f"here is one not spent on another payment." + ), + ) + if self._escalations >= rules.max_escalations_per_run: return Veto( rule="escalation:run_cap", diff --git a/recoup/agent/config.py b/recoup/agent/config.py index 5ba9455..8572d0a 100644 --- a/recoup/agent/config.py +++ b/recoup/agent/config.py @@ -151,6 +151,7 @@ class MandateRules(BaseModel): class EscalationRules(BaseModel): human_review_above_paise: int max_escalations_per_run: int + max_escalations_per_payment: int = 1 class ExecutionRules(BaseModel): diff --git a/recoup/agent/context.py b/recoup/agent/context.py index 6e5085b..1634e94 100644 --- a/recoup/agent/context.py +++ b/recoup/agent/context.py @@ -170,6 +170,7 @@ class DecisionContext: now: datetime attempts: int = 0 + escalations: int = 0 contacts_in_window: int = 0 last_contact_at: datetime | None = None consecutive_failures: int = 0 @@ -227,6 +228,7 @@ def __init__(self, policy: PolicyConfig, model: RecoveryModel | None = None) -> self.model = model or RecoveryModel(policy) self._attempts: dict[str, int] = {} + self._escalations: dict[str, int] = {} self._consecutive_failures: dict[str, int] = {} self._contacts: dict[str, list[datetime]] = {} self._downtimes: dict[str, DowntimeEntity] = {} @@ -242,6 +244,15 @@ def note_attempt(self, payment_id: str, succeeded: bool) -> None: self._consecutive_failures.get(payment_id, 0) + 1 ) + def note_escalation(self, payment_id: str) -> None: + """Handing one payment to a human, counted per payment. + + Tracked here beside attempts and contacts rather than inside the + compliance gate, so the rule that reads it is a pure function of the + context it is given. + """ + self._escalations[payment_id] = self._escalations.get(payment_id, 0) + 1 + def note_contact(self, customer_ref: str, at: datetime) -> None: self._contacts.setdefault(customer_ref, []).append(at) @@ -293,6 +304,7 @@ def build( classification=classification, now=now, attempts=self._attempts.get(payment.id, 0), + escalations=self._escalations.get(payment.id, 0), consecutive_failures=self._consecutive_failures.get(payment.id, 0), contacts_in_window=self.contacts_in_window(payment.customer_ref, now), last_contact_at=self.last_contact(payment.customer_ref), From df2af9cd534383e542cc8d22b85279dadd4ab45e Mon Sep 17 00:00:00 2001 From: iamsiddhesh-dev Date: Fri, 4 Sep 2026 12:07:05 +0530 Subject: [PATCH 3/5] fix: say a case was escalated rather than that it ran out of options --- recoup/agent/llm/explainer.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/recoup/agent/llm/explainer.py b/recoup/agent/llm/explainer.py index 17145b7..70699f5 100644 --- a/recoup/agent/llm/explainer.py +++ b/recoup/agent/llm/explainer.py @@ -59,6 +59,8 @@ CHANNELS = ("sms", "whatsapp", "voice", "email") +ESCALATE = "ESCALATE_HUMAN" + # Claims that are simply false when the payment was not recovered. Deliberately # specific phrases rather than the word "recovered", which appears legitimately in # sentences like "we stopped before spending more than the payment could recover". @@ -223,6 +225,11 @@ def validate(text: str, facts: CaseFacts) -> str | None: # --------------------------------------------------------------------------- +def _times(n: int) -> str: + """"1 times" is the kind of thing that makes a report look generated.""" + return f"{n} time" if n == 1 else f"{n} times" + + def summarise(facts: CaseFacts) -> str: """An explanation composed from the facts, with no model involved. @@ -239,27 +246,32 @@ def summarise(facts: CaseFacts) -> str: + (f", diagnosed as {cause}." if cause else " and could not be diagnosed.") ) + escalated = ESCALATE in facts.actions + if facts.actions: did = [] if facts.attempts: - did.append(f"retried it {facts.attempts} time{'s' if facts.attempts > 1 else ''}") + did.append(f"retried it {_times(facts.attempts)}") if facts.contacts: channels = " and ".join(dict.fromkeys(facts.channels)) or "the customer" - did.append( - f"contacted the customer {facts.contacts} " - f"time{'s' if facts.contacts > 1 else ''} by {channels}" - ) + did.append(f"contacted the customer {_times(facts.contacts)} by {channels}") + if escalated: + did.append("handed it to human review") spent = rupees(facts.cost_paise, precise_below=10_000) middle = ( f" The agent {' and '.join(did)}, spending {spent}." if did - else f" The agent acted {len(facts.actions)} times, spending {spent}." + else f" The agent acted {_times(len(facts.actions))}, spending {spent}." ) else: middle = " The agent took no action on it." if facts.recovered: closing = f" It was recovered, returning {rupees(facts.recovered_paise)}." + elif escalated: + # Whether a person then resolved it is outside this system, and the run + # does not claim credit for it either way. + closing = " The outcome of that review is outside this record." elif facts.vetoes: rules = ", ".join(dict.fromkeys(facts.vetoes)) closing = f" It was not recovered. Compliance refused further action under {rules}." From deb74c774c533e88c7f8dacd431227cc64375994 Mon Sep 17 00:00:00 2001 From: iamsiddhesh-dev Date: Fri, 4 Sep 2026 12:07:05 +0530 Subject: [PATCH 4/5] test: assert no payment is escalated or actioned twice --- tests/test_compliance.py | 44 ++++++++++++++++++++++++++++++++++++---- tests/test_eval.py | 37 +++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 4 deletions(-) diff --git a/tests/test_compliance.py b/tests/test_compliance.py index 2710d21..87a0306 100644 --- a/tests/test_compliance.py +++ b/tests/test_compliance.py @@ -338,19 +338,55 @@ def test_large_payments_may_be_escalated(gate, builder): def test_escalation_is_capped_per_run(gate, builder, rules): - """An agent that escalates everything is not a product.""" - context = _context(builder, amount=9000000) + """An agent that escalates everything is not a product. - for _ in range(rules.escalation.max_escalations_per_run): + A distinct payment each time. Reusing one would now be stopped by the + per-payment rule instead, and the test would pass while measuring nothing. + """ + for n in range(rules.escalation.max_escalations_per_run): + context = _context(builder, amount=9000000, id=f"pay_{n:05d}") chosen, _ = gate.screen([_candidate(ActionKind.ESCALATE_HUMAN)], context) gate.note_executed(chosen) - chosen, vetoes = gate.screen([_candidate(ActionKind.ESCALATE_HUMAN)], context) + chosen, vetoes = gate.screen( + [_candidate(ActionKind.ESCALATE_HUMAN)], + _context(builder, amount=9000000, id="pay_99999"), + ) assert chosen is None assert vetoes[0].rule == "escalation:run_cap" +def test_a_payment_is_only_escalated_once(gate, builder): + """Four payments once took twelve human slots each and starved the rest. + + The run cap is a shared pool of fifty. Nothing stopped a single payment + drawing from it repeatedly, so 48 of the 50 slots went to four payments. + """ + context = _context(builder, amount=9000000, id="pay_00001") + chosen, _ = gate.screen([_candidate(ActionKind.ESCALATE_HUMAN)], context) + assert chosen is not None + + builder.note_escalation("pay_00001") + again = _context(builder, amount=9000000, id="pay_00001") + + chosen, vetoes = gate.screen([_candidate(ActionKind.ESCALATE_HUMAN)], again) + + assert chosen is None + assert vetoes[0].rule == "escalation:already_escalated" + + +def test_escalating_one_payment_does_not_block_another(gate, builder): + builder.note_escalation("pay_00001") + + chosen, _ = gate.screen( + [_candidate(ActionKind.ESCALATE_HUMAN)], + _context(builder, amount=9000000, id="pay_00002"), + ) + + assert chosen is not None + + def test_run_caps_reset(gate, builder, rules): context = _context(builder, amount=9000000) for _ in range(rules.escalation.max_escalations_per_run): diff --git a/tests/test_eval.py b/tests/test_eval.py index d3af0ba..7675940 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -10,6 +10,8 @@ from __future__ import annotations +from collections import Counter + import pytest from recoup.agent.config import ComplianceConfig, PolicyConfig @@ -193,6 +195,41 @@ def test_escalation_respects_its_run_cap(results): assert metrics.actions_by_kind.get("ESCALATE_HUMAN", 0) <= cap +def test_no_payment_is_escalated_twice(ledger): + """The run cap alone let four payments take twelve human slots each. + + Escalation records STOPPED and hands the case away, but it also does not + *succeed*, and the runner rescheduled anything that had not succeeded. So the + same payment came back a minute later and was escalated again, 48 of the 50 + slots going to four payments while every other case that needed a human got + nothing. See FAILURES.md. + """ + for arm in ledger.arms(): + escalated = Counter( + event.payment_id + for event in ledger.events(arm=arm) + if event.kind is EventKind.EXECUTED + and event.data.get("action") == "ESCALATE_HUMAN" + ) + repeated = {pid: n for pid, n in escalated.items() if n > 1} + assert not repeated, f"{arm} escalated the same payment more than once: {repeated}" + + +def test_an_escalated_payment_is_not_acted_on_again(ledger): + """Handing a case to a human ends the agent's involvement with it.""" + for arm in ledger.arms(): + handed_over: set[str] = set() + for event in ledger.events(arm=arm): + if event.kind is not EventKind.EXECUTED: + continue + action = event.data.get("action", "") + assert event.payment_id not in handed_over, ( + f"{arm}/{event.payment_id}: {action} after escalation" + ) + if action == "ESCALATE_HUMAN": + handed_over.add(event.payment_id) + + # --------------------------------------------------------------------------- # Reproducibility # --------------------------------------------------------------------------- From 5700801691151545ed85d9bb4a98dc08fbcaf04a Mon Sep 17 00:00:00 2001 From: iamsiddhesh-dev Date: Fri, 4 Sep 2026 12:07:06 +0530 Subject: [PATCH 5/5] docs: record the escalation loop and the cost it removed --- FAILURES.md | 71 +++++++++++++++++++++++++++++++++++++++++++++ README.md | 30 +++++++++++++------ reports/claims.json | 12 ++++---- 3 files changed, 98 insertions(+), 15 deletions(-) diff --git a/FAILURES.md b/FAILURES.md index 947b8c9..8917537 100644 --- a/FAILURES.md +++ b/FAILURES.md @@ -259,3 +259,74 @@ producing a confident zero. **The general shape.** A measurement that reports "no effect" deserves more scrutiny than one reporting a large effect, not less. A large effect is usually real; no effect is often the instrument being disconnected. + +--- + +## Four payments quietly consumed the entire human-review budget + +**Found by:** reading a case brief while building the explainer, and noticing it +listed `ESCALATE_HUMAN` twelve times for one payment. + +Escalation hands a case to a person. The executor records it, writes a `STOPPED` +event, and returns an `Execution` — with `succeeded=False`, because no money came +back. The runner's rule for what to do next was: + +```python +if not executed.succeeded and when < horizon: + reschedule(RECONSIDER) +``` + +So every escalated payment came back a minute later. The policy proposed +escalation again — nothing about the situation had changed — the gate allowed it, +and the loop ran until the per-payment decision cap of twelve stopped it. + +The result on the committed seed: **48 escalations across just four payments, +twelve each, against a run cap of fifty.** ₹5,280 spent on forty-four handovers of +cases that had already been handed over. Two more repeats on any one of them and +the pool would have been exhausted, and every subsequent payment that genuinely +warranted a human would have been refused one. + +**Why nothing caught it.** Every individual piece behaved correctly. The executor +did record the escalation and did stop touching the payment. The compliance gate +did enforce its run cap — 48 is under 50, so it never fired. The policy did price +escalation correctly each time it was asked. `actions_by_kind` showed +`ESCALATE_HUMAN: 48` and 48 looks like forty-eight escalated payments, which would +have been a reasonable number. The failure was only visible in the *distribution*, +and nothing summarised that. + +It also cost nothing detectable in recovery, because escalation never recovers +money in this simulator — so the totals moved by ₹5,280 of cost and not one rupee +of revenue. A bug that only makes you slightly poorer is much harder to notice +than one that breaks something. + +**Fixed in two places, deliberately.** + +`Execution` now carries `terminal` alongside `succeeded`. They are different +questions — handing a case to a human did not recover the money and is also not a +failed attempt worth retrying — and collapsing them into one boolean is what +created the loop. That is the root cause. + +The compliance gate separately refuses a second escalation on the same payment, +via `max_escalations_per_payment`. This is defence in depth and it is the more +important of the two: "one payment, one human" is a *policy*, and a policy that +holds only because of the shape of a scheduling loop somewhere else is not +enforced, it is coincidental. The previous entry in this file is about a +compliance rule that looked present and did nothing; the lesson generalises to +rules that are absent because another layer happens to make them unnecessary. + +That rule does not fire on the committed seed — with the runner fixed, a second +escalation is never proposed. A rule that never fires is exactly the shape of the +mandate bug above, so it is covered by unit tests that construct the condition +directly and assert the veto, rather than being trusted because the run looks +clean. + +**What changed in the numbers.** Recovery, contacts and refusals are identical to +the rupee: escalation never recovered anything, so removing forty-four of them +removed only cost. Agent spend fell from ₹8,336 to ₹3,056 and net margin rose from +₹1,43,347 to ₹1,48,627. The headline incremental figure of ₹3,94,791 is unchanged, +because it is measured on recovery. + +**The general shape.** A cap on a shared resource is not a cap on any one consumer +of it. `max_escalations_per_run` was doing exactly what it said and was still the +wrong rule on its own, because the interesting failure was not "too many +escalations" but "too few payments receiving them". diff --git a/README.md b/README.md index 0900265..a4dd614 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ that decision actually earned — against a holdout. > **On a 5,000-payment batch — 1,605 failures, ₹30,02,856 at risk — Recoup recovers > ₹5,41,724, of which ₹3,94,791 is incremental over the naive fixed-retry schedule -> merchants actually run. It spends ₹8,336 on 926 contacts to do it, and refuses 903 +> merchants actually run. It spends ₹3,056 on 926 contacts to do it, and refuses 903 > actions on compliance grounds.** > > Seed `20260902`. Reproduce with `make eval` — no API key required. @@ -22,9 +22,9 @@ Incremental recovery is what the agent is actually worth. | Arm | What it does | Recovered | Contacts | Cost | Net margin | |---|---|---:|---:|---:|---:| | `naive_baseline` | fixed retry schedule, no regard for cause | ₹1,46,933 | 0 | ₹2,509 | ₹38,632 | -| `contact_only` | always contact, never reason about it | ₹4,14,362 | 975 | ₹6,386 | ₹1,09,636 | -| **`recoup_agent`** | **expected-value policy + compliance gate** | **₹5,41,724** | **926** | **₹8,336** | **₹1,43,347** | -| `recoup_agent_no_llm` | ablation — deterministic classifier only | ₹5,41,963 | 927 | ₹8,338 | ₹1,43,412 | +| `contact_only` | always contact, never reason about it | ₹4,14,362 | 975 | ₹1,106 | ₹1,14,916 | +| **`recoup_agent`** | **expected-value policy + compliance gate** | **₹5,41,724** | **926** | **₹3,056** | **₹1,48,627** | +| `recoup_agent_no_llm` | ablation — deterministic classifier only | ₹5,41,963 | 927 | ₹3,058 | ₹1,48,692 | All four arms run against the same world, the same failures, in the same order, with the same underlying luck. Outcomes are drawn from `(seed, payment_id, attempt)`, so two arms @@ -152,7 +152,7 @@ arm's stream is hashed when written, and the audit screen re-hashes it in front |---|---| | Unmapped free-text error → taxonomy + a proposed mapping rule for human review | Retry timing → empirical success model (cause × issuer × hour) | | Recovery copy, generated as **templates** in English, Hinglish and Hindi, behind a validator | Action selection → expected-value argmax | -| | Compliance → hard rules, no model in the loop | +| A case narrated in prose for the merchant, from facts already in the ledger | Compliance → hard rules, no model in the loop | | | All money math → arithmetic | | | Root-cause mapping for known error codes → lookup table | @@ -172,16 +172,28 @@ Two rules make the model safe to put in front of a customer: card number is indistinguishable from fraud. Copy mentioning a credential is discarded and a hand-written fallback used instead, with the rejection recorded. -### Four model calls per run +The case explainer inverts the first rule rather than repeating it: it **may only use +numbers it was given.** Every digit run in a generated explanation must appear in that +case's own brief, so a narrative cannot invent "three retry attempts" for a payment that +had one — on the screen whose entire purpose is showing that the numbers add up. It also +cannot name a channel that was not used or claim a recovery that did not happen. Anything +that fails falls back to an explanation composed from the same facts with no model +involved, so every case has one. + +### Five model calls per run The free-tier budget settled the architecture before taste could. A per-payment call would need ~1,500 requests per run, which exceeds Gemini Flash's daily allowance by 75× and Groq's token budget by 6×. **A per-payment LLM call here is not merely poor judgment; it is impossible.** -So the calls are batched: every unmapped error in the run goes up in one request, and the -copy matrix — 7 causes × 3 languages × 4 channels — in three more, one per language. -Four calls per run, against a twenty-per-day quota. +So the calls are batched: every unmapped error in the run goes up in one request, the copy +matrix — 7 causes × 3 languages × 4 channels — in three more, one per language, and every +case explanation in a fifth. Five calls per run, against a twenty-per-day quota. + +Nothing is generated on a page load. Explaining a case on demand would be a call per view, +so a judge clicking through ten cases would spend half the daily allowance on a read-only +screen — and two visits to the same case could disagree with each other. Responses are content-addressed and **committed to `cache/llm/`**, so the reported numbers reproduce on a clean clone with no key and no network. Keys are only needed to regenerate diff --git a/reports/claims.json b/reports/claims.json index 5c41270..1f45e1e 100644 --- a/reports/claims.json +++ b/reports/claims.json @@ -1,7 +1,7 @@ { "contact_only.contacts": 975, - "contact_only.cost_paise": 638550, - "contact_only.digest": "f78d36267c9c098ad11d8b3749493713244e0a272adb4aaddb5dcc1f0169fc88", + "contact_only.cost_paise": 110550, + "contact_only.digest": "a8370be3cc3efc8fd542a08d04f018e6e4045a600f94bb7abf0b1582e5b06756", "contact_only.recovered_count": 222, "contact_only.recovered_paise": 41436153, "contact_only.unresolved": 51, @@ -18,15 +18,15 @@ "naive_baseline.unresolved": 51, "naive_baseline.vetoes": 78, "recoup_agent.contacts": 926, - "recoup_agent.cost_paise": 833631, - "recoup_agent.digest": "708c0755d5e70520e3ae6a33ef49a8202b650ec5e60e81f71cb08841e621aac9", + "recoup_agent.cost_paise": 305631, + "recoup_agent.digest": "4943eb3dbedf8025b7eff6678531eb7c8d403b92745af32768f1c8accbbf43c2", "recoup_agent.recovered_count": 319, "recoup_agent.recovered_paise": 54172446, "recoup_agent.unresolved": 0, "recoup_agent.vetoes": 903, "recoup_agent_no_llm.contacts": 927, - "recoup_agent_no_llm.cost_paise": 833771, - "recoup_agent_no_llm.digest": "f40ec4a069bbe8e31a2018fafc1160c92ef91d46899de595ab31282673e57778", + "recoup_agent_no_llm.cost_paise": 305771, + "recoup_agent_no_llm.digest": "b7baf5c99489e2ebebb5747bd77f174bf31ed0916718f5c603b2fb77a0883e53", "recoup_agent_no_llm.recovered_count": 320, "recoup_agent_no_llm.recovered_paise": 54196291, "recoup_agent_no_llm.unresolved": 51,