From 6a0d0e11608c4a2d6c975a794a40eaeccb174b43 Mon Sep 17 00:00:00 2001 From: itseffi <15998472+itseffi@users.noreply.github.com> Date: Sun, 24 May 2026 21:18:39 +0200 Subject: [PATCH] v2 Co-Authored-By: Claude Opus 4.7 --- agenticevals/verifiers/llm_rubric.py | 86 +++++++++++++++++++++------- tests/test_llm_rubric.py | 35 +++++++++++ 2 files changed, 101 insertions(+), 20 deletions(-) diff --git a/agenticevals/verifiers/llm_rubric.py b/agenticevals/verifiers/llm_rubric.py index ef398eb..1de1d76 100644 --- a/agenticevals/verifiers/llm_rubric.py +++ b/agenticevals/verifiers/llm_rubric.py @@ -31,12 +31,18 @@ def verify(self, context: VerifierContext, spec: VerifierSpec) -> list[Criterion ) ] threshold = float(spec.config.get("threshold", 0.5)) - try: - judged = self._judge(context, spec) - except Exception as exc: - # The judge erroring (transient API failure, unparseable output) is - # not evidence about the agent. Abstain: weight 0 so it cannot drag - # down the weighted reward, and non-required so it cannot fail an + repetitions = max(1, int(spec.config.get("repetitions", 1))) + judgments: list[dict[str, Any]] = [] + last_error: Exception | None = None + for rep in range(repetitions): + try: + judgments.append(self._judge(context, spec, rep=rep)) + except Exception as exc: # noqa: BLE001 - aggregated/abstained below + last_error = exc + if not judgments: + # Every judge call failed (transient API error, unparseable output). + # That is not evidence about the agent. Abstain: weight 0 so it cannot + # drag down the weighted reward, and non-required so it cannot fail an # otherwise-passing run. The error stays recorded for inspection. return [ criterion( @@ -45,38 +51,41 @@ def verify(self, context: VerifierContext, spec: VerifierSpec) -> list[Criterion score=0.0, weight=0.0, passed=False, - detail=f"judge abstained: {exc}", + detail=f"judge abstained: {last_error}", required=False, deterministic=False, - error=str(exc), + error=str(last_error), ) ] - if "passed" in judged: - passed = bool(judged["passed"]) - score = float(judged.get("score", 1.0 if passed else 0.0)) - else: - score = float(judged.get("score", 0.0)) - passed = score >= threshold - detail = str(judged.get("reason", "")) + verdict = _aggregate_judgments(judgments, threshold) return [ criterion( name=spec.name or "llm_rubric", verifier_type=self.verifier_type, - score=score, + score=verdict["score"], weight=spec.weight, - passed=passed, - detail=detail, + passed=verdict["passed"], + detail=verdict["reason"], required=spec.required, deterministic=False, - evidence={"threshold": threshold, "judge": judged}, + evidence={ + "threshold": threshold, + "repetitions": len(judgments), + "votes": verdict["votes"], + "judgments": judgments, + }, ) ] - def _judge(self, context: VerifierContext, spec: VerifierSpec) -> dict[str, Any]: + def _judge(self, context: VerifierContext, spec: VerifierSpec, *, rep: int = 0) -> dict[str, Any]: provider = str(spec.config.get("provider", "openai")) model = str(spec.config.get("model", "gpt-4o-mini" if provider == "openai" else "claude-sonnet-4-6")) timeout = int(spec.config.get("timeout", 60)) prompt = _rubric_prompt(context, spec) + if rep > 0: + # Distinguish repeated samples so the response cache returns an + # independent draw rather than the first cached verdict. + prompt = f"{prompt}\n\n(independent sample {rep + 1})" if provider == "anthropic": response = anthropic_messages( model, @@ -91,6 +100,43 @@ def _judge(self, context: VerifierContext, spec: VerifierSpec) -> dict[str, Any] raise ValueError(f"unsupported llm_rubric provider: {provider}") +def _verdict_from_judgment(judged: dict[str, Any], threshold: float) -> tuple[bool, float]: + if "passed" in judged: + passed = bool(judged["passed"]) + score = float(judged.get("score", 1.0 if passed else 0.0)) + else: + score = float(judged.get("score", 0.0)) + passed = score >= threshold + return passed, score + + +def _aggregate_judgments(judgments: list[dict[str, Any]], threshold: float) -> dict[str, Any]: + """Majority-vote a set of judge results; break ties on the mean score. + + Repeated judging surfaces variance that a single cached call hides; the + aggregate verdict is the majority pass/fail, and the score is the mean. + """ + votes: list[bool] = [] + scores: list[float] = [] + reasons: list[str] = [] + for judged in judgments: + passed, score = _verdict_from_judgment(judged, threshold) + votes.append(passed) + scores.append(score) + if judged.get("reason"): + reasons.append(str(judged["reason"])) + mean_score = sum(scores) / len(scores) if scores else 0.0 + pass_votes = sum(1 for vote in votes if vote) + if pass_votes * 2 == len(votes): + passed = mean_score >= threshold + else: + passed = pass_votes * 2 > len(votes) + reason = "; ".join(reasons[:3]) + if len(votes) > 1: + reason = f"majority {pass_votes}/{len(votes)} passed; {reason}".rstrip("; ") + return {"passed": passed, "score": round(mean_score, 6), "votes": votes, "reason": reason} + + def _rubric_prompt(context: VerifierContext, spec: VerifierSpec) -> str: rubric = str(spec.config.get("rubric", "Judge whether the agent satisfied the task.")) transcript = _compact_transcript(context, spec) diff --git a/tests/test_llm_rubric.py b/tests/test_llm_rubric.py index bf55b35..1b8057a 100644 --- a/tests/test_llm_rubric.py +++ b/tests/test_llm_rubric.py @@ -7,6 +7,7 @@ from agenticevals.verifiers.base import VerifierContext from agenticevals.verifiers.llm_rubric import ( LLMRubricVerifier, + _aggregate_judgments, _compact_transcript, _parse_json_object, _rubric_prompt, @@ -72,6 +73,40 @@ def test_fixture_score_is_flagged_as_not_a_real_judgment(self): self.assertTrue(result.evidence.get("fixture")) +class RubricRepetitionTests(unittest.TestCase): + def test_majority_vote_passes_when_most_judgments_pass(self): + agg = _aggregate_judgments( + [{"passed": True, "score": 0.9}, {"passed": True, "score": 0.8}, {"passed": False, "score": 0.2}], 0.5 + ) + self.assertTrue(agg["passed"]) + self.assertEqual(agg["votes"], [True, True, False]) + + def test_majority_vote_fails_when_most_fail(self): + agg = _aggregate_judgments([{"passed": False}, {"passed": False}, {"passed": True}], 0.5) + self.assertFalse(agg["passed"]) + + def test_even_split_breaks_tie_on_mean_score(self): + agg = _aggregate_judgments([{"passed": True, "score": 0.4}, {"passed": False, "score": 0.4}], 0.5) + self.assertFalse(agg["passed"]) # mean 0.4 < threshold 0.5 + + def test_verify_runs_the_judge_repeatedly_and_aggregates(self): + verifier = LLMRubricVerifier() + calls = {"n": 0} + seq = [{"passed": True}, {"passed": True}, {"passed": False}] + + def fake(context, spec, rep=0): + i = calls["n"] + calls["n"] += 1 + return seq[i] + + verifier._judge = fake # type: ignore[method-assign] + spec = VerifierSpec(type="llm_rubric", config={"repetitions": 3}) + result = verifier.verify(_context(), spec)[0] + self.assertEqual(calls["n"], 3) + self.assertTrue(result.passed) + self.assertEqual(result.evidence["repetitions"], 3) + + class RubricParseTests(unittest.TestCase): def test_parses_prose_wrapped_json(self): text = 'Here is my verdict: {"reason": "ok", "passed": true} — thanks!'