From fbbf06bf9e479f1c4c0e2db737b6df65ad94fa98 Mon Sep 17 00:00:00 2001 From: divo12 Date: Sat, 22 Aug 2026 23:43:24 +0530 Subject: [PATCH 1/2] require confirmed failures for eval export --- ...failure-mining-evals-tuning-ab-research.md | 201 ++++++++++++++++++ src/ofw/__init__.py | 8 + src/ofw/diagnosis.py | 123 ++++++++++- src/ofw/exports.py | 15 +- tests/test_diagnosis.py | 66 ++++++ tests/test_e2e_release.py | 20 +- tests/test_exports.py | 21 ++ tests/test_fit.py | 2 +- 8 files changed, 448 insertions(+), 8 deletions(-) create mode 100644 docs/research/2026-08-22-failure-mining-evals-tuning-ab-research.md diff --git a/docs/research/2026-08-22-failure-mining-evals-tuning-ab-research.md b/docs/research/2026-08-22-failure-mining-evals-tuning-ab-research.md new file mode 100644 index 0000000..8f52770 --- /dev/null +++ b/docs/research/2026-08-22-failure-mining-evals-tuning-ab-research.md @@ -0,0 +1,201 @@ +# Failure mining, agent evals, behavior tuning, and A/B testing + +Generated: 2026-08-22 +Scope: production agent traces → trusted failures → evals → harness edits → paired comparison → governed promotion +Method: current OFW code audit plus primary papers and official platform documentation + +## Executive conclusion + +The existing OFW architecture is directionally correct: it keeps the model and verifier fixed, mines revision-attributed production traces, preserves evidence anchors, separates frontier/regression/selection/admission data, edits only declared files, evaluates champion and candidate on the same cases, and stops unattended promotion at a review artifact. + +Three gaps are important enough to implement now: + +1. **A diagnosed cluster is currently exported before anyone confirms the diagnosis.** Production systems and recent trajectory-attribution work distinguish detection from attribution. OFW should keep `PROPOSED` clusters in review and require a content-bound confirmation before they can enter an eval or sealed holdout. +2. **The paired gate reports point estimates but no uncertainty.** Agent behavior is stochastic. OFW should persist wins, losses, ties, discordant-pair count, and an exact one-sided sign-test probability. A policy may require statistical evidence, while small early suites may explicitly use effect-size-only gating. +3. **The next tuning iteration lacks a compact drill-down index.** Raw results exist, but a proposer has no durable map from failure cluster and source trace to baseline/candidate feedback, prediction error, and raw artifacts. OFW should write this index without replacing or summarizing away the source evidence. + +Two tempting additions are deliberately deferred: + +- **Implicit dissatisfaction mining** should produce review candidates, not verified failures. User corrections, rephrasing, and abandonment are promising signals, but collecting message content is sensitive and the current OFW collection contract intentionally avoids it. +- **Online production A/B experimentation** needs real deployment ownership, traffic allocation, exposure logging, guardrail metrics, and sufficient sample size. OFW can consume such results later; it should not pretend its offline paired replay is a live randomized experiment. + +## 1. What failure mining should mean + +### 1.1 Detection, attribution, and remediation are different claims + +TRACE mines historical trajectories for corrections, rephrasing, abandonment, and other dissatisfaction cues, then separately performs multi-component attribution and exploratory verification. Its reported results—72.7% root-cause attribution and 82% end-to-end fix effectiveness on 60 dissatisfaction traces—also show that attribution is useful but not infallible. A trace signal therefore cannot automatically authorize a harness edit. [TRACE: TRajectory Attribution for Automated Context Engineering](https://arxiv.org/abs/2608.09153) + +AgentEval reaches the same conclusion from a workflow perspective: modeling step dependencies as a DAG materially improves failure detection and root-cause accuracy over flat or end-to-end grading. It reports 0.89 versus 0.41 failure-detection recall and 72% root-cause accuracy against an 81% human ceiling across three production workflows. [AgentEval](https://arxiv.org/abs/2604.23581) + +AgentDebugX similarly organizes debugging as Detect → Attribute → Recover → Rerun and emphasizes that the step where an error surfaces may not be the cause. [AgentDebugX](https://arxiv.org/abs/2607.18754) + +**OFW decision:** retain strict independent evidence for `VERIFIED_FAILURE`; keep diagnosis as a proposal; add a separate, content-bound cluster-review transition before eval export. Do not infer a verified failure from an error span or user behavior alone. + +### 1.2 Full trajectories matter, but summaries should be indexes + +Meta-Harness stores each candidate’s source, scores, and execution traces in a filesystem and lets a coding agent retrieve evidence selectively. In its classification ablation, scores-only reached 34.6 median accuracy, scores plus generated summaries reached 34.9, and full trace access reached 50.0. The paper’s inference is that compression can remove the information required to locate an upstream cause. [Meta-Harness](https://arxiv.org/abs/2603.28052) + +AHE resolves the scale problem with layered experience observability: a compact corpus supports navigation, while the raw evidence remains available for drill-down. It also adds decision observability, binding an edit’s predicted effects to later task-level outcomes. [Agentic Harness Engineering](https://arxiv.org/abs/2604.25850) + +**OFW decision:** create a typed experience index containing source trace/snapshot references, baseline and candidate verifier feedback, case deltas, and raw benchmark-result paths. It is an index, not a lossy replacement for the existing manifests. + +### 1.3 Trace shape and privacy are part of mining correctness + +OpenTelemetry’s GenAI conventions define agent, model, retrieval, and tool spans, including stable agent/version, conversation, tool-name, call-id, error, and usage attributes. Message content, system instructions, tool arguments, and results are opt-in because they may contain sensitive data. [OpenTelemetry GenAI conventions](https://github.com/open-telemetry/semantic-conventions-genai/blob/main/docs/gen-ai/gen-ai-agent-spans.md), [GenAI span conventions](https://github.com/open-telemetry/semantic-conventions-genai/blob/main/docs/gen-ai/gen-ai-spans.md) + +Langfuse likewise recommends stable trace/observation names, environment separation, meaningful input/output, and evaluation context because downstream datasets and evaluators depend on trace structure. [Langfuse observability best practices](https://langfuse.com/docs/observability/best-practices) + +**OFW decision:** do not expand content capture in these PRs. First preserve the existing metadata-only privacy boundary. A later opt-in content policy must define redaction, retention, consent, and which signals may enter review. + +## 2. Turning production failures into evals + +### 2.1 Production examples need curation and provenance + +Both Langfuse and LangSmith support turning selected production traces into dataset items while retaining source trace/observation linkage. LangSmith additionally supports annotation queues in which reviewers correct inputs, outputs, and references before adding them to a dataset. [Langfuse datasets](https://langfuse.com/docs/evaluation/experiments/datasets), [LangSmith dataset management](https://docs.langchain.com/langsmith/manage-datasets-in-application), [LangSmith annotation queues](https://docs.langchain.com/langsmith/annotation-queues) + +Langfuse versions dataset membership, and experiments bind dataset items, traces, observations, and scores. That lineage is necessary to explain which production example produced an eval and which version was run. [Langfuse experiment data model](https://langfuse.com/docs/evaluation/experiments/data-model) + +**OFW decision:** preserve the existing immutable snapshot and partition ledger. Add review provenance at the cluster boundary rather than copying examples into a second database. + +### 2.2 A good agent task validates outcome and process + +Anthropic distinguishes the transcript from the outcome: an agent may claim success while the environment state proves otherwise. It recommends multiple graders, multiple trials, reference solutions, balanced positive/negative cases, separate capability and regression suites, transcript inspection, and ongoing task maintenance. It suggests that 20–50 real tasks can be enough to begin when effect sizes are large. [Anthropic, “Demystifying evals for AI agents”](https://www.anthropic.com/engineering/demystifying-evals-for-ai-agents) + +OpenAI’s grader APIs similarly separate dataset schema from testing criteria and support deterministic, model-based, Python, and composite graders. OpenAI’s GDPval uses blinded expert pairwise comparisons and detailed rubrics; its automated grader remains an estimate rather than a replacement for experts. [OpenAI Evals API](https://platform.openai.com/docs/api-reference/evals), [OpenAI Graders](https://platform.openai.com/docs/api-reference/graders), [GDPval](https://openai.com/index/gdpval/) + +Agent trajectory research adds another warning: outcome-only metrics can hide inefficient or unsound reasoning, while step-level dependencies can expose propagated failures. [Trajectory-Aware Comprehensive Evaluation](https://arxiv.org/abs/2602.21230), [AgentDiagnose](https://aclanthology.org/2025.emnlp-demos.15/) + +**OFW decision:** keep the frozen environment/verifier as the outcome authority and retain complete trial records. Cluster confirmation will control dataset admission; a future step-level grader can use the same evidence-anchor model without changing the ledger. + +### 2.3 Holdouts must remain selection instruments, not tuning data + +Langfuse dataset versions and experiment runs, LangSmith dataset splits, and standard model-selection practice all separate search data from final evaluation. Meta-Harness explicitly deduplicates and decontaminates its held-out math corpus. General model-selection guidance warns that repeated selection on a holdout turns it into training data. [Langfuse datasets](https://langfuse.com/docs/evaluation/experiments/datasets), [LangSmith datasets](https://docs.langchain.com/langsmith/manage-datasets-in-application), [Model evaluation and selection](https://arxiv.org/abs/1811.12808), [Meta-Harness](https://arxiv.org/abs/2603.28052) + +**OFW decision:** retain the existing family-level `PartitionLedger`, one-time admission, and candidate-facing holdout isolation. Cluster review must not reveal selection or admission payloads to the proposer. + +## 3. Tuning agent behavior without weight training + +### 3.1 Textual feedback carries more information than scalar reward + +GEPA reflects over trajectories, tool outputs, and domain-specific text feedback, then evolves candidates using Pareto selection. The paper reports a 6% average improvement over GRPO with up to 35× fewer rollouts. DSPy’s official GEPA guidance specifically notes that component-level feedback lets the optimizer identify which aspect needs improvement. [GEPA](https://arxiv.org/abs/2507.19457), [DSPy GEPA documentation](https://dspy.ai/tutorials/gepa_ai_program/) + +Meta-Harness goes further for code-level harnesses: its proposer reads prior source and raw traces rather than relying on a fixed critique template. AHE makes the editable component action space explicit and attributes each predicted fix/regression against later results. [Meta-Harness](https://arxiv.org/abs/2603.28052), [AHE](https://arxiv.org/abs/2604.25850) + +ACE shows why memory updates should be incremental rather than repeated full rewrites: its generator/reflector/curator design aims to avoid brevity bias and context collapse. [Agentic Context Engineering](https://arxiv.org/abs/2510.04618) + +**OFW decision:** do not add a provider-specific proposer to core. Add a durable provider-neutral experience index. Candidate generators can then inspect dense feedback and raw artifacts through ordinary file tools. + +### 3.2 Optimize the harness surface, not only the system prompt + +AHE’s ablations attribute gains to tools, middleware, and long-term memory rather than the system prompt; its frozen harness also transfers across model families. Meta-Harness searches harness code, while GEPA’s published experiments focus on text/prompt evolution. [AHE](https://arxiv.org/abs/2604.25850), [Meta-Harness](https://arxiv.org/abs/2603.28052), [GEPA](https://arxiv.org/abs/2507.19457) + +**OFW decision:** retain file-level editable components and frozen verifier/runtime/model boundaries. The new work will not add a prompt-only optimizer or weight fine-tuning. + +## 4. Paired offline comparison and online A/B testing + +### 4.1 Offline replay is paired evaluation, not a production A/B test + +OFW currently evaluates champion and candidate on the same case/repeat identity. That pairing is valuable because it removes case-mix variation, but it is still offline replay. Anthropic describes production A/B testing as real-traffic comparison that measures user outcomes and may require days or weeks to reach significance. [Anthropic agent eval guidance](https://www.anthropic.com/engineering/demystifying-evals-for-ai-agents) + +**OFW decision:** name the current evidence “paired offline comparison.” Do not claim live causal impact until a deployment adapter supplies randomized exposure and outcome records. + +### 4.2 Stochastic agents require repeated trials and uncertainty + +Anthropic recommends multiple trials and distinguishes pass@k (at least one success) from pass^k (all trials succeed). Which one matters depends on whether occasional success or reliable consistency is the product requirement. [Anthropic agent eval guidance](https://www.anthropic.com/engineering/demystifying-evals-for-ai-agents) + +General model-selection literature recommends uncertainty estimates, bootstrap methods, and multiple-comparison correction when choosing among many candidates. [Raschka, “Model Evaluation, Model Selection, and Algorithm Selection”](https://arxiv.org/abs/1811.12808) + +Chatbot Arena demonstrates pairwise estimation with confidence intervals and notes the cost of simultaneous comparisons; its adaptive sampling focuses observations on close competitors. [Chatbot Arena](https://arxiv.org/abs/2403.04132) + +**OFW decision:** record an exact paired sign-test probability from discordant champion/candidate outcomes. Keep effect-size thresholds and critical-regression rules primary. Statistical gating is opt-in and must declare a minimum number of discordant pairs; absence of evidence is not evidence of equality. + +### 4.3 Model judges require calibration and order control + +LangSmith’s comparative evaluation API can randomize answer order to mitigate positional bias. Independent research confirms that position bias varies materially across judges and tasks. Human-anchored calibration remains necessary, especially for subjective rubrics. [LangSmith pairwise evaluation](https://docs.langchain.com/langsmith/evaluate-pairwise), [Judging the Judges](https://arxiv.org/abs/2406.07791) + +**OFW decision:** continue preferring deterministic verifiers for hard gates. Model-judge adapters remain possible, but a future adapter must freeze the rubric/model, balance presentation order, and retain calibration evidence. + +## 5. Current OFW gap audit + +| Area | Current evidence | Gap | Decision | +|---|---|---|---| +| Failure truth | Revision attribution, trusted score sources, ambiguous/conflicting quarantine | Diagnosis `PROPOSED` state does not prevent eval export | PR14: content-bound cluster review and export admission gate | +| Failure attribution | Typed mechanisms, components, source traces, observation/score anchors, lineage | No human/independent confirmation transition | PR14 | +| Dataset leakage | Family-level immutable ledger and one-shot admission | No change required | Preserve | +| Trials | Repeat index and paired attempt identity exist | No uncertainty or discordant-pair report | PR15: exact paired evidence | +| Candidate tuning | File allowlist, manifest, expected effects, full result manifests | No layered next-iteration index of feedback/raw evidence | PR16: experience index | +| Judge quality | Frozen Python/command verifiers | No calibrated model-judge adapter | Defer until a real adapter is requested | +| Implicit dissatisfaction | Metadata/status/scores collected; content minimized | Corrections/rephrasing/abandonment unavailable and sensitive | Defer behind opt-in content policy | +| Online A/B | Offline pairing, PR/deploy adapter, post-monitor job type | No random exposure or production outcome contract | Defer until deployment owner supplies traffic/outcomes | + +## 6. Stacked implementation plan + +### PR14 — Confirmed failure clusters before eval admission + +- Add immutable `ClusterReview` bound to cluster id, revision, content digest, reviewer, decision, and time. +- Add a pure curation operation that emits a new content-addressed `DiagnosisResult` with confirmed/rejected states. +- Route unconfirmed and rejected failure clusters to `REVIEW`; only `CONFIRMED`/`TARGETED` clusters may enter eval, memory, selection, or admission partitions. +- TDD: proposed stays review, valid confirmation exports, stale/forged review fails, rejected stays review, holdout payload remains hidden. + +### PR15 — Uncertainty-aware paired comparison + +- Add typed `PairedEvidence` per partition: wins, losses, ties, discordant count, net delta, exact one-sided sign-test probability. +- Add explicit `StatisticalGateMode`: effect-size-only or exact-sign-test. +- Require minimum discordant evidence and maximum probability only in exact mode; preserve critical-regression and cost/latency gates. +- TDD: all ties, one-sided wins, symmetric evidence, insufficient evidence, repeated stochastic trials, and policy digest/cache binding. + +### PR16 — Drill-down optimization experience index + +- Write one content-bound experience manifest per Fit campaign. +- Index cluster/source trace/snapshot, case partition, baseline/candidate verdict, all verifier feedback, prediction error, and raw benchmark result paths. +- Validate the index on cached Fit reads and expose a typed reader for provider-specific proposers. +- TDD: feedback preserved byte-for-byte, source trace linkage, rejected/winner histories, artifact tamper rejection, no holdout payload copied into proposer-visible fields. + +### PR17 — Research-backed end-to-end release update + +- Update the offline trace-to-review fixture to confirm clusters, emit paired evidence, and validate the experience index. +- Re-run full typing, security, package, and coverage gates. + +## 7. Rejected over-engineering + +- No vector database for clustering or history retrieval; filesystem manifests and exact identifiers are sufficient in local v0. +- No generic online experimentation service; production exposure allocation is not currently owned by OFW. +- No automatic implicit-signal promotion to verified failure. +- No provider-specific optimizer in core. +- No weight tuning. +- No uncalibrated LLM judge as the sole promotion gate. + +## Sources + +1. [Anthropic: Demystifying evals for AI agents](https://www.anthropic.com/engineering/demystifying-evals-for-ai-agents) +2. [Agentic Harness Engineering](https://arxiv.org/abs/2604.25850) +3. [Meta-Harness](https://arxiv.org/abs/2603.28052) +4. [GEPA](https://arxiv.org/abs/2507.19457) +5. [DSPy GEPA documentation](https://dspy.ai/tutorials/gepa_ai_program/) +6. [Agentic Context Engineering](https://arxiv.org/abs/2510.04618) +7. [TRACE: Trajectory Attribution for Automated Context Engineering](https://arxiv.org/abs/2608.09153) +8. [AgentEval](https://arxiv.org/abs/2604.23581) +9. [AgentDebugX](https://arxiv.org/abs/2607.18754) +10. [AgentDiagnose](https://aclanthology.org/2025.emnlp-demos.15/) +11. [Automated structural testing of LLM-based agents](https://arxiv.org/abs/2601.18827) +12. [Langfuse evaluation overview](https://langfuse.com/docs/evaluation/overview) +13. [Langfuse datasets](https://langfuse.com/docs/evaluation/experiments/datasets) +14. [Langfuse experiment data model](https://langfuse.com/docs/evaluation/experiments/data-model) +15. [LangSmith evaluation concepts](https://docs.langchain.com/langsmith/evaluation-concepts) +16. [LangSmith dataset management](https://docs.langchain.com/langsmith/manage-datasets-in-application) +17. [LangSmith annotation queues](https://docs.langchain.com/langsmith/annotation-queues) +18. [LangSmith pairwise evaluation](https://docs.langchain.com/langsmith/evaluate-pairwise) +19. [OpenTelemetry GenAI agent conventions](https://github.com/open-telemetry/semantic-conventions-genai/blob/main/docs/gen-ai/gen-ai-agent-spans.md) +20. [OpenAI Evals API](https://platform.openai.com/docs/api-reference/evals) +21. [OpenAI Graders](https://platform.openai.com/docs/api-reference/graders) +22. [OpenAI GDPval grading](https://openai.com/index/gdpval/) +23. [Chatbot Arena](https://arxiv.org/abs/2403.04132) +24. [Judging the Judges](https://arxiv.org/abs/2406.07791) +25. [Model evaluation and selection](https://arxiv.org/abs/1811.12808) +26. [Trajectory-Aware Comprehensive Evaluation](https://arxiv.org/abs/2602.21230) + +## Methodology and confidence + +The research used 12 search queries and deep-read 18 primary papers or official documentation pages. Product-documentation claims are treated as descriptions of product behavior, not independent empirical evidence. Empirical claims come from papers and are reported with their study scope. The design recommendations are OFW inferences, explicitly labeled as decisions above. + +Confidence is high for the three immediate gaps because each is supported by multiple independent sources and directly observable in the current code. Confidence is medium for implicit dissatisfaction mining because the strongest recent evidence uses content that OFW intentionally does not collect. Confidence is low for a generic online A/B implementation without a concrete deployment/traffic owner; it is therefore deferred. diff --git a/src/ofw/__init__.py b/src/ofw/__init__.py index aeffd89..aac52c8 100644 --- a/src/ofw/__init__.py +++ b/src/ofw/__init__.py @@ -53,11 +53,15 @@ ) from ofw.diagnosis import ( ClusterId, + ClusterReview, + ClusterReviewDecision, + ClusterReviewerId, ClusterRevisionRef, ClusterState, DiagnosisError, DiagnosisErrorCode, DiagnosisResult, + DiagnosisReview, DiagnosisRun, EvidenceAnchor, EvidenceAnchorKind, @@ -287,6 +291,9 @@ def promote( "ClusterPartitionRule", "ClusterFamilyId", "ClusterId", + "ClusterReview", + "ClusterReviewDecision", + "ClusterReviewerId", "ClusterState", "ChangePrediction", "ConsentStatus", @@ -299,6 +306,7 @@ def promote( "CommandVerifier", "DockerCompose", "DiagnosisResult", + "DiagnosisReview", "DiagnosisRun", "DataLicense", "DiagnosisError", diff --git a/src/ofw/diagnosis.py b/src/ofw/diagnosis.py index 255ea58..bcc260b 100644 --- a/src/ofw/diagnosis.py +++ b/src/ofw/diagnosis.py @@ -5,7 +5,7 @@ import hashlib import math import sys -from dataclasses import dataclass +from dataclasses import dataclass, replace from datetime import datetime from enum import IntEnum, StrEnum from pathlib import Path @@ -59,6 +59,12 @@ class ClusterState(StrEnum): TARGETED = "targeted" RESOLVED = "resolved" REOPENED = "reopened" + REJECTED = "rejected" + + +class ClusterReviewDecision(StrEnum): + CONFIRM = "confirm" + REJECT = "reject" class DiagnosisStatus(StrEnum): @@ -70,6 +76,7 @@ class DiagnosisErrorCode(StrEnum): STALE_HARNESS = "stale_harness" REVISION_MISMATCH = "revision_mismatch" ARTIFACT_INVALID = "artifact_invalid" + REVIEW_INVALID = "review_invalid" class DiagnosisError(Exception): @@ -200,6 +207,35 @@ def __str__(self) -> str: return self.value +@dataclass(frozen=True, slots=True) +class ClusterReviewerId: + value: str + + def __post_init__(self) -> None: + if not self.value.strip() or "\0" in self.value: + raise DiagnosisError(DiagnosisErrorCode.REVIEW_INVALID, self.value) + + def __str__(self) -> str: + return self.value + + +@dataclass(frozen=True, slots=True) +class ClusterReview: + cluster_id: ClusterId + cluster_revision: int + cluster_content_digest: Sha256Digest + reviewer_id: ClusterReviewerId + decision: ClusterReviewDecision + reviewed_at: datetime + + def __post_init__(self) -> None: + if self.cluster_revision < 1 or self.reviewed_at.utcoffset() is None: + raise DiagnosisError( + DiagnosisErrorCode.REVIEW_INVALID, + self.cluster_id.value, + ) + + @dataclass(frozen=True, slots=True) class ClusterRevisionRef: id: ClusterId @@ -244,6 +280,7 @@ class DiagnosisResult: diagnoses: tuple[TraceDiagnosis, ...] clusters: tuple[FailureCluster, ...] root: Path + reviews: tuple[ClusterReview, ...] = () @property def abstained_count(self) -> int: @@ -263,6 +300,34 @@ def to_json(self) -> str: tuple[TraceDiagnosis, ...] ) _RESULT_ADAPTER: TypeAdapter[DiagnosisResult] = TypeAdapter(DiagnosisResult) +_REVIEWS_ADAPTER: TypeAdapter[tuple[ClusterReview, ...]] = TypeAdapter(tuple[ClusterReview, ...]) + + +@dataclass(frozen=True, slots=True) +class DiagnosisReview: + source: DiagnosisResult + reviews: tuple[ClusterReview, ...] + + def run(self) -> DiagnosisResult: + _validate_reviews(self.source, self.reviews) + combined = (*self.source.reviews, *self.reviews) + review_digest = digest_bytes(_REVIEWS_ADAPTER.dump_json(combined)) + run_id = DiagnosisRunId( + "diagnosis_" + + hashlib.sha256( + f"{self.source.id}\0{review_digest}\0{int(DiagnosisSchemaVersion.V1)}".encode() + ).hexdigest() + ) + result = replace( + self.source, + id=run_id, + clusters=tuple( + _reviewed_cluster(cluster, self.reviews) for cluster in self.source.clusters + ), + reviews=combined, + ) + write_artifact(result.manifest_path, f"{result.to_json()}\n".encode()) + return result @dataclass(frozen=True, slots=True) @@ -313,6 +378,7 @@ def run(self) -> DiagnosisResult: diagnoses, clusters, revision.root, + () if self.previous is None else self.previous.reviews, ) write_artifact(result.manifest_path, f"{result.to_json()}\n".encode()) return result @@ -332,6 +398,53 @@ def _diagnose( return diagnosis +def _validate_reviews( + source: DiagnosisResult, + reviews: tuple[ClusterReview, ...], +) -> None: + keys = tuple((review.cluster_id, review.cluster_revision) for review in reviews) + previous_keys = tuple((review.cluster_id, review.cluster_revision) for review in source.reviews) + if not reviews or len(set(keys)) != len(keys) or any(key in previous_keys for key in keys): + raise DiagnosisError(DiagnosisErrorCode.REVIEW_INVALID, str(source.id)) + for review in reviews: + cluster = next( + (item for item in source.clusters if item.id == review.cluster_id), + None, + ) + if ( + cluster is None + or cluster.revision != review.cluster_revision + or cluster.content_digest != review.cluster_content_digest + or cluster.state not in (ClusterState.PROPOSED, ClusterState.REOPENED) + ): + raise DiagnosisError( + DiagnosisErrorCode.REVIEW_INVALID, + review.cluster_id.value, + ) + + +def _reviewed_cluster( + cluster: FailureCluster, + reviews: tuple[ClusterReview, ...], +) -> FailureCluster: + review = next( + ( + item + for item in reviews + if item.cluster_id == cluster.id and item.cluster_revision == cluster.revision + ), + None, + ) + if review is None: + return cluster + state = ( + ClusterState.CONFIRMED + if review.decision is ClusterReviewDecision.CONFIRM + else ClusterState.REJECTED + ) + return replace(cluster, state=state) + + def _clusters( diagnoses: tuple[TraceDiagnosis, ...], diagnoser_digest: Sha256Digest, @@ -401,7 +514,11 @@ def _cluster_revision( state = ( ClusterState.PROPOSED if previous is None - else (ClusterState.REOPENED if previous.state is ClusterState.RESOLVED else previous.state) + else ( + ClusterState.REOPENED + if previous.state in (ClusterState.RESOLVED, ClusterState.REJECTED) + else previous.state + ) ) return FailureCluster( _cluster_id(mechanism), @@ -423,7 +540,7 @@ def _cluster_revision( def _resolved_cluster(previous: FailureCluster) -> FailureCluster: - if previous.state is ClusterState.RESOLVED: + if previous.state in (ClusterState.RESOLVED, ClusterState.REJECTED): return previous return FailureCluster( previous.id, diff --git a/src/ofw/exports.py b/src/ofw/exports.py index f92077f..e3be8ba 100644 --- a/src/ofw/exports.py +++ b/src/ofw/exports.py @@ -11,7 +11,7 @@ from pydantic import TypeAdapter from ofw.contracts import ComponentKind, HarnessRevision, HarnessRevisionId, Sha256Digest -from ofw.diagnosis import DiagnosisResult, FailureCluster, read_snapshot +from ofw.diagnosis import ClusterState, DiagnosisResult, FailureCluster, read_snapshot from ofw.mine import ( MineResult, SnapshotObservation, @@ -385,6 +385,17 @@ def _eligible(self) -> tuple[TraceAdmission, ...]: def _ledger_entry(self, admission: TraceAdmission) -> LedgerEntry: snapshot = read_snapshot(admission, self.mine) family_id = _trace_family(snapshot) + cluster = _cluster_for_trace(self.diagnosis, admission.trace_id) + if admission.partition is TracePartition.VERIFIED_FAILURE and ( + cluster is None or cluster.state not in (ClusterState.CONFIRMED, ClusterState.TARGETED) + ): + return LedgerEntry( + admission.trace_id, + family_id, + None if cluster is None else ClusterFamilyId(cluster.id.value), + ExportPartition.REVIEW, + _snapshot_reference(admission), + ) previous = self._previous_partition(family_id, admission.trace_id) if previous is not None: is_good = admission.partition is TracePartition.VERIFIED_GOOD @@ -393,7 +404,6 @@ def _ledger_entry(self, admission: TraceAdmission) -> LedgerEntry: LeakageErrorCode.FAMILY_CONFLICT, family_id.value, ) - cluster = _cluster_for_trace(self.diagnosis, admission.trace_id) cluster_family = None if cluster is None else ClusterFamilyId(cluster.id.value) return LedgerEntry( admission.trace_id, @@ -410,7 +420,6 @@ def _ledger_entry(self, admission: TraceAdmission) -> LedgerEntry: ExportPartition.TRAINING, _snapshot_reference(admission), ) - cluster = _cluster_for_trace(self.diagnosis, admission.trace_id) if cluster is None: return LedgerEntry( admission.trace_id, diff --git a/tests/test_diagnosis.py b/tests/test_diagnosis.py index 05a5c1a..49b8916 100644 --- a/tests/test_diagnosis.py +++ b/tests/test_diagnosis.py @@ -13,10 +13,14 @@ from pydantic import TypeAdapter from ofw import ( + ClusterReview, + ClusterReviewDecision, + ClusterReviewerId, ClusterRevisionRef, ClusterState, DiagnosisError, DiagnosisErrorCode, + DiagnosisReview, DiagnosisRun, FunctionName, Harness, @@ -226,6 +230,68 @@ def test_verified_failures_form_evidence_bound_mechanism_clusters(tmp_path: Path assert TraceId("good") not in tuple(diagnosis.trace_id for diagnosis in result.diagnoses) +def test_cluster_review_is_content_bound_and_deterministic(tmp_path: Path) -> None: + mine, harness = _mine_result(tmp_path) + diagnosis = DiagnosisRun(harness, mine, _diagnoser("diagnose")).run() + cluster = diagnosis.clusters[0] + review = ClusterReview( + cluster.id, + cluster.revision, + cluster.content_digest, + ClusterReviewerId("reviewer-1"), + ClusterReviewDecision.CONFIRM, + datetime(2026, 8, 22, 1, tzinfo=UTC), + ) + operation = DiagnosisReview(diagnosis, (review,)) + + first = operation.run() + second = operation.run() + + reviewed = next(item for item in first.clusters if item.id == cluster.id) + assert first == second + assert reviewed.state is ClusterState.CONFIRMED + assert first.reviews == (review,) + assert first.id != diagnosis.id + + +def test_stale_cluster_review_is_rejected(tmp_path: Path) -> None: + mine, harness = _mine_result(tmp_path) + diagnosis = DiagnosisRun(harness, mine, _diagnoser("diagnose")).run() + cluster = diagnosis.clusters[0] + stale = ClusterReview( + cluster.id, + cluster.revision, + Sha256Digest("sha256:stale"), + ClusterReviewerId("reviewer-1"), + ClusterReviewDecision.CONFIRM, + datetime(2026, 8, 22, 1, tzinfo=UTC), + ) + + with pytest.raises(DiagnosisError) as raised: + DiagnosisReview(diagnosis, (stale,)).run() + + assert raised.value.code is DiagnosisErrorCode.REVIEW_INVALID + + +def test_rejected_cluster_remains_a_review_artifact(tmp_path: Path) -> None: + mine, harness = _mine_result(tmp_path) + diagnosis = DiagnosisRun(harness, mine, _diagnoser("diagnose")).run() + cluster = diagnosis.clusters[0] + rejection = ClusterReview( + cluster.id, + cluster.revision, + cluster.content_digest, + ClusterReviewerId("reviewer-1"), + ClusterReviewDecision.REJECT, + datetime(2026, 8, 22, 1, tzinfo=UTC), + ) + + result = DiagnosisReview(diagnosis, (rejection,)).run() + + reviewed = next(item for item in result.clusters if item.id == cluster.id) + assert reviewed.state is ClusterState.REJECTED + + def test_invalid_evidence_anchor_becomes_abstention(tmp_path: Path) -> None: mine, harness = _mine_result(tmp_path) diff --git a/tests/test_e2e_release.py b/tests/test_e2e_release.py index 1ae07b6..dcf8903 100644 --- a/tests/test_e2e_release.py +++ b/tests/test_e2e_release.py @@ -20,9 +20,13 @@ ChangePrediction, ClusterFamilyId, ClusterPartitionRule, + ClusterReview, + ClusterReviewDecision, + ClusterReviewerId, ComponentKind, ConsentStatus, DataLicense, + DiagnosisReview, EvidenceOrigin, ExportPartition, ExportPolicy, @@ -334,6 +338,20 @@ def test_offline_trace_to_review_release( ) diagnosis_run = DiagnosisRun(harness, mine, diagnoser) diagnosis = diagnosis_run.run() + diagnosis = DiagnosisReview( + diagnosis, + tuple( + ClusterReview( + cluster.id, + cluster.revision, + cluster.content_digest, + ClusterReviewerId("offline-reviewer"), + ClusterReviewDecision.CONFIRM, + _NOW, + ) + for cluster in diagnosis.clusters + ), + ).run() partitions = ( ("frontier", ExportPartition.FRONTIER), ("regression", ExportPartition.REGRESSION), @@ -396,7 +414,7 @@ def test_offline_trace_to_review_release( 0.0, ), ) - fit_policy = FitPolicy(0.5, 1.0, 0, 1.0, 0.0, 1.0, 1.0) + fit_policy = FitPolicy(0.5, 1.0, 0, 10.0, 0.0, 1.0, 1.0) campaign = ofw.fit( harness, bundle, diff --git a/tests/test_exports.py b/tests/test_exports.py index a86b25e..9b2e3eb 100644 --- a/tests/test_exports.py +++ b/tests/test_exports.py @@ -263,6 +263,27 @@ def test_holdout_artifacts_are_separate_from_developer_suite(tmp_path: Path) -> ) +def test_unconfirmed_failure_clusters_stay_in_review(tmp_path: Path) -> None: + revision, mine, diagnosis = _inputs(tmp_path) + proposed = replace( + diagnosis, + clusters=tuple( + replace(cluster, state=ClusterState.PROPOSED) for cluster in diagnosis.clusters + ), + ) + + bundle = MineExports(revision, mine, proposed, _policy()).run() + + failure_entries = tuple( + entry for entry in bundle.ledger.entries if entry.cluster_family_id is not None + ) + assert failure_entries + assert all(entry.partition is ExportPartition.REVIEW for entry in failure_entries) + assert not bundle.developer_evals.cases + assert not bundle.selection_holdout.cases + assert not bundle.admission_holdout.cases + + def test_export_bundle_is_idempotent_and_content_addressed(tmp_path: Path) -> None: revision, mine, diagnosis = _inputs(tmp_path) exports = MineExports(revision, mine, diagnosis, _policy()) diff --git a/tests/test_fit.py b/tests/test_fit.py index 2d776eb..e774beb 100644 --- a/tests/test_fit.py +++ b/tests/test_fit.py @@ -234,7 +234,7 @@ def _fit_policy() -> FitPolicy: minimum_target_delta=0.5, minimum_regression_score=1.0, maximum_critical_regressions=0, - maximum_latency_delta=1.0, + maximum_latency_delta=10.0, maximum_cost_delta=0.0, minimum_selection_pass_rate=1.0, minimum_admission_pass_rate=1.0, From a7e354b3b9ce6521a207cbfba7d64527f46ad93c Mon Sep 17 00:00:00 2001 From: divo12 Date: Sat, 22 Aug 2026 23:48:29 +0530 Subject: [PATCH 2/2] close cluster review transition gaps --- ...failure-mining-evals-tuning-ab-research.md | 4 +- src/ofw/diagnosis.py | 8 +++- src/ofw/exports.py | 7 ++++ tests/test_diagnosis.py | 42 +++++++++++++++++++ tests/test_exports.py | 33 +++++++++++++++ 5 files changed, 91 insertions(+), 3 deletions(-) diff --git a/docs/research/2026-08-22-failure-mining-evals-tuning-ab-research.md b/docs/research/2026-08-22-failure-mining-evals-tuning-ab-research.md index 8f52770..73572e3 100644 --- a/docs/research/2026-08-22-failure-mining-evals-tuning-ab-research.md +++ b/docs/research/2026-08-22-failure-mining-evals-tuning-ab-research.md @@ -1,7 +1,7 @@ # Failure mining, agent evals, behavior tuning, and A/B testing -Generated: 2026-08-22 -Scope: production agent traces → trusted failures → evals → harness edits → paired comparison → governed promotion +Generated: 2026-08-22 +Scope: production agent traces → trusted failures → evals → harness edits → paired comparison → governed promotion Method: current OFW code audit plus primary papers and official platform documentation ## Executive conclusion diff --git a/src/ofw/diagnosis.py b/src/ofw/diagnosis.py index bcc260b..860fb81 100644 --- a/src/ofw/diagnosis.py +++ b/src/ofw/diagnosis.py @@ -516,7 +516,13 @@ def _cluster_revision( if previous is None else ( ClusterState.REOPENED - if previous.state in (ClusterState.RESOLVED, ClusterState.REJECTED) + if previous.state + in ( + ClusterState.CONFIRMED, + ClusterState.TARGETED, + ClusterState.RESOLVED, + ClusterState.REJECTED, + ) else previous.state ) ) diff --git a/src/ofw/exports.py b/src/ofw/exports.py index e3be8ba..7f9cf0a 100644 --- a/src/ofw/exports.py +++ b/src/ofw/exports.py @@ -397,6 +397,13 @@ def _ledger_entry(self, admission: TraceAdmission) -> LedgerEntry: _snapshot_reference(admission), ) previous = self._previous_partition(family_id, admission.trace_id) + if ( + previous is ExportPartition.REVIEW + and admission.partition is TracePartition.VERIFIED_FAILURE + and cluster is not None + and cluster.state in (ClusterState.CONFIRMED, ClusterState.TARGETED) + ): + previous = None if previous is not None: is_good = admission.partition is TracePartition.VERIFIED_GOOD if is_good != (previous is ExportPartition.TRAINING): diff --git a/tests/test_diagnosis.py b/tests/test_diagnosis.py index 49b8916..3829b33 100644 --- a/tests/test_diagnosis.py +++ b/tests/test_diagnosis.py @@ -85,6 +85,17 @@ def _repository(tmp_path: Path) -> tuple[Path, HarnessRevisionId]: " (EvidenceAnchor(EvidenceAnchorKind.OBSERVATION, observation.id.value),),\n" " (component,), Severity.HIGH, 0.9,\n" " )\n" + "def diagnose_changed(snapshot: TraceSnapshot) -> TraceDiagnosis:\n" + " observation = snapshot.observations[0]\n" + " mechanism = 'tool-schema' if observation.name == 'tool' else 'prompt-gap'\n" + " component = (\n" + " ComponentKind.TOOL if observation.name == 'tool' else ComponentKind.PROMPT\n" + " )\n" + " return TraceDiagnosis.proposed(\n" + " snapshot.trace.id, MechanismKey(mechanism), mechanism, 'changed diagnosis',\n" + " (EvidenceAnchor(EvidenceAnchorKind.OBSERVATION, observation.id.value),),\n" + " (component,), Severity.HIGH, 0.9,\n" + " )\n" "def invalid_anchor(snapshot: TraceSnapshot) -> TraceDiagnosis:\n" " return TraceDiagnosis.proposed(\n" " snapshot.trace.id, MechanismKey('invalid-anchor'), 'invalid', 'invalid',\n" @@ -292,6 +303,37 @@ def test_rejected_cluster_remains_a_review_artifact(tmp_path: Path) -> None: assert reviewed.state is ClusterState.REJECTED +def test_changed_confirmed_cluster_requires_a_new_review(tmp_path: Path) -> None: + mine, harness = _mine_result(tmp_path) + diagnosis = DiagnosisRun(harness, mine, _diagnoser("diagnose")).run() + cluster = diagnosis.clusters[0] + confirmed = DiagnosisReview( + diagnosis, + ( + ClusterReview( + cluster.id, + cluster.revision, + cluster.content_digest, + ClusterReviewerId("reviewer-1"), + ClusterReviewDecision.CONFIRM, + datetime(2026, 8, 22, 1, tzinfo=UTC), + ), + ), + ).run() + + changed = DiagnosisRun( + harness, + mine, + _diagnoser("diagnose_changed"), + previous=confirmed, + ).run() + + changed_cluster = next(item for item in changed.clusters if item.id == cluster.id) + assert changed_cluster.revision == cluster.revision + 1 + assert changed_cluster.content_digest != cluster.content_digest + assert changed_cluster.state is ClusterState.REOPENED + + def test_invalid_evidence_anchor_becomes_abstention(tmp_path: Path) -> None: mine, harness = _mine_result(tmp_path) diff --git a/tests/test_exports.py b/tests/test_exports.py index 9b2e3eb..2851cbe 100644 --- a/tests/test_exports.py +++ b/tests/test_exports.py @@ -284,6 +284,39 @@ def test_unconfirmed_failure_clusters_stay_in_review(tmp_path: Path) -> None: assert not bundle.admission_holdout.cases +def test_confirmed_cluster_can_graduate_from_previous_review_partition(tmp_path: Path) -> None: + revision, mine, diagnosis = _inputs(tmp_path) + proposed = replace( + diagnosis, + clusters=tuple( + replace(cluster, state=ClusterState.PROPOSED) for cluster in diagnosis.clusters + ), + ) + review_bundle = MineExports(revision, mine, proposed, _policy()).run() + confirmed = replace( + diagnosis, + clusters=tuple( + replace(cluster, state=ClusterState.CONFIRMED) for cluster in diagnosis.clusters + ), + ) + + graduated = MineExports( + revision, + mine, + confirmed, + _policy(), + previous=review_bundle, + ).run() + + failure_entries = tuple( + entry for entry in graduated.ledger.entries if entry.cluster_family_id is not None + ) + assert failure_entries + assert all(entry.partition is not ExportPartition.REVIEW for entry in failure_entries) + assert graduated.developer_evals.cases + assert graduated.admission_holdout.cases + + def test_export_bundle_is_idempotent_and_content_addressed(tmp_path: Path) -> None: revision, mine, diagnosis = _inputs(tmp_path) exports = MineExports(revision, mine, diagnosis, _policy())