From 6875ecf4bb2dc784d5f47815c41c517f1eb2201f Mon Sep 17 00:00:00 2001 From: divo12 Date: Sun, 23 Aug 2026 01:11:36 +0530 Subject: [PATCH 1/2] index fit experience without holdout leakage --- ...failure-mining-evals-tuning-ab-research.md | 4 +- docs/specs/2026-08-23-fit-experience-index.md | 32 ++ src/ofw/__init__.py | 18 ++ src/ofw/benchmarking.py | 3 + src/ofw/fit.py | 293 +++++++++++++++++- tests/test_fit.py | 66 +++- 6 files changed, 409 insertions(+), 7 deletions(-) create mode 100644 docs/specs/2026-08-23-fit-experience-index.md diff --git a/docs/research/2026-08-22-failure-mining-evals-tuning-ab-research.md b/docs/research/2026-08-22-failure-mining-evals-tuning-ab-research.md index 2e25c73..02c44e7 100644 --- a/docs/research/2026-08-22-failure-mining-evals-tuning-ab-research.md +++ b/docs/research/2026-08-22-failure-mining-evals-tuning-ab-research.md @@ -37,7 +37,7 @@ Meta-Harness stores each candidate’s source, scores, and execution traces in a AHE resolves the scale problem with layered experience observability: a compact corpus supports navigation, while the raw evidence remains available for drill-down. It also adds decision observability, binding an edit’s predicted effects to later task-level outcomes. [Agentic Harness Engineering](https://arxiv.org/abs/2604.25850) -**OFW decision:** create a typed experience index containing source trace/snapshot references, baseline and candidate verifier feedback, case deltas, and raw benchmark-result paths. It is an index, not a lossy replacement for the existing manifests. +**OFW decision:** create a typed experience index containing developer source trace/snapshot references, paired baseline and candidate verifier feedback, case deltas, and raw developer benchmark-result paths. Selection and admission remain sealed and contribute only their pass/fail decisions. The index is not a lossy replacement for the existing manifests. ### 1.3 Trace shape and privacy are part of mining correctness @@ -182,7 +182,7 @@ LangSmith’s comparative evaluation API can randomize answer order to mitigate ### PR17 — Drill-down optimization experience index - Write one content-bound experience manifest per Fit campaign. -- Index cluster/source trace/snapshot, case partition, baseline/candidate verdict, all verifier feedback, prediction error, and raw benchmark result paths. +- Index developer cluster/source trace/snapshot, case partition, paired baseline/candidate verdicts, all developer verifier feedback, prediction error, and raw developer benchmark result paths. - Validate the index on cached Fit reads and expose a typed reader for provider-specific proposers. - TDD: feedback preserved byte-for-byte, source trace linkage, rejected/winner histories, artifact tamper rejection, no holdout payload copied into proposer-visible fields. diff --git a/docs/specs/2026-08-23-fit-experience-index.md b/docs/specs/2026-08-23-fit-experience-index.md new file mode 100644 index 0000000..3a17042 --- /dev/null +++ b/docs/specs/2026-08-23-fit-experience-index.md @@ -0,0 +1,32 @@ +# Fit experience index + +Each completed `FitCampaign` writes one content-bound `experience.json` beside its result manifest. The index is the provider-neutral observation surface for the next candidate proposer. It does not choose edits or summarize away source evidence. + +## Developer-visible evidence + +For every attempted candidate, the index preserves: + +- the frozen candidate manifest and prediction attribution; +- final candidate status and gate reason; +- the candidate diff artifact reference; +- content-digested references to the raw champion and candidate developer benchmark results; +- each developer case’s cluster family, source trace, trace family, snapshot reference, and partition; and +- every paired champion/candidate run result, verifier verdict, score, textual feedback, metric, evidence reference, and case delta. + +Textual verifier feedback is stored unchanged. The compact index is navigational: the raw developer snapshot, benchmark results, and candidate diff remain available for drill-down. + +## Holdout boundary + +Selection and admission cases are evaluator-only. The proposer-visible index records only whether each stage ran, its completion status, and whether its frozen threshold passed. It never stores holdout case IDs, trace IDs, family IDs, snapshot references, prompts, outputs, verifier feedback, or raw benchmark paths. + +This asymmetry is intentional. Developer evidence teaches the next proposer; holdouts decide whether a frozen candidate survives. Returning holdout diagnostics would turn repeated selection into training and invalidate the gate. + +## Integrity + +`FitResult.experience_digest` binds the exact index bytes. `read_fit_experience(result)` validates campaign, export bundle, input digest, harness revision, candidate ordering, statuses, gate reasons, prediction attribution, developer-result linkage, and every referenced developer artifact digest. Cached `FitCampaign.run()` performs the same validation before returning a prior result. + +Tampering with the index, candidate diff, or raw developer benchmark result therefore fails with `FitErrorCode.RESULT_INVALID`. Existing candidate/revision validation remains authoritative for the harness, export snapshots, and candidate manifest. + +## Why this is the minimum useful shape + +A scalar leaderboard cannot tell an optimizer which tool, prompt, skill, subagent, or middleware behavior caused a failure. Copying all artifacts into a second store adds drift without adding evidence. The index instead supplies AHE-style component and decision observability while retaining Meta-Harness-style drill-down to raw developer traces. A filesystem manifest and exact identifiers are sufficient for local v0; no vector database, summarizer, or provider-specific proposer belongs in this PR. diff --git a/src/ofw/__init__.py b/src/ofw/__init__.py index 3664655..df08a79 100644 --- a/src/ofw/__init__.py +++ b/src/ofw/__init__.py @@ -88,20 +88,29 @@ PrivacyTransform, ) from ofw.fit import ( + CandidateExperience, CandidateOutcome, CandidateStatus, CaseDelta, + DeveloperCaseExperience, + FitArtifactReference, FitCampaign, FitError, FitErrorCode, + FitExperience, + FitExperienceSchemaVersion, FitPolicy, FitResult, GateReason, + HoldoutDecision, + HoldoutStage, + PairedAttemptExperience, PairedEvidence, PairedEvidencePolicy, StatisticalGateMode, paired_evidence, paired_evidence_passes, + read_fit_experience, ) from ofw.harness import EditableFile, Harness, Subagent, Tool, editable from ofw.mine import ( @@ -290,6 +299,7 @@ def promote( "CandidateError", "CandidateErrorCode", "CandidateEvidence", + "CandidateExperience", "CandidatePolicy", "CandidateOutcome", "CandidateStatus", @@ -324,6 +334,7 @@ def promote( "DeploymentAdapter", "DeploymentReference", "DeploymentRequest", + "DeveloperCaseExperience", "EditableFile", "EvidenceAnchor", "EvidenceAnchorKind", @@ -335,9 +346,12 @@ def promote( "FailureCluster", "FailureDisposition", "FileEdit", + "FitArtifactReference", "FitCampaign", "FitError", "FitErrorCode", + "FitExperience", + "FitExperienceSchemaVersion", "FitPolicy", "FitResult", "GitCommit", @@ -357,10 +371,13 @@ def promote( "HermesAgentVersion", "HermesDiagnoser", "hermes_python_command", + "HoldoutDecision", + "HoldoutStage", "FunctionName", "GateReason", "PairedEvidence", "PairedEvidencePolicy", + "PairedAttemptExperience", "Langfuse", "LangfuseOtelSpanAttributes", "LangfuseProject", @@ -455,5 +472,6 @@ def promote( "paired_evidence_passes", "promote", "propagate_attributes", + "read_fit_experience", "serve", ] diff --git a/src/ofw/benchmarking.py b/src/ofw/benchmarking.py index 162b917..42d8216 100644 --- a/src/ofw/benchmarking.py +++ b/src/ofw/benchmarking.py @@ -79,6 +79,7 @@ def digest(self) -> Sha256Digest: @dataclass(frozen=True, slots=True) class CaseAttempt: case_id: str + source_case_id: str partition: ExportPartition critical: bool repeat: int @@ -242,6 +243,7 @@ def _run_suite( attempts.append( CaseAttempt( case_id, + case.id, case.partition, case.critical, repeat, @@ -365,6 +367,7 @@ def _semantic(attempts: tuple[CaseAttempt, ...]) -> tuple[CaseAttempt, ...]: return tuple( CaseAttempt( attempt.case_id, + attempt.source_case_id, attempt.partition, attempt.critical, attempt.repeat, diff --git a/src/ofw/fit.py b/src/ofw/fit.py index e3e568d..3ae4fbe 100644 --- a/src/ofw/fit.py +++ b/src/ofw/fit.py @@ -5,7 +5,7 @@ import hashlib import math from dataclasses import dataclass, replace -from enum import StrEnum +from enum import IntEnum, StrEnum from pathlib import Path from pydantic import TypeAdapter, ValidationError @@ -22,15 +22,24 @@ CandidateBuild, CandidateError, CandidateId, + CandidateManifest, read_candidate_manifest, validate_candidate_artifacts, validate_candidate_revision, ) -from ofw.contracts import HarnessRevision, Sha256Digest -from ofw.exports import ExportBundle, ExportPartition +from ofw.contracts import HarnessRevision, HarnessRevisionId, Sha256Digest +from ofw.exports import ( + ClusterFamilyId, + EvalCase, + ExportBundle, + ExportPartition, + SnapshotReference, + TraceFamilyId, +) from ofw.harness import Harness from ofw.mine import digest_bytes, write_artifact -from ofw.runtime import MetricKind, VerifierResult +from ofw.observability.langfuse.domain import TraceId +from ofw.runtime import MetricKind, RunResult, VerifierResult class FitErrorCode(StrEnum): @@ -79,6 +88,15 @@ class AdmissionState(StrEnum): ERROR = "error" +class FitExperienceSchemaVersion(IntEnum): + V1 = 1 + + +class HoldoutStage(StrEnum): + SELECTION = "selection" + ADMISSION = "admission" + + @dataclass(frozen=True, slots=True) class PairedEvidencePolicy: mode: StatisticalGateMode @@ -210,15 +228,86 @@ class CandidateOutcome: admission_result: BenchmarkResult | None = None +@dataclass(frozen=True, slots=True) +class FitArtifactReference: + path: Path + digest: Sha256Digest + + +@dataclass(frozen=True, slots=True) +class PairedAttemptExperience: + case_id: str + repeat: int + synthetic: bool + weight: float + baseline: RunResult + candidate: RunResult + baseline_verifiers: tuple[VerifierResult, ...] + candidate_verifiers: tuple[VerifierResult, ...] + delta: CaseDelta + + +@dataclass(frozen=True, slots=True) +class DeveloperCaseExperience: + case_id: str + trace_id: TraceId + family_id: TraceFamilyId + cluster_family_id: ClusterFamilyId + partition: ExportPartition + snapshot: SnapshotReference + attempts: tuple[PairedAttemptExperience, ...] + + +@dataclass(frozen=True, slots=True) +class HoldoutDecision: + stage: HoldoutStage + status: BenchmarkStatus + passed: bool + + +@dataclass(frozen=True, slots=True) +class CandidateExperience: + candidate_id: CandidateId + status: CandidateStatus + reason: GateReason + manifest: CandidateManifest + attribution: ManifestAttribution + candidate_diff: FitArtifactReference + developer_baseline: FitArtifactReference + developer_candidate: FitArtifactReference + developer_cases: tuple[DeveloperCaseExperience, ...] + holdouts: tuple[HoldoutDecision, ...] + + +@dataclass(frozen=True, slots=True) +class FitExperience: + schema_version: FitExperienceSchemaVersion + fit_id: str + export_bundle_id: str + input_digest: Sha256Digest + revision_id: HarnessRevisionId + candidates: tuple[CandidateExperience, ...] + root: Path + + @property + def path(self) -> Path: + return self.root / ".ofw" / "fit" / self.fit_id / "experience.json" + + def to_json(self) -> str: + return _EXPERIENCE_ADAPTER.dump_json(self).decode() + + @dataclass(frozen=True, slots=True) class FitResult: id: str + export_bundle_id: str benchmark_id: str policy_digest: Sha256Digest input_digest: Sha256Digest baseline: Baseline outcomes: tuple[CandidateOutcome, ...] winner_id: CandidateId | None + experience_digest: Sha256Digest root: Path @property @@ -229,6 +318,10 @@ def manifest_path(self) -> Path: def digest_path(self) -> Path: return self.manifest_path.with_suffix(".sha256") + @property + def experience_path(self) -> Path: + return self.manifest_path.with_name("experience.json") + def to_json(self) -> str: return _FIT_ADAPTER.dump_json(self).decode() @@ -279,6 +372,7 @@ def digest(self) -> Sha256Digest: _FIT_ADAPTER: TypeAdapter[FitResult] = TypeAdapter(FitResult) +_EXPERIENCE_ADAPTER: TypeAdapter[FitExperience] = TypeAdapter(FitExperience) _ADMISSION_ADAPTER: TypeAdapter[AdmissionRecord] = TypeAdapter(AdmissionRecord) _DIGEST_ADAPTER: TypeAdapter[Sha256Digest] = TypeAdapter(Sha256Digest) _INPUT_ADAPTER: TypeAdapter[FitInputFingerprint] = TypeAdapter(FitInputFingerprint) @@ -409,14 +503,27 @@ def _run(self) -> FitResult: ) outcomes = _replace_outcome(outcomes, rejected) finalist.build.workspace.close() + experience = _fit_experience( + self._campaign_id(), + self.bundle, + input_digest, + champion, + self.candidates, + outcomes, + self.fit_policy, + ) + experience_payload = f"{experience.to_json()}\n".encode() + write_artifact(experience.path, experience_payload) result = FitResult( self._campaign_id(), + self.bundle.id, baseline.benchmark_id, self.fit_policy.digest, input_digest, baseline, outcomes, winner_id, + digest_bytes(experience_payload), self.harness.root, ) payload = f"{result.to_json()}\n".encode() @@ -467,6 +574,7 @@ def _read_existing(self) -> FitResult | None: input_digest = self._validate_inputs(champion_revision) if ( result.id != self._campaign_id() + or result.export_bundle_id != self.bundle.id or result.benchmark_id != self.bundle.benchmark.id or result.policy_digest != self.fit_policy.digest or result.input_digest != input_digest @@ -482,6 +590,7 @@ def _read_existing(self) -> FitResult | None: ) if not winner.workspace.root.exists(): raise FitError(FitErrorCode.CANDIDATE_DRIFT, winner.candidate.id.value) + read_fit_experience(result) return result def _validate_inputs(self, champion_revision: HarnessRevision) -> Sha256Digest: @@ -529,6 +638,182 @@ def _validate_inputs(self, champion_revision: HarnessRevision) -> Sha256Digest: ).digest +def read_fit_experience(result: FitResult) -> FitExperience: + try: + payload = result.experience_path.read_bytes() + experience = _EXPERIENCE_ADAPTER.validate_json(payload) + except (OSError, ValidationError) as error: + raise FitError(FitErrorCode.RESULT_INVALID, str(result.experience_path)) from error + outcome_ids = tuple(outcome.candidate_id for outcome in result.outcomes) + if ( + digest_bytes(payload) != result.experience_digest + or experience.fit_id != result.id + or experience.export_bundle_id != result.export_bundle_id + or experience.input_digest != result.input_digest + or experience.revision_id != result.baseline.revision_id + or tuple(candidate.candidate_id for candidate in experience.candidates) != outcome_ids + ): + raise FitError(FitErrorCode.RESULT_INVALID, str(result.experience_path)) + for candidate, outcome in zip(experience.candidates, result.outcomes, strict=True): + if ( + candidate.status is not outcome.status + or candidate.reason is not outcome.reason + or candidate.attribution != outcome.attribution + or candidate.developer_candidate.path != outcome.developer_result.manifest_path + ): + raise FitError(FitErrorCode.RESULT_INVALID, candidate.candidate_id.value) + _validate_experience_artifacts(result.root, candidate) + return experience + + +def _fit_experience( + fit_id: str, + bundle: ExportBundle, + input_digest: Sha256Digest, + baseline: BenchmarkResult, + builds: tuple[CandidateBuild, ...], + outcomes: tuple[CandidateOutcome, ...], + policy: FitPolicy, +) -> FitExperience: + candidates = tuple( + _candidate_experience( + build, + next(outcome for outcome in outcomes if outcome.candidate_id == build.candidate.id), + bundle.developer_evals.cases, + baseline, + policy, + ) + for build in builds + ) + return FitExperience( + FitExperienceSchemaVersion.V1, + fit_id, + bundle.id, + input_digest, + bundle.revision_id, + candidates, + bundle.root, + ) + + +def _candidate_experience( + build: CandidateBuild, + outcome: CandidateOutcome, + cases: tuple[EvalCase, ...], + baseline: BenchmarkResult, + policy: FitPolicy, +) -> CandidateExperience: + return CandidateExperience( + outcome.candidate_id, + outcome.status, + outcome.reason, + read_candidate_manifest(build.candidate.manifest_path), + outcome.attribution, + _artifact_reference(build.candidate.diff_path), + _artifact_reference(baseline.manifest_path), + _artifact_reference(outcome.developer_result.manifest_path), + tuple( + _developer_case_experience(case, baseline, outcome.developer_result) + for case in cases + ), + _holdout_decisions(outcome, policy), + ) + + +def _developer_case_experience( + case: EvalCase, + baseline: BenchmarkResult, + candidate: BenchmarkResult, +) -> DeveloperCaseExperience: + attempts = tuple( + _paired_attempt_experience(baseline_attempt, candidate_attempt) + for baseline_attempt in baseline.attempts + for candidate_attempt in candidate.attempts + if baseline_attempt.source_case_id == case.id + and candidate_attempt.source_case_id == case.id + and _attempt_key(baseline_attempt) == _attempt_key(candidate_attempt) + ) + return DeveloperCaseExperience( + case.id, + case.trace_id, + case.family_id, + case.cluster_family_id, + case.partition, + case.snapshot, + attempts, + ) + + +def _paired_attempt_experience( + baseline: CaseAttempt, + candidate: CaseAttempt, +) -> PairedAttemptExperience: + return PairedAttemptExperience( + baseline.case_id, + baseline.repeat, + baseline.synthetic, + baseline.weight, + baseline.run, + candidate.run, + baseline.verifiers, + candidate.verifiers, + _case_delta(baseline, candidate), + ) + + +def _holdout_decisions( + outcome: CandidateOutcome, + policy: FitPolicy, +) -> tuple[HoldoutDecision, ...]: + decisions: tuple[HoldoutDecision, ...] = () + if outcome.selection_result is not None: + decisions = ( + HoldoutDecision( + HoldoutStage.SELECTION, + outcome.selection_result.status, + outcome.selection_result.status is BenchmarkStatus.COMPLETE + and outcome.selection_result.weighted_pass_rate + >= policy.minimum_selection_pass_rate, + ), + ) + if outcome.admission_result is not None: + decisions = ( + *decisions, + HoldoutDecision( + HoldoutStage.ADMISSION, + outcome.admission_result.status, + outcome.admission_result.status is BenchmarkStatus.COMPLETE + and outcome.admission_result.weighted_pass_rate + >= policy.minimum_admission_pass_rate, + ), + ) + return decisions + + +def _artifact_reference(path: Path) -> FitArtifactReference: + try: + return FitArtifactReference(path, digest_bytes(path.read_bytes())) + except OSError as error: + raise FitError(FitErrorCode.RESULT_INVALID, str(path)) from error + + +def _validate_experience_artifacts(root: Path, candidate: CandidateExperience) -> None: + references = ( + candidate.candidate_diff, + candidate.developer_baseline, + candidate.developer_candidate, + ) + try: + allowed = (root / ".ofw").resolve(strict=True) + for reference in references: + path = reference.path.resolve(strict=True) + path.relative_to(allowed) + if digest_bytes(path.read_bytes()) != reference.digest: + raise FitError(FitErrorCode.RESULT_INVALID, str(path)) + except (OSError, ValueError) as error: + raise FitError(FitErrorCode.RESULT_INVALID, candidate.candidate_id.value) from error + + def read_admission_record(path: Path) -> AdmissionRecord: try: return _ADMISSION_ADAPTER.validate_json(path.read_bytes()) diff --git a/tests/test_fit.py b/tests/test_fit.py index 07cb147..ae73dab 100644 --- a/tests/test_fit.py +++ b/tests/test_fit.py @@ -9,9 +9,11 @@ from typing import cast import pytest +from pydantic import TypeAdapter from ofw import ( BenchmarkPolicy, + BenchmarkResult, CandidateBuilder, CandidateEvidence, CandidatePolicy, @@ -36,6 +38,7 @@ StatisticalGateMode, Tool, ofw, + read_fit_experience, ) from ofw.candidate import CandidateBuild from ofw.contracts import HarnessRevision, Sha256Digest @@ -94,7 +97,9 @@ def _harness(tmp_path: Path) -> Harness: " frontier = 'frontier' in output or 'selection' in output or 'admission' in output\n" " passed = ('FIXED' in output) if frontier else ('BROKEN' not in output)\n" " verdict = VerifierVerdict.PASS if passed else VerifierVerdict.FAIL\n" - " return VerifierResult(verdict, 1.0 if passed else 0.0, 'fixture')\n", + " return VerifierResult(\n" + " verdict, 1.0 if passed else 0.0, 'feedback:' + output\n" + " )\n", encoding="utf-8", ) _run_git(root, "init", "-q") @@ -379,6 +384,7 @@ def test_paired_gates_reject_regression_and_admit_one_winner(tmp_path: Path) -> result = campaign.run() assert campaign.run() == result + experience = read_fit_experience(result) assert result.winner_id == good.candidate.id good_outcome = next( outcome for outcome in result.outcomes if outcome.candidate_id == good.candidate.id @@ -398,6 +404,50 @@ def test_paired_gates_reject_regression_and_admit_one_winner(tmp_path: Path) -> assert bad_outcome.selection_result is None assert not bad.workspace.root.exists() assert good.workspace.root.exists() + assert tuple(item.candidate_id for item in experience.candidates) == ( + good.candidate.id, + bad.candidate.id, + ) + good_experience = experience.candidates[0] + assert good_experience.status is good_outcome.status + assert good_experience.manifest.hypothesis == "Fix frontier only." + assert good_experience.developer_baseline.path.is_file() + assert good_experience.developer_candidate.path == good_outcome.developer_result.manifest_path + assert tuple(case.trace_id for case in good_experience.developer_cases) == ( + TraceId("frontier-case"), + TraceId("regression-case"), + ) + indexed_feedback = tuple( + verifier.feedback + for case in good_experience.developer_cases + for attempt in case.attempts + for verifier in attempt.candidate_verifiers + ) + source_feedback = tuple( + verifier.feedback + for attempt in good_outcome.developer_result.attempts + for verifier in attempt.verifiers + ) + assert indexed_feedback == source_feedback + baseline_result = TypeAdapter(BenchmarkResult).validate_json( + good_experience.developer_baseline.path.read_bytes() + ) + indexed_baseline_feedback = tuple( + verifier.feedback + for case in good_experience.developer_cases + for attempt in case.attempts + for verifier in attempt.baseline_verifiers + ) + source_baseline_feedback = tuple( + verifier.feedback + for attempt in baseline_result.attempts + for verifier in attempt.verifiers + ) + assert indexed_baseline_feedback == source_baseline_feedback + assert tuple(decision.passed for decision in good_experience.holdouts) == (True, True) + experience_payload = result.experience_path.read_bytes() + assert b"selection-case" not in experience_payload + assert b"admission-case" not in experience_payload assert any( delta.case_id == "frontier-case" and delta.pass_delta == 1 for delta in good_outcome.deltas ) @@ -435,6 +485,12 @@ def test_admission_failure_returns_no_winner_and_discards_finalist(tmp_path: Pat assert result.winner_id is None assert not candidate.workspace.root.exists() + original_experience = result.experience_path.read_bytes() + result.experience_path.write_bytes(original_experience + b" ") + with pytest.raises(FitError) as raised: + campaign.run() + assert raised.value.code is FitErrorCode.RESULT_INVALID + result.experience_path.write_bytes(original_experience) result.manifest_path.write_bytes(result.manifest_path.read_bytes() + b" ") with pytest.raises(FitError) as raised: campaign.run() @@ -462,6 +518,14 @@ def test_no_winner_cache_rejects_candidate_artifact_drift(tmp_path: Path) -> Non assert result.winner_id is None assert campaign.run() == result + developer_path = result.outcomes[0].developer_result.manifest_path + original_developer_result = developer_path.read_bytes() + developer_path.write_bytes(original_developer_result + b" ") + with pytest.raises(FitError) as raised: + campaign.run() + assert raised.value.code is FitErrorCode.RESULT_INVALID + developer_path.write_bytes(original_developer_result) + candidate.candidate.diff_path.write_bytes(candidate.candidate.diff_path.read_bytes() + b"\n") with pytest.raises(FitError) as raised: From 9ca014429e40a322107ee4d875de4ea64be777fb Mon Sep 17 00:00:00 2001 From: divo12 Date: Sun, 23 Aug 2026 01:23:00 +0530 Subject: [PATCH 2/2] bind experience to canonical source artifacts --- docs/specs/2026-08-23-fit-experience-index.md | 2 +- src/ofw/fit.py | 85 ++++++++++++++++-- tests/test_fit.py | 89 +++++++++++++++++++ 3 files changed, 170 insertions(+), 6 deletions(-) diff --git a/docs/specs/2026-08-23-fit-experience-index.md b/docs/specs/2026-08-23-fit-experience-index.md index 3a17042..9bbf45f 100644 --- a/docs/specs/2026-08-23-fit-experience-index.md +++ b/docs/specs/2026-08-23-fit-experience-index.md @@ -23,7 +23,7 @@ This asymmetry is intentional. Developer evidence teaches the next proposer; hol ## Integrity -`FitResult.experience_digest` binds the exact index bytes. `read_fit_experience(result)` validates campaign, export bundle, input digest, harness revision, candidate ordering, statuses, gate reasons, prediction attribution, developer-result linkage, and every referenced developer artifact digest. Cached `FitCampaign.run()` performs the same validation before returning a prior result. +`FitResult.experience_digest` binds the exact index bytes. `read_fit_experience(result)` validates campaign, export bundle, input digest, harness revision, candidate ordering, statuses, gate reasons, prediction attribution, developer-result linkage, and every referenced developer artifact digest. Cached `FitCampaign.run()` additionally rebuilds the developer case index from the raw champion/candidate benchmark artifacts and export cases before returning a prior result. Tampering with the index, candidate diff, or raw developer benchmark result therefore fails with `FitErrorCode.RESULT_INVALID`. Existing candidate/revision validation remains authoritative for the harness, export snapshots, and candidate manifest. diff --git a/src/ofw/fit.py b/src/ofw/fit.py index 3ae4fbe..369ebc2 100644 --- a/src/ofw/fit.py +++ b/src/ofw/fit.py @@ -373,6 +373,7 @@ def digest(self) -> Sha256Digest: _FIT_ADAPTER: TypeAdapter[FitResult] = TypeAdapter(FitResult) _EXPERIENCE_ADAPTER: TypeAdapter[FitExperience] = TypeAdapter(FitExperience) +_BENCHMARK_ADAPTER: TypeAdapter[BenchmarkResult] = TypeAdapter(BenchmarkResult) _ADMISSION_ADAPTER: TypeAdapter[AdmissionRecord] = TypeAdapter(AdmissionRecord) _DIGEST_ADAPTER: TypeAdapter[Sha256Digest] = TypeAdapter(Sha256Digest) _INPUT_ADAPTER: TypeAdapter[FitInputFingerprint] = TypeAdapter(FitInputFingerprint) @@ -590,7 +591,15 @@ def _read_existing(self) -> FitResult | None: ) if not winner.workspace.root.exists(): raise FitError(FitErrorCode.CANDIDATE_DRIFT, winner.candidate.id.value) - read_fit_experience(result) + experience = read_fit_experience(result) + _validate_fit_experience_sources( + experience, + self.bundle.developer_evals.cases, + self.candidates, + result.outcomes, + self.fit_policy, + result.baseline, + ) return result def _validate_inputs(self, champion_revision: HarnessRevision) -> Sha256Digest: @@ -662,7 +671,7 @@ def read_fit_experience(result: FitResult) -> FitExperience: or candidate.developer_candidate.path != outcome.developer_result.manifest_path ): raise FitError(FitErrorCode.RESULT_INVALID, candidate.candidate_id.value) - _validate_experience_artifacts(result.root, candidate) + _validate_experience_artifacts(result, candidate) return experience @@ -720,6 +729,51 @@ def _candidate_experience( ) +def _validate_fit_experience_sources( + experience: FitExperience, + cases: tuple[EvalCase, ...], + builds: tuple[CandidateBuild, ...], + outcomes: tuple[CandidateOutcome, ...], + policy: FitPolicy, + baseline: Baseline, +) -> None: + for indexed, build, outcome in zip(experience.candidates, builds, outcomes, strict=True): + if ( + indexed.manifest != read_candidate_manifest(build.candidate.manifest_path) + or indexed.candidate_diff.path != build.candidate.diff_path + or indexed.candidate_diff.digest != build.candidate.diff_digest + ): + raise FitError(FitErrorCode.RESULT_INVALID, indexed.candidate_id.value) + baseline_result = _read_benchmark_reference(indexed.developer_baseline) + candidate_result = _read_benchmark_reference(indexed.developer_candidate) + if ( + baseline_result.benchmark_id != baseline.benchmark_id + or baseline_result.revision_id != baseline.revision_id + or baseline_result.policy_digest != baseline.policy_digest + or baseline_result.semantic_digest != baseline.semantic_digest + or baseline_result.candidate_id is not None + or candidate_result != outcome.developer_result + or indexed.developer_cases + != tuple( + _developer_case_experience(case, baseline_result, candidate_result) + for case in cases + ) + or indexed.holdouts != _holdout_decisions(outcome, policy) + ): + raise FitError(FitErrorCode.RESULT_INVALID, indexed.candidate_id.value) + + +def _read_benchmark_reference(reference: FitArtifactReference) -> BenchmarkResult: + try: + payload = reference.path.read_bytes() + result = _BENCHMARK_ADAPTER.validate_json(payload) + except (OSError, ValidationError) as error: + raise FitError(FitErrorCode.RESULT_INVALID, str(reference.path)) from error + if digest_bytes(payload) != reference.digest: + raise FitError(FitErrorCode.RESULT_INVALID, str(reference.path)) + return result + + def _developer_case_experience( case: EvalCase, baseline: BenchmarkResult, @@ -797,20 +851,41 @@ def _artifact_reference(path: Path) -> FitArtifactReference: raise FitError(FitErrorCode.RESULT_INVALID, str(path)) from error -def _validate_experience_artifacts(root: Path, candidate: CandidateExperience) -> None: +def _validate_experience_artifacts( + result: FitResult, + candidate: CandidateExperience, +) -> None: references = ( candidate.candidate_diff, candidate.developer_baseline, candidate.developer_candidate, ) try: - allowed = (root / ".ofw").resolve(strict=True) + allowed = (result.root / ".ofw").resolve(strict=True) for reference in references: path = reference.path.resolve(strict=True) path.relative_to(allowed) if digest_bytes(path.read_bytes()) != reference.digest: raise FitError(FitErrorCode.RESULT_INVALID, str(path)) - except (OSError, ValueError) as error: + expected_diff = ( + allowed / "candidates" / candidate.candidate_id.value / "candidate.patch" + ).resolve(strict=True) + if candidate.candidate_diff.path.resolve(strict=True) != expected_diff: + raise FitError(FitErrorCode.RESULT_INVALID, str(candidate.candidate_diff.path)) + baseline = _BENCHMARK_ADAPTER.validate_json( + candidate.developer_baseline.path.read_bytes() + ) + if ( + candidate.developer_baseline.path.resolve(strict=True) + != baseline.manifest_path.resolve(strict=True) + or baseline.candidate_id is not None + or baseline.benchmark_id != result.benchmark_id + or baseline.revision_id != result.baseline.revision_id + or baseline.policy_digest != result.baseline.policy_digest + or baseline.semantic_digest != result.baseline.semantic_digest + ): + raise FitError(FitErrorCode.RESULT_INVALID, str(candidate.developer_baseline.path)) + except (OSError, ValidationError, ValueError) as error: raise FitError(FitErrorCode.RESULT_INVALID, candidate.candidate_id.value) from error diff --git a/tests/test_fit.py b/tests/test_fit.py index ae73dab..1434a9a 100644 --- a/tests/test_fit.py +++ b/tests/test_fit.py @@ -4,6 +4,7 @@ import hashlib import subprocess +from dataclasses import replace from datetime import timedelta from pathlib import Path from typing import cast @@ -22,6 +23,7 @@ ClusterId, ComponentKind, FileEdit, + FitArtifactReference, FitCampaign, FitError, FitErrorCode, @@ -65,6 +67,7 @@ paired_evidence_passes, read_admission_record, ) +from ofw.mine import digest_bytes from ofw.observability.langfuse.domain import TraceId @@ -448,6 +451,39 @@ def test_paired_gates_reject_regression_and_admit_one_winner(tmp_path: Path) -> experience_payload = result.experience_path.read_bytes() assert b"selection-case" not in experience_payload assert b"admission-case" not in experience_payload + copied_diff = result.experience_path.with_name("copied-candidate.patch") + copied_diff.write_bytes(good_experience.candidate_diff.path.read_bytes()) + copied_baseline = result.experience_path.with_name("copied-baseline.json") + copied_baseline.write_bytes(good_experience.developer_baseline.path.read_bytes()) + rewired_candidates = ( + replace( + good_experience, + candidate_diff=FitArtifactReference( + copied_diff, + digest_bytes(copied_diff.read_bytes()), + ), + ), + replace( + good_experience, + developer_baseline=FitArtifactReference( + copied_baseline, + digest_bytes(copied_baseline.read_bytes()), + ), + ), + ) + for rewired_candidate in rewired_candidates: + rewired = replace( + experience, + candidates=(rewired_candidate, *experience.candidates[1:]), + ) + rewired_payload = f"{rewired.to_json()}\n".encode() + result.experience_path.write_bytes(rewired_payload) + with pytest.raises(FitError) as raised: + read_fit_experience( + replace(result, experience_digest=digest_bytes(rewired_payload)) + ) + assert raised.value.code is FitErrorCode.RESULT_INVALID + result.experience_path.write_bytes(experience_payload) assert any( delta.case_id == "frontier-case" and delta.pass_delta == 1 for delta in good_outcome.deltas ) @@ -497,6 +533,59 @@ def test_admission_failure_returns_no_winner_and_discards_finalist(tmp_path: Pat assert raised.value.code is FitErrorCode.RESULT_INVALID +def test_cached_experience_must_match_raw_developer_artifacts(tmp_path: Path) -> None: + harness = _harness(tmp_path) + revision = harness.current_revision + assert revision is not None + candidate = _candidate( + revision, + "def run(value: str) -> str:\n" + " return value + (' FIXED' if 'regression' not in value else '')\n", + "Fix frontier only.", + ) + campaign = FitCampaign( + harness, + _bundle(revision), + BenchmarkPolicy(1, 10, 0, 0.25), + _fit_policy(), + (candidate,), + ) + result = campaign.run() + experience = read_fit_experience(result) + candidate_experience = experience.candidates[0] + case_experience = candidate_experience.developer_cases[0] + attempt = case_experience.attempts[0] + tampered_attempt = replace( + attempt, + candidate_verifiers=( + replace(attempt.candidate_verifiers[0], feedback="feedback:tampered"), + ), + ) + tampered_case = replace(case_experience, attempts=(tampered_attempt,)) + tampered_candidate = replace( + candidate_experience, + developer_cases=(tampered_case, *candidate_experience.developer_cases[1:]), + ) + tampered_experience = replace( + experience, + candidates=(tampered_candidate, *experience.candidates[1:]), + ) + experience_payload = f"{tampered_experience.to_json()}\n".encode() + result.experience_path.write_bytes(experience_payload) + updated_result = replace(result, experience_digest=digest_bytes(experience_payload)) + result_payload = f"{updated_result.to_json()}\n".encode() + result.manifest_path.write_bytes(result_payload) + result.digest_path.write_bytes( + TypeAdapter(Sha256Digest).dump_json(digest_bytes(result_payload)) + b"\n" + ) + + with pytest.raises(FitError) as raised: + campaign.run() + + assert raised.value.code is FitErrorCode.RESULT_INVALID + candidate.workspace.close() + + def test_no_winner_cache_rejects_candidate_artifact_drift(tmp_path: Path) -> None: harness = _harness(tmp_path) revision = harness.current_revision