Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions docs/specs/2026-08-22-offline-e2e-release.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
# OpenFlyWheel offline end-to-end release proof

The permanent release gate is:

```bash
uv run pytest -q tests/test_e2e_release.py
```

The fixture uses no external provider account or production credentials. A loopback HTTP server exposes the read-only Langfuse health, observation, and score endpoints. One stamped harness revision then completes this exact lineage:

```text
Langfuse-compatible traces
→ revision-attributed collection
→ Mine admission and immutable snapshots
→ evidence-bound diagnosis and clusters
→ leakage-safe training/eval/selection/admission exports
→ controlled tool-file candidate
→ paired baseline/candidate gates and one-shot admission
→ durable scheduler PROMOTE job
→ isolated Git commit, review PR reference, and reverse patch
```

The planted fixture contains one verified-good trace and four failures assigned to frontier, regression, selection, and admission partitions. The candidate fixes frontier and sealed holdouts while preserving the regression case. A real heartbeat materializes the seven-job DAG; the scheduler restarts before promotion and preserves its budget ledger. The test proves one winner, one PR, no deploy, no Langfuse write, and a non-empty rollback artifact.

This is the local-v0 release boundary. Distributed scheduling, cloud workspaces, provider-backed candidate generation, and production deployment remain explicit adapters rather than hidden behavior in the offline proof.
158 changes: 50 additions & 108 deletions src/ofw/__init__.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
"""Public OpenFlyWheel harness API."""

import sys
from collections.abc import Sequence
from datetime import datetime
from pathlib import Path
from threading import Event
from typing import TYPE_CHECKING

from langfuse import (
Langfuse,
Expand All @@ -24,6 +27,7 @@
BenchmarkStatus,
)
from ofw.candidate import (
CandidateBuild,
CandidateBuilder,
CandidateError,
CandidateErrorCode,
Expand Down Expand Up @@ -92,7 +96,6 @@
Mine,
MineError,
MineErrorCode,
MineResult,
MiningPolicy,
ScoreName,
SnapshotContentReference,
Expand All @@ -117,8 +120,6 @@
ObservationContentMatch,
ObservationContentQuery,
ObservationContentReference,
ObservationRecord,
TraceId,
)
from ofw.observability.langfuse.service import (
collect,
Expand Down Expand Up @@ -226,116 +227,53 @@
PromotionService = GitPromotionService


class _OfwNamespace:
__slots__ = ()
def fit(
harness: Harness,
bundle: ExportBundle,
candidates: tuple[CandidateBuild, ...],
*,
benchmark_policy: BenchmarkPolicy,
policy: FitPolicy,
) -> FitCampaign:
"""Create a side-effect-free durable Fit campaign handle."""
return FitCampaign(harness, bundle, benchmark_policy, policy, candidates)

LocalProcess = LocalProcess
DockerCompose = DockerCompose
ProcessLimits = ProcessLimits
ProcessCommand = ProcessCommand
CommandLoop = CommandLoop
PythonLoop = PythonLoop
PythonEntrypoint = PythonEntrypoint
ModuleName = ModuleName
FunctionName = FunctionName
ModelFingerprint = ModelFingerprint
CommandVerifier = CommandVerifier
PythonVerifier = PythonVerifier
CanaryCase = CanaryCase
CaseId = CaseId
ServiceName = ServiceName
MiningPolicy = MiningPolicy
ScoreName = ScoreName
PythonDiagnoser = PythonDiagnoser
ExportPolicy = ExportPolicy
BenchmarkPolicy = BenchmarkPolicy
BenchmarkRunner = BenchmarkRunner
CandidatePolicy = CandidatePolicy
CandidateBuilder = CandidateBuilder
FitPolicy = FitPolicy
AutomationPolicy = SchedulerAutomationPolicy
LocalScheduler = SQLiteScheduler
Money = Money
QuietHours = QuietHours
StageBudgets = StageBudgets
PromotionPolicy = PromotionPolicy
PromotionService = GitPromotionService
def serve(
harnesses: Sequence[Harness],
policy: SchedulerAutomationPolicy,
evidence: EvidenceReader,
stop: Event,
*,
store_path: Path,
owner: HeartbeatOwner,
) -> None:
revisions: tuple[HarnessRevisionId, ...] = ()
for harness in harnesses:
revision = harness.current_revision
if revision is None:
raise SchedulerError(SchedulerErrorCode.STALE_HARNESS, harness.name)
revisions = (*revisions, revision.id)
scheduler = SQLiteScheduler(store_path, policy)
try:
SchedulerDaemon(scheduler, owner, revisions, evidence).serve(stop)
finally:
scheduler.close()

def editable(self, path: Path) -> EditableFile:
return editable(path)

def collect(
self,
revision: HarnessRevision,
*,
window: TraceWindow,
store_path: Path | None = None,
) -> CollectionResult:
return collect(revision, window=window, store_path=store_path)
def promote(
request: PromotionRequest,
*,
now: datetime,
pull_requests: PullRequestPublisher | None = None,
deployments: DeploymentAdapter | None = None,
) -> PromotionResult:
return GitPromotionService(pull_requests, deployments).run(request, now)

def serve(
self,
harnesses: tuple[Harness, ...],
policy: SchedulerAutomationPolicy,
evidence: EvidenceReader,
stop: Event,
*,
store_path: Path,
owner: HeartbeatOwner,
) -> None:
revisions: tuple[HarnessRevisionId, ...] = ()
for harness in harnesses:
revision = harness.current_revision
if revision is None:
raise SchedulerError(SchedulerErrorCode.STALE_HARNESS, harness.name)
revisions = (*revisions, revision.id)
scheduler = SQLiteScheduler(store_path, policy)
try:
SchedulerDaemon(scheduler, owner, revisions, evidence).serve(stop)
finally:
scheduler.close()

def promote(
self,
request: PromotionRequest,
*,
now: datetime,
pull_requests: PullRequestPublisher | None = None,
deployments: DeploymentAdapter | None = None,
) -> PromotionResult:
return GitPromotionService(pull_requests, deployments).run(request, now)

def search_observation_content(
self,
collection: CollectionResult,
query: ObservationContentQuery,
) -> tuple[ObservationContentHit, ...]:
return search_observation_content(collection, query)

def read_trace_observations(
self,
collection: CollectionResult,
trace_id: TraceId,
limit: int,
) -> tuple[ObservationRecord, ...]:
return read_trace_observations(collection, trace_id, limit)

def read_observation_content(
self,
collection: CollectionResult,
reference: ObservationContentReference,
) -> ObservationContent:
return read_observation_content(collection, reference)

def read_snapshot_content(
self,
result: MineResult,
reference: SnapshotContentReference,
) -> ObservationContent:
return read_snapshot_content(result, reference)


ofw = _OfwNamespace()
if TYPE_CHECKING:
import ofw as ofw
else:
ofw = sys.modules[__name__]

__all__ = [
"AssetAccess",
Expand All @@ -352,6 +290,7 @@ def read_snapshot_content(
"BenchmarkStatus",
"BlockerCode",
"BudgetStatus",
"CandidateBuild",
"CandidateBuilder",
"CandidateError",
"CandidateErrorCode",
Expand Down Expand Up @@ -512,13 +451,16 @@ def read_snapshot_content(
"WorkerId",
"collect",
"editable",
"fit",
"get_client",
"is_default_export_span",
"observe",
"ofw",
"promote",
"propagate_attributes",
"read_observation_content",
"read_snapshot_content",
"read_trace_observations",
"search_observation_content",
"serve",
]
3 changes: 3 additions & 0 deletions src/ofw/fit.py
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,9 @@ class FitCampaign:
fit_policy: FitPolicy
candidates: tuple[CandidateBuild, ...]

def wait(self) -> FitResult:
return self.run()

def run(self) -> FitResult:
existing = self._read_existing()
if existing is not None:
Expand Down
2 changes: 2 additions & 0 deletions src/ofw/harness.py
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,8 @@ def connect_middleware(self, *sources: Path | EditableFile) -> Harness:
self._register_files(ComponentKind.MIDDLEWARE, sources)
return self

connect_middle = connect_middleware

def connect_observability(self, project: LangfuseProject) -> Harness:
self._current_revision = None
self._observability = project
Expand Down
Loading