From c913572e9293b849a78f2c8b2d54840f2628b03e Mon Sep 17 00:00:00 2001 From: divo12 Date: Sat, 22 Aug 2026 16:31:39 +0530 Subject: [PATCH 01/18] implement execution lifecycle and verifiers --- src/ofw/__init__.py | 58 +++++ src/ofw/_runner.py | 30 +++ src/ofw/contracts.py | 40 +++ src/ofw/harness.py | 122 +++++++-- src/ofw/runtime.py | 588 ++++++++++++++++++++++++++++++++++++++++++ tests/test_runtime.py | 252 ++++++++++++++++++ tests/test_typing.py | 2 + 7 files changed, 1074 insertions(+), 18 deletions(-) create mode 100644 src/ofw/_runner.py create mode 100644 src/ofw/runtime.py create mode 100644 tests/test_runtime.py diff --git a/src/ofw/__init__.py b/src/ofw/__init__.py index 2c2be27..83b36fc 100644 --- a/src/ofw/__init__.py +++ b/src/ofw/__init__.py @@ -35,11 +35,49 @@ ) from ofw.observability.langfuse.domain import CollectionResult from ofw.observability.langfuse.service import collect +from ofw.runtime import ( + CanaryCase, + CaseId, + CommandLoop, + CommandVerifier, + DockerCompose, + FunctionName, + LocalProcess, + ModelFingerprint, + ModuleName, + ProcessCommand, + ProcessLimits, + PythonEntrypoint, + PythonLoop, + PythonVerifier, + RunErrorCode, + RunResult, + RunStatus, + ServiceName, + VerifierResult, + VerifierVerdict, +) class _OfwNamespace: __slots__ = () + LocalProcess = LocalProcess + DockerCompose = DockerCompose + ProcessLimits = ProcessLimits + ProcessCommand = ProcessCommand + CommandLoop = CommandLoop + PythonLoop = PythonLoop + PythonEntrypoint = PythonEntrypoint + ModuleName = ModuleName + FunctionName = FunctionName + ModelFingerprint = ModelFingerprint + CommandVerifier = CommandVerifier + PythonVerifier = PythonVerifier + CanaryCase = CanaryCase + CaseId = CaseId + ServiceName = ServiceName + def editable(self, path: Path) -> EditableFile: return editable(path) @@ -57,10 +95,15 @@ def collect( __all__ = [ "AssetAccess", + "CanaryCase", + "CaseId", "ComponentKind", "CollectionError", "CollectionErrorCode", "CollectionResult", + "CommandLoop", + "CommandVerifier", + "DockerCompose", "EditableFile", "GitCommit", "Harness", @@ -70,15 +113,30 @@ def collect( "HarnessRevision", "HarnessRevisionId", "HarnessValidationError", + "FunctionName", "Langfuse", "LangfuseOtelSpanAttributes", "LangfuseProject", "LangfuseSpan", + "LocalProcess", + "ModelFingerprint", + "ModuleName", "RepositorySnapshot", + "ProcessCommand", + "ProcessLimits", + "PythonEntrypoint", + "PythonLoop", + "PythonVerifier", + "RunErrorCode", + "RunResult", + "RunStatus", "Sha256Digest", + "ServiceName", "Subagent", "Tool", "TraceWindow", + "VerifierResult", + "VerifierVerdict", "WorkspaceFile", "collect", "editable", diff --git a/src/ofw/_runner.py b/src/ofw/_runner.py new file mode 100644 index 0000000..9da394a --- /dev/null +++ b/src/ofw/_runner.py @@ -0,0 +1,30 @@ +"""Child-process entrypoint for a file-backed Python lifecycle function.""" + +from __future__ import annotations + +import importlib +import inspect +import sys +from collections.abc import Callable +from typing import cast + + +def main() -> int: + if len(sys.argv) != 3: + return 2 + module = importlib.import_module(sys.argv[1]) + functions = tuple( + function + for name, function in inspect.getmembers(module, inspect.isfunction) + if name == sys.argv[2] + ) + if len(functions) != 1: + return 2 + function: Callable[[str], str] = cast(Callable[[str], str], functions[0]) + output: str = function(sys.stdin.read()) # type: ignore[misc] + sys.stdout.write(output) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/ofw/contracts.py b/src/ofw/contracts.py index 980de0c..3b2ea54 100644 --- a/src/ofw/contracts.py +++ b/src/ofw/contracts.py @@ -47,6 +47,9 @@ class HarnessErrorCode(StrEnum): GIT_COMMAND_FAILED = "git_command_failed" MANIFEST_WRITE_FAILED = "manifest_write_failed" SENSITIVE_ASSET = "sensitive_asset" + RUNTIME_INCOMPLETE = "runtime_incomplete" + CANARY_FAILED = "canary_failed" + DUPLICATE_VERIFIER = "duplicate_verifier" class HarnessValidationError(Exception): @@ -111,6 +114,23 @@ class RepositorySnapshot: dirty_digest: Sha256Digest | None +@dataclass(frozen=True, slots=True) +class RuntimeConfiguration: + execution: Sha256Digest + lifecycle: Sha256Digest + verifiers: tuple[Sha256Digest, ...] + + def canonical_json(self) -> str: + verifiers = ",".join(_quote(str(verifier)) for verifier in self.verifiers) + return ( + "{" + f'"execution":{_quote(str(self.execution))},' + f'"lifecycle":{_quote(str(self.lifecycle))},' + f'"verifiers":[{verifiers}]' + "}" + ) + + @dataclass(frozen=True, slots=True) class HarnessRevisionContent: schema_version: HarnessSchemaVersion @@ -118,6 +138,8 @@ class HarnessRevisionContent: repository: RepositorySnapshot components: tuple[HarnessComponent, ...] observability: LangfuseConnectionManifest | None + runtime: RuntimeConfiguration | None + canary_digest: Sha256Digest | None def canonical_json(self) -> str: return _render_content(self) @@ -132,11 +154,17 @@ class HarnessRevision: repository: RepositorySnapshot components: tuple[HarnessComponent, ...] observability: LangfuseConnectionManifest | None + runtime: RuntimeConfiguration | None + canary_digest: Sha256Digest | None @property def manifest_path(self) -> Path: return self.root / ".ofw" / "revisions" / str(self.id) / "manifest.json" + @property + def canary_path(self) -> Path: + return self.manifest_path.with_name("canary.json") + @property def assets(self) -> tuple[HarnessAsset, ...]: return tuple(asset for component in self.components for asset in component.assets) @@ -173,6 +201,8 @@ def to_json(self) -> str: f'"root":{_quote(self.root.as_posix())},' f'"repository":{_render_repository(self.repository)},' f'"observability":{_render_observability(self.observability)},' + f'"runtime":{_render_runtime(self.runtime)},' + f'"canary_digest":{_render_digest(self.canary_digest)},' f'"components":[{components}]' "}" ) @@ -186,6 +216,8 @@ def _render_content(content: HarnessRevisionContent) -> str: f'"harness_name":{_quote(content.harness_name)},' f'"repository":{_render_repository(content.repository)},' f'"observability":{_render_observability(content.observability)},' + f'"runtime":{_render_runtime(content.runtime)},' + f'"canary_digest":{_render_digest(content.canary_digest)},' f'"components":[{components}]' "}" ) @@ -209,6 +241,14 @@ def _render_observability(connection: LangfuseConnectionManifest | None) -> str: return "null" if connection is None else connection.to_json() +def _render_runtime(runtime: RuntimeConfiguration | None) -> str: + return "null" if runtime is None else runtime.canonical_json() + + +def _render_digest(digest: Sha256Digest | None) -> str: + return "null" if digest is None else _quote(str(digest)) + + def _render_component(component: HarnessComponent) -> str: assets = ",".join(_render_asset(asset) for asset in component.assets) return ( diff --git a/src/ofw/harness.py b/src/ofw/harness.py index 57db9a0..01e3b55 100644 --- a/src/ofw/harness.py +++ b/src/ofw/harness.py @@ -24,10 +24,20 @@ HarnessSchemaVersion, HarnessValidationError, RepositorySnapshot, + RuntimeConfiguration, Sha256Digest, WorkspaceFile, ) from ofw.observability.langfuse.contracts import LangfuseProject +from ofw.runtime import ( + CanaryCase, + CanaryReport, + ExecutionEnvironment, + LifecycleAdapter, + VerifierAdapter, + run_canary, + runtime_configuration, +) logger = logging.getLogger(__name__) @@ -87,6 +97,9 @@ class Harness: root: Path _files: list[_FileRegistration] = field(default_factory=list, init=False, repr=False) _observability: LangfuseProject | None = field(default=None, init=False, repr=False) + _execution: ExecutionEnvironment | None = field(default=None, init=False, repr=False) + _lifecycle: LifecycleAdapter | None = field(default=None, init=False, repr=False) + _verifiers: list[VerifierAdapter] = field(default_factory=list, init=False, repr=False) def __post_init__(self) -> None: if _NAME_PATTERN.fullmatch(self.name) is None: @@ -136,6 +149,21 @@ def connect_observability(self, project: LangfuseProject) -> Harness: self._observability = project return self + def connect_execute(self, environment: ExecutionEnvironment) -> Harness: + self._execution = environment + return self + + def connect_lifecycle(self, lifecycle: LifecycleAdapter) -> Harness: + self._lifecycle = lifecycle + return self + + def connect_verifiers(self, *verifiers: VerifierAdapter) -> Harness: + for verifier in verifiers: + if any(existing.name == verifier.name for existing in self._verifiers): + raise HarnessValidationError(HarnessErrorCode.DUPLICATE_VERIFIER, verifier.name) + self._verifiers.append(verifier) + return self + def _register_files( self, component: ComponentKind, @@ -144,7 +172,7 @@ def _register_files( for source in sources: self._files.append(_registration(component, source, None)) - def process(self) -> HarnessRevision: + def process(self, *, canary: CanaryCase | None = None) -> HarnessRevision: logger.debug("Compiling harness revision: %s", self.name) root = _resolve_root(self.root) if not _has_component(self._files, ComponentKind.PROMPT): @@ -152,32 +180,84 @@ def process(self) -> HarnessRevision: components = _compile_components(root, self._files) repository = _snapshot_repository(root) + runtime = self._runtime(root) content = HarnessRevisionContent( schema_version=HarnessSchemaVersion.V1, harness_name=self.name, repository=repository, components=components, observability=(None if self._observability is None else self._observability.manifest()), + runtime=runtime, + canary_digest=None, ) - content_digest = _digest_text(content.canonical_json()) - revision = HarnessRevision( - schema_version=content.schema_version, - id=HarnessRevisionId(f"ofw_{content_digest.value[7:]}"), - harness_name=content.harness_name, - root=root, - repository=content.repository, - components=content.components, - observability=content.observability, - ) + revision = _revision_from_content(content, root) + report: CanaryReport | None = None + if canary is not None: + if self._execution is None or self._lifecycle is None or not self._verifiers: + raise HarnessValidationError(HarnessErrorCode.RUNTIME_INCOMPLETE, self.name) + report = run_canary( + revision, + canary, + self._execution, + self._lifecycle, + tuple(self._verifiers), + ) + if not report.passed: + _write_canary(revision, report) + raise HarnessValidationError(HarnessErrorCode.CANARY_FAILED, canary.id.value) + content = HarnessRevisionContent( + schema_version=content.schema_version, + harness_name=content.harness_name, + repository=content.repository, + components=content.components, + observability=content.observability, + runtime=content.runtime, + canary_digest=report.digest, + ) + revision = _revision_from_content(content, root) _write_manifest(revision) + if report is not None: + _write_canary(revision, report) logger.debug("Compiled harness revision %s", revision.id) return revision + def _runtime(self, root: Path) -> RuntimeConfiguration | None: + connections = ( + self._execution is not None, + self._lifecycle is not None, + bool(self._verifiers), + ) + if not any(connections): + return None + if not all(connections) or self._execution is None or self._lifecycle is None: + raise HarnessValidationError(HarnessErrorCode.RUNTIME_INCOMPLETE, self.name) + return runtime_configuration( + root, + self._execution, + self._lifecycle, + tuple(self._verifiers), + ) + def _has_component(registrations: list[_FileRegistration], kind: ComponentKind) -> bool: return any(registration.component is kind for registration in registrations) +def _revision_from_content(content: HarnessRevisionContent, root: Path) -> HarnessRevision: + content_digest = _digest_text(content.canonical_json()) + return HarnessRevision( + schema_version=content.schema_version, + id=HarnessRevisionId(f"ofw_{content_digest.value[7:]}"), + harness_name=content.harness_name, + root=root, + repository=content.repository, + components=content.components, + observability=content.observability, + runtime=content.runtime, + canary_digest=content.canary_digest, + ) + + def _resolve_root(root: Path) -> Path: try: resolved = root.expanduser().resolve(strict=True) @@ -389,13 +469,19 @@ def _digest_bytes(value: bytes) -> Sha256Digest: def _write_manifest(revision: HarnessRevision) -> None: - manifest_path = revision.manifest_path - payload = f"{revision.to_json()}\n" + _write_revision_file(revision.manifest_path, f"{revision.to_json()}\n") + + +def _write_canary(revision: HarnessRevision, report: CanaryReport) -> None: + _write_revision_file(revision.canary_path, f"{report.to_json()}\n") + + +def _write_revision_file(path: Path, payload: str) -> None: try: - manifest_path.parent.mkdir(parents=True, exist_ok=True) + path.parent.mkdir(parents=True, exist_ok=True) descriptor, temporary_name = tempfile.mkstemp( - dir=manifest_path.parent, - prefix=".manifest-", + dir=path.parent, + prefix=f".{path.stem}-", suffix=".json", text=True, ) @@ -405,11 +491,11 @@ def _write_manifest(revision: HarnessRevision) -> None: stream.write(payload) stream.flush() os.fsync(stream.fileno()) - temporary_path.replace(manifest_path) + temporary_path.replace(path) finally: temporary_path.unlink(missing_ok=True) except OSError as error: raise HarnessValidationError( HarnessErrorCode.MANIFEST_WRITE_FAILED, - str(manifest_path), + str(path), ) from error diff --git a/src/ofw/runtime.py b/src/ofw/runtime.py new file mode 100644 index 0000000..5157e1b --- /dev/null +++ b/src/ofw/runtime.py @@ -0,0 +1,588 @@ +"""Disposable execution, lifecycle, verifier, and canary adapters.""" + +from __future__ import annotations + +import hashlib +import inspect +import re +import shutil +import subprocess # nosec B404 +import sys +import tempfile +import time +from collections.abc import Callable +from dataclasses import dataclass +from datetime import timedelta +from enum import IntEnum, StrEnum +from pathlib import Path + +from pydantic import TypeAdapter + +from ofw.contracts import HarnessRevision, RuntimeConfiguration, Sha256Digest + +_NAME_PATTERN = re.compile(r"[a-z][a-z0-9_-]*") +_MODULE_PATTERN = re.compile(r"[a-zA-Z_][a-zA-Z0-9_.]*") +_FUNCTION_PATTERN = re.compile(r"[a-zA-Z_][a-zA-Z0-9_]*") + + +class RunStatus(StrEnum): + SUCCESS = "success" + ERROR = "error" + TIMEOUT = "timeout" + + +class RunErrorCode(StrEnum): + NON_ZERO_EXIT = "non_zero_exit" + TIMEOUT = "timeout" + + +class VerifierVerdict(StrEnum): + PASS = "pass" # nosec B105 + FAIL = "fail" + ABSTAIN = "abstain" + ERROR = "error" + + +class VerifierExitCode(IntEnum): + PASS = 0 + FAIL = 1 + ABSTAIN = 2 + + +@dataclass(frozen=True, slots=True) +class CaseId: + value: str + + def __post_init__(self) -> None: + if _NAME_PATTERN.fullmatch(self.value) is None: + raise ValueError("invalid case id") + + +@dataclass(frozen=True, slots=True) +class CanaryCase: + id: CaseId + payload: str + + +@dataclass(frozen=True, slots=True) +class ModuleName: + value: str + + def __post_init__(self) -> None: + if _MODULE_PATTERN.fullmatch(self.value) is None: + raise ValueError("invalid module name") + + +@dataclass(frozen=True, slots=True) +class FunctionName: + value: str + + def __post_init__(self) -> None: + if _FUNCTION_PATTERN.fullmatch(self.value) is None: + raise ValueError("invalid function name") + + +@dataclass(frozen=True, slots=True) +class PythonEntrypoint: + module: ModuleName + function: FunctionName + + +@dataclass(frozen=True, slots=True) +class ServiceName: + value: str + + def __post_init__(self) -> None: + if _NAME_PATTERN.fullmatch(self.value) is None: + raise ValueError("invalid service name") + + +@dataclass(frozen=True, slots=True) +class ModelFingerprint: + provider: str + model: str + reasoning: str + + def __post_init__(self) -> None: + if not self.provider or not self.model or not self.reasoning: + raise ValueError("model fingerprint fields are required") + + +@dataclass(frozen=True, slots=True) +class ProcessCommand: + arguments: tuple[str, ...] + + def __post_init__(self) -> None: + if not self.arguments or any(not argument for argument in self.arguments): + raise ValueError("command arguments are required") + + +@dataclass(frozen=True, slots=True) +class ProcessLimits: + timeout: timedelta + + def __post_init__(self) -> None: + if self.timeout <= timedelta(0): + raise ValueError("timeout must be positive") + + +@dataclass(frozen=True, slots=True) +class ProcessResult: + exit_code: int | None + stdout: str + timed_out: bool + duration_seconds: float + + +@dataclass(frozen=True, slots=True) +class RunResult: + case_id: CaseId + status: RunStatus + output: str | None + error_code: RunErrorCode | None + duration_seconds: float + + @classmethod + def success(cls, case_id: CaseId, output: str) -> RunResult: + return cls(case_id, RunStatus.SUCCESS, output, None, 0.0) + + +@dataclass(frozen=True, slots=True) +class Metric: + name: str + value: float + + +@dataclass(frozen=True, slots=True) +class EvidenceReference: + value: str + + +@dataclass(frozen=True, slots=True) +class VerifierResult: + verdict: VerifierVerdict + score: float | None + feedback: str + metrics: tuple[Metric, ...] = () + evidence: tuple[EvidenceReference, ...] = () + retryable: bool = False + + +@dataclass(frozen=True, slots=True) +class CheckReport: + passed: bool + detail: str + + +@dataclass(frozen=True, slots=True) +class EnvironmentFingerprint: + digest: Sha256Digest + + +@dataclass(frozen=True, slots=True) +class PreparedEnvironment: + root: Path + source_root: Path + temporary: tempfile.TemporaryDirectory[str] + limits: ProcessLimits + command_prefix: tuple[str, ...] = () + + def run(self, command: ProcessCommand, payload: str) -> ProcessResult: + started = time.monotonic() + try: + # The revision freezes argv; every call uses shell=False. + completed = subprocess.run( # nosec B603 + (*self.command_prefix, *command.arguments), + cwd=self.root, + input=payload, + capture_output=True, + text=True, + timeout=self.limits.timeout.total_seconds(), + check=False, + ) + except subprocess.TimeoutExpired: + return ProcessResult(None, "", True, time.monotonic() - started) + return ProcessResult( + completed.returncode, + completed.stdout.rstrip("\n"), + False, + time.monotonic() - started, + ) + + +@dataclass(frozen=True, slots=True) +class LocalProcess: + limits: ProcessLimits + + def fingerprint(self, root: Path) -> Sha256Digest: + del root + return _digest_text( + f"local\0{self.limits.timeout.total_seconds()}\0{sys.version}\0{sys.platform}" + ) + + def prepare(self, revision: HarnessRevision, case: CanaryCase) -> PreparedEnvironment: + del case + return _prepare_workspace(revision.root, self.limits) + + def health(self, prepared: PreparedEnvironment) -> CheckReport: + return CheckReport(prepared.root.is_dir(), "local workspace") + + def snapshot(self, prepared: PreparedEnvironment) -> EnvironmentFingerprint: + return EnvironmentFingerprint(_digest_tree(prepared.root)) + + def reset(self, prepared: PreparedEnvironment) -> None: + _restore_workspace(prepared) + + def destroy(self, prepared: PreparedEnvironment) -> None: + prepared.temporary.cleanup() + + +@dataclass(frozen=True, slots=True) +class DockerCompose: + compose_file: Path + service: ServiceName + executable: Path + limits: ProcessLimits + + def fingerprint(self, root: Path) -> Sha256Digest: + compose = root / self.compose_file + return _digest_text( + f"docker\0{self.service.value}\0{self.executable}\0" + f"{self.limits.timeout.total_seconds()}\0{_digest_file(compose)}" + ) + + def prepare(self, revision: HarnessRevision, case: CanaryCase) -> PreparedEnvironment: + del case + prepared = _prepare_workspace( + revision.root, + self.limits, + ( + str(self.executable), + "compose", + "-f", + self.compose_file.as_posix(), + "exec", + "-T", + self.service.value, + ), + ) + if not (prepared.root / self.compose_file).is_file(): + self.destroy(prepared) + raise RuntimeError("compose file is missing") + started = _compose_control(self, prepared, ("up", "-d", self.service.value)) + if started.exit_code != 0: + self.destroy(prepared) + raise RuntimeError("docker compose failed to start") + return prepared + + def health(self, prepared: PreparedEnvironment) -> CheckReport: + result = _compose_control(self, prepared, ("ps", "--status", "running")) + return CheckReport(result.exit_code == 0, "docker compose") + + def snapshot(self, prepared: PreparedEnvironment) -> EnvironmentFingerprint: + return EnvironmentFingerprint(_digest_tree(prepared.root)) + + def reset(self, prepared: PreparedEnvironment) -> None: + stopped = _compose_control(self, prepared, ("down", "--volumes", "--remove-orphans")) + if stopped.exit_code != 0: + raise RuntimeError("docker compose reset failed") + _restore_workspace(prepared) + started = _compose_control(self, prepared, ("up", "-d", self.service.value)) + if started.exit_code != 0: + raise RuntimeError("docker compose reset failed") + + def destroy(self, prepared: PreparedEnvironment) -> None: + _compose_control(self, prepared, ("down", "--volumes", "--remove-orphans")) + prepared.temporary.cleanup() + + +ExecutionEnvironment = LocalProcess | DockerCompose + + +@dataclass(frozen=True, slots=True) +class CommandLoop: + command: ProcessCommand + models: tuple[ModelFingerprint, ...] = () + + def fingerprint(self, root: Path) -> Sha256Digest: + return _command_fingerprint("command-loop", self.command, self.models, root) + + def invoke( + self, + case: CanaryCase, + prepared: PreparedEnvironment, + revision: HarnessRevision, + ) -> RunResult: + del revision + return _run_result(case.id, prepared.run(self.command, case.payload)) + + +@dataclass(frozen=True, slots=True) +class PythonLoop: + entrypoint: PythonEntrypoint + models: tuple[ModelFingerprint, ...] = () + + def fingerprint(self, root: Path) -> Sha256Digest: + module_path = root / Path(*self.entrypoint.module.value.split(".")).with_suffix(".py") + source_digest = _digest_file(module_path) if module_path.is_file() else "missing" + return _digest_text( + f"python-loop\0{self.entrypoint.module.value}\0" + f"{self.entrypoint.function.value}\0{source_digest}\0{_models_text(self.models)}" + ) + + def invoke( + self, + case: CanaryCase, + prepared: PreparedEnvironment, + revision: HarnessRevision, + ) -> RunResult: + del revision + command = ProcessCommand( + ( + sys.executable, + "-m", + "ofw._runner", + self.entrypoint.module.value, + self.entrypoint.function.value, + ) + ) + return _run_result(case.id, prepared.run(command, case.payload)) + + +LifecycleAdapter = CommandLoop | PythonLoop +VerifierFunction = Callable[[RunResult], VerifierResult] + + +@dataclass(frozen=True, slots=True) +class PythonVerifier: + name: str + function: VerifierFunction + + def __post_init__(self) -> None: + if _NAME_PATTERN.fullmatch(self.name) is None: + raise ValueError("invalid verifier name") + + def fingerprint(self, root: Path) -> Sha256Digest: + del root + source = inspect.getsourcefile(self.function) + if source is None: + raise ValueError("verifier must be file-backed") + return _digest_text(f"python-verifier\0{self.name}\0{_digest_file(Path(source))}") + + def verify( + self, + result: RunResult, + prepared: PreparedEnvironment, + ) -> VerifierResult: + del prepared + try: + return self.function(result) + except Exception: + return VerifierResult( + VerifierVerdict.ERROR, + None, + "python verifier failed", + retryable=True, + ) + + +@dataclass(frozen=True, slots=True) +class CommandVerifier: + name: str + command: ProcessCommand + + def __post_init__(self) -> None: + if _NAME_PATTERN.fullmatch(self.name) is None: + raise ValueError("invalid verifier name") + + def fingerprint(self, root: Path) -> Sha256Digest: + return _command_fingerprint("command-verifier", self.command, (), root) + + def verify( + self, + result: RunResult, + prepared: PreparedEnvironment, + ) -> VerifierResult: + process = prepared.run(self.command, result.output or "") + if process.timed_out: + return VerifierResult( + VerifierVerdict.ERROR, + None, + "command verifier timed out", + retryable=True, + ) + if process.exit_code == VerifierExitCode.PASS: + return VerifierResult(VerifierVerdict.PASS, 1.0, process.stdout) + if process.exit_code == VerifierExitCode.FAIL: + return VerifierResult(VerifierVerdict.FAIL, 0.0, process.stdout) + if process.exit_code == VerifierExitCode.ABSTAIN: + return VerifierResult(VerifierVerdict.ABSTAIN, None, process.stdout) + return VerifierResult(VerifierVerdict.ERROR, None, "command verifier failed") + + +VerifierAdapter = PythonVerifier | CommandVerifier + + +@dataclass(frozen=True, slots=True) +class CanaryReport: + case_id: CaseId + health: CheckReport + run: RunResult + verifiers: tuple[VerifierResult, ...] + + @property + def digest(self) -> Sha256Digest: + return _digest_bytes(_CANARY_ADAPTER.dump_json(self)) + + @property + def passed(self) -> bool: + return ( + self.health.passed + and self.run.status is RunStatus.SUCCESS + and bool(self.verifiers) + and all(result.verdict is VerifierVerdict.PASS for result in self.verifiers) + ) + + def to_json(self) -> str: + return _CANARY_ADAPTER.dump_json(self).decode() + + +_CANARY_ADAPTER: TypeAdapter[CanaryReport] = TypeAdapter(CanaryReport) + + +def runtime_configuration( + root: Path, + execution: ExecutionEnvironment, + lifecycle: LifecycleAdapter, + verifiers: tuple[VerifierAdapter, ...], +) -> RuntimeConfiguration: + return RuntimeConfiguration( + execution.fingerprint(root), + lifecycle.fingerprint(root), + tuple(verifier.fingerprint(root) for verifier in verifiers), + ) + + +def run_canary( + revision: HarnessRevision, + case: CanaryCase, + execution: ExecutionEnvironment, + lifecycle: LifecycleAdapter, + verifiers: tuple[VerifierAdapter, ...], +) -> CanaryReport: + prepared = execution.prepare(revision, case) + try: + health = execution.health(prepared) + run = lifecycle.invoke(case, prepared, revision) + results = tuple(verifier.verify(run, prepared) for verifier in verifiers) + return CanaryReport(case.id, health, run, results) + finally: + execution.destroy(prepared) + + +def _prepare_workspace( + source_root: Path, + limits: ProcessLimits, + command_prefix: tuple[str, ...] = (), +) -> PreparedEnvironment: + temporary = tempfile.TemporaryDirectory(prefix="ofw-runtime-") + root = Path(temporary.name) / "workspace" + _copy_workspace(source_root, root) + return PreparedEnvironment(root, source_root, temporary, limits, command_prefix) + + +def _copy_workspace(source: Path, destination: Path) -> None: + shutil.copytree( + source, + destination, + ignore=_ignored_artifacts, + ) + + +def _ignored_artifacts(directory: str, names: list[str]) -> set[str]: + del directory + ignored = (".git", ".ofw", ".venv", "__pycache__", "build", "dist") + return {name for name in names if name in ignored} + + +def _restore_workspace(prepared: PreparedEnvironment) -> None: + shutil.rmtree(prepared.root) + _copy_workspace(prepared.source_root, prepared.root) + + +def _compose_control( + adapter: DockerCompose, + prepared: PreparedEnvironment, + arguments: tuple[str, ...], +) -> ProcessResult: + command = ProcessCommand( + ( + str(adapter.executable), + "compose", + "-f", + adapter.compose_file.as_posix(), + *arguments, + ) + ) + local = PreparedEnvironment( + prepared.root, + prepared.source_root, + prepared.temporary, + prepared.limits, + ) + return local.run(command, "") + + +def _run_result(case_id: CaseId, process: ProcessResult) -> RunResult: + if process.timed_out: + return RunResult( + case_id, RunStatus.TIMEOUT, None, RunErrorCode.TIMEOUT, process.duration_seconds + ) + if process.exit_code != 0: + return RunResult( + case_id, + RunStatus.ERROR, + None, + RunErrorCode.NON_ZERO_EXIT, + process.duration_seconds, + ) + return RunResult(case_id, RunStatus.SUCCESS, process.stdout, None, process.duration_seconds) + + +def _command_fingerprint( + kind: str, + command: ProcessCommand, + models: tuple[ModelFingerprint, ...], + root: Path, +) -> Sha256Digest: + sources = tuple( + _digest_file(root / argument) + for argument in command.arguments + if (root / argument).is_file() + ) + return _digest_text("\0".join((kind, *command.arguments, *sources, _models_text(models)))) + + +def _models_text(models: tuple[ModelFingerprint, ...]) -> str: + return "\0".join(f"{model.provider}:{model.model}:{model.reasoning}" for model in models) + + +def _digest_tree(root: Path) -> Sha256Digest: + payload = "\0".join( + f"{path.relative_to(root).as_posix()}:{_digest_file(path)}" + for path in sorted(root.rglob("*")) + if path.is_file() + ) + return _digest_text(payload) + + +def _digest_file(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _digest_text(value: str) -> Sha256Digest: + return _digest_bytes(value.encode()) + + +def _digest_bytes(value: bytes) -> Sha256Digest: + return Sha256Digest(f"sha256:{hashlib.sha256(value).hexdigest()}") diff --git a/tests/test_runtime.py b/tests/test_runtime.py new file mode 100644 index 0000000..d3a289d --- /dev/null +++ b/tests/test_runtime.py @@ -0,0 +1,252 @@ +"""Execution, lifecycle, verifier, and canary behavior.""" + +from __future__ import annotations + +import subprocess +import sys +from datetime import timedelta +from pathlib import Path + +import pytest + +from ofw import ( + CanaryCase, + CaseId, + CommandLoop, + CommandVerifier, + DockerCompose, + FunctionName, + Harness, + HarnessErrorCode, + HarnessRevision, + HarnessValidationError, + LocalProcess, + ModelFingerprint, + ModuleName, + ProcessCommand, + ProcessLimits, + PythonEntrypoint, + PythonLoop, + PythonVerifier, + RunErrorCode, + RunResult, + RunStatus, + ServiceName, + VerifierResult, + VerifierVerdict, +) + + +def _run_git(root: Path, *arguments: str) -> None: + subprocess.run( + ("git", "-C", str(root), *arguments), + check=True, + capture_output=True, + text=True, + ) + + +def _repository(tmp_path: Path) -> Path: + root = tmp_path / "runtime-agent" + root.mkdir() + (root / "prompt.md").write_text("Be accurate.\n", encoding="utf-8") + (root / "agent_loop.py").write_text( + "def run_case(value: str) -> str:\n return value.upper()\n", + encoding="utf-8", + ) + (root / "compose.yaml").write_text( + "services:\n agent:\n image: fixture-agent:latest\n", + encoding="utf-8", + ) + _run_git(root, "init", "-q") + _run_git(root, "config", "user.email", "fixture@example.test") + _run_git(root, "config", "user.name", "FixtureCo") + _run_git(root, "add", ".") + _run_git(root, "commit", "-qm", "fixture baseline") + return root + + +def _revision(root: Path) -> HarnessRevision: + harness = Harness("runtime-agent", root=root) + harness.connect_prompt(Path("prompt.md")) + return harness.process() + + +def _uppercase_verifier(result: RunResult) -> VerifierResult: + verdict = VerifierVerdict.PASS if result.output == "SHIP" else VerifierVerdict.FAIL + return VerifierResult(verdict=verdict, score=1.0, feedback="uppercase output") + + +def _broken_verifier(result: RunResult) -> VerifierResult: + del result + raise RuntimeError("fixture verifier failure") + + +def _failing_verifier(result: RunResult) -> VerifierResult: + del result + return VerifierResult(VerifierVerdict.FAIL, 0.0, "fixture rejection") + + +def _python_loop() -> PythonLoop: + return PythonLoop( + entrypoint=PythonEntrypoint(ModuleName("agent_loop"), FunctionName("run_case")), + models=(ModelFingerprint("openai", "gpt-5", "medium"),), + ) + + +def test_process_runs_local_python_canary_and_records_frozen_evidence(tmp_path: Path) -> None: + root = _repository(tmp_path) + harness = Harness("runtime-agent", root=root) + harness.connect_prompt(Path("prompt.md")) + harness.connect_execute(LocalProcess(ProcessLimits(timedelta(seconds=2)))) + harness.connect_lifecycle(_python_loop()) + harness.connect_verifiers(PythonVerifier("uppercase", _uppercase_verifier)) + + revision = harness.process(canary=CanaryCase(CaseId("smoke"), "ship")) + + assert revision.runtime is not None + assert revision.canary_digest is not None + assert revision.canary_path.is_file() + assert "pass" in revision.canary_path.read_text(encoding="utf-8") + + +def test_partial_runtime_configuration_is_rejected(tmp_path: Path) -> None: + root = _repository(tmp_path) + harness = Harness("runtime-agent", root=root) + harness.connect_prompt(Path("prompt.md")) + harness.connect_execute(LocalProcess(ProcessLimits(timedelta(seconds=1)))) + + with pytest.raises(HarnessValidationError) as raised: + harness.process() + + assert raised.value.code is HarnessErrorCode.RUNTIME_INCOMPLETE + + +def test_duplicate_verifier_name_is_rejected(tmp_path: Path) -> None: + harness = Harness("runtime-agent", root=_repository(tmp_path)) + + with pytest.raises(HarnessValidationError) as raised: + harness.connect_verifiers( + PythonVerifier("duplicate", _uppercase_verifier), + PythonVerifier("duplicate", _failing_verifier), + ) + + assert raised.value.code is HarnessErrorCode.DUPLICATE_VERIFIER + + +def test_failed_canary_blocks_revision_creation(tmp_path: Path) -> None: + root = _repository(tmp_path) + harness = Harness("runtime-agent", root=root) + harness.connect_prompt(Path("prompt.md")) + harness.connect_execute(LocalProcess(ProcessLimits(timedelta(seconds=2)))) + harness.connect_lifecycle(_python_loop()) + harness.connect_verifiers(PythonVerifier("reject", _failing_verifier)) + + with pytest.raises(HarnessValidationError) as raised: + harness.process(canary=CanaryCase(CaseId("rejected"), "ship")) + + assert raised.value.code is HarnessErrorCode.CANARY_FAILED + + +def test_local_process_reports_timeout_and_nonzero_exit(tmp_path: Path) -> None: + root = _repository(tmp_path) + revision = _revision(root) + timeout_script = root / "timeout.py" + timeout_script.write_text("import time\ntime.sleep(2)\n", encoding="utf-8") + crash_script = root / "crash.py" + crash_script.write_text("raise SystemExit(7)\n", encoding="utf-8") + environment = LocalProcess(ProcessLimits(timedelta(milliseconds=50))) + prepared = environment.prepare(revision, CanaryCase(CaseId("failure"), "input")) + try: + timed_out = CommandLoop(ProcessCommand((sys.executable, "timeout.py"))).invoke( + CanaryCase(CaseId("timeout"), "input"), prepared, revision + ) + crashed = CommandLoop(ProcessCommand((sys.executable, "crash.py"))).invoke( + CanaryCase(CaseId("crash"), "input"), prepared, revision + ) + finally: + environment.destroy(prepared) + + assert timed_out.status is RunStatus.TIMEOUT + assert timed_out.error_code is RunErrorCode.TIMEOUT + assert crashed.status is RunStatus.ERROR + assert crashed.error_code is RunErrorCode.NON_ZERO_EXIT + + +def test_local_process_reset_restores_workspace(tmp_path: Path) -> None: + root = _repository(tmp_path) + revision = _revision(root) + environment = LocalProcess(ProcessLimits(timedelta(seconds=1))) + prepared = environment.prepare(revision, CanaryCase(CaseId("reset"), "input")) + copied_prompt = prepared.root / "prompt.md" + copied_prompt.write_text("mutated\n", encoding="utf-8") + + environment.reset(prepared) + + assert copied_prompt.read_text(encoding="utf-8") == "Be accurate.\n" + environment.destroy(prepared) + assert not prepared.root.exists() + + +def test_python_and_command_verifiers_report_typed_outcomes(tmp_path: Path) -> None: + root = _repository(tmp_path) + revision = _revision(root) + environment = LocalProcess(ProcessLimits(timedelta(seconds=1))) + prepared = environment.prepare(revision, CanaryCase(CaseId("verify"), "input")) + abstain_script = prepared.root / "abstain.py" + abstain_script.write_text( + "import sys\nprint('not enough evidence')\nraise SystemExit(2)\n", + encoding="utf-8", + ) + run = RunResult.success(CaseId("verify"), "SHIP") + try: + abstained = CommandVerifier( + "command", + ProcessCommand((sys.executable, "abstain.py")), + ).verify(run, prepared) + errored = PythonVerifier("broken", _broken_verifier).verify(run, prepared) + finally: + environment.destroy(prepared) + + assert abstained.verdict is VerifierVerdict.ABSTAIN + assert errored.verdict is VerifierVerdict.ERROR + assert errored.retryable + + +def test_docker_compose_adapter_uses_disposable_native_lifecycle(tmp_path: Path) -> None: + root = _repository(tmp_path) + revision = _revision(root) + environment = DockerCompose( + compose_file=Path("compose.yaml"), + service=ServiceName("agent"), + executable=Path("/usr/bin/true"), + limits=ProcessLimits(timedelta(seconds=1)), + ) + prepared = environment.prepare(revision, CanaryCase(CaseId("docker"), "input")) + + assert environment.health(prepared).passed + (prepared.root / "prompt.md").write_text("mutated\n", encoding="utf-8") + environment.reset(prepared) + assert (prepared.root / "prompt.md").read_text(encoding="utf-8") == "Be accurate.\n" + environment.destroy(prepared) + assert not prepared.root.exists() + + +def test_runtime_fingerprint_changes_without_changing_assets(tmp_path: Path) -> None: + root = _repository(tmp_path) + first = Harness("runtime-agent", root=root) + first.connect_prompt(Path("prompt.md")) + first.connect_execute(LocalProcess(ProcessLimits(timedelta(seconds=1)))) + first.connect_lifecycle(_python_loop()) + first.connect_verifiers(PythonVerifier("uppercase", _uppercase_verifier)) + second = Harness("runtime-agent", root=root) + second.connect_prompt(Path("prompt.md")) + second.connect_execute(LocalProcess(ProcessLimits(timedelta(seconds=2)))) + second.connect_lifecycle(_python_loop()) + second.connect_verifiers(PythonVerifier("uppercase", _uppercase_verifier)) + + first_revision = first.process() + second_revision = second.process() + + assert first_revision.id != second_revision.id + assert first_revision.components == second_revision.components diff --git a/tests/test_typing.py b/tests/test_typing.py index cf656b9..3ee6098 100644 --- a/tests/test_typing.py +++ b/tests/test_typing.py @@ -12,3 +12,5 @@ def test_package_declares_inline_types_and_namespace_methods() -> None: assert Path(package_file).with_name("py.typed").is_file() assert callable(ofw.collect) assert callable(ofw.editable) + assert callable(ofw.LocalProcess) + assert callable(ofw.ProcessLimits) From 81ce150c461f7f0915aba17b715bdb1fef363199 Mon Sep 17 00:00:00 2001 From: divo12 Date: Sat, 22 Aug 2026 16:36:12 +0530 Subject: [PATCH 02/18] isolate concurrent compose canaries --- src/ofw/runtime.py | 34 +++++++++++++++++++++++----------- tests/test_runtime.py | 19 +++++++++++++++++++ 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/src/ofw/runtime.py b/src/ofw/runtime.py index 5157e1b..b240d9b 100644 --- a/src/ofw/runtime.py +++ b/src/ofw/runtime.py @@ -253,14 +253,14 @@ def fingerprint(self, root: Path) -> Sha256Digest: def prepare(self, revision: HarnessRevision, case: CanaryCase) -> PreparedEnvironment: del case - prepared = _prepare_workspace( - revision.root, - self.limits, + workspace = _prepare_workspace(revision.root, self.limits) + prepared = PreparedEnvironment( + workspace.root, + workspace.source_root, + workspace.temporary, + workspace.limits, ( - str(self.executable), - "compose", - "-f", - self.compose_file.as_posix(), + *_compose_prefix(self, workspace), "exec", "-T", self.service.value, @@ -517,10 +517,7 @@ def _compose_control( ) -> ProcessResult: command = ProcessCommand( ( - str(adapter.executable), - "compose", - "-f", - adapter.compose_file.as_posix(), + *_compose_prefix(adapter, prepared), *arguments, ) ) @@ -533,6 +530,21 @@ def _compose_control( return local.run(command, "") +def _compose_prefix( + adapter: DockerCompose, + prepared: PreparedEnvironment, +) -> tuple[str, ...]: + project = hashlib.sha256(prepared.temporary.name.encode()).hexdigest()[:16] + return ( + str(adapter.executable), + "compose", + "-f", + adapter.compose_file.as_posix(), + "-p", + f"ofw-{project}", + ) + + def _run_result(case_id: CaseId, process: ProcessResult) -> RunResult: if process.timed_out: return RunResult( diff --git a/tests/test_runtime.py b/tests/test_runtime.py index d3a289d..bdc2348 100644 --- a/tests/test_runtime.py +++ b/tests/test_runtime.py @@ -232,6 +232,25 @@ def test_docker_compose_adapter_uses_disposable_native_lifecycle(tmp_path: Path) assert not prepared.root.exists() +def test_parallel_docker_environments_have_distinct_projects(tmp_path: Path) -> None: + root = _repository(tmp_path) + revision = _revision(root) + environment = DockerCompose( + compose_file=Path("compose.yaml"), + service=ServiceName("agent"), + executable=Path("/usr/bin/true"), + limits=ProcessLimits(timedelta(seconds=1)), + ) + case = CanaryCase(CaseId("parallel"), "input") + first = environment.prepare(revision, case) + second = environment.prepare(revision, case) + try: + assert first.command_prefix != second.command_prefix + finally: + environment.destroy(first) + environment.destroy(second) + + def test_runtime_fingerprint_changes_without_changing_assets(tmp_path: Path) -> None: root = _repository(tmp_path) first = Harness("runtime-agent", root=root) From 338f7fd47af3157847c791387447b8cf174b6825 Mon Sep 17 00:00:00 2001 From: divo12 Date: Sat, 22 Aug 2026 16:43:59 +0530 Subject: [PATCH 03/18] isolate and validate Python verifiers --- src/ofw/_verifier_runner.py | 45 +++++++++++++++ src/ofw/contracts.py | 3 +- src/ofw/harness.py | 35 ++++++------ src/ofw/runtime.py | 69 +++++++++++++++++------ tests/test_runtime.py | 107 +++++++++++++++++++++++++++++------- 5 files changed, 200 insertions(+), 59 deletions(-) create mode 100644 src/ofw/_verifier_runner.py diff --git a/src/ofw/_verifier_runner.py b/src/ofw/_verifier_runner.py new file mode 100644 index 0000000..a08121f --- /dev/null +++ b/src/ofw/_verifier_runner.py @@ -0,0 +1,45 @@ +"""Timed child-process entrypoint for a file-backed Python verifier.""" + +from __future__ import annotations + +import importlib +import inspect +import sys +from collections.abc import Callable +from typing import cast + +from pydantic import TypeAdapter, ValidationError + +from ofw.runtime import RunResult, VerifierResult + +_RUN_ADAPTER: TypeAdapter[RunResult] = TypeAdapter(RunResult) +_VERIFIER_ADAPTER: TypeAdapter[VerifierResult] = TypeAdapter(VerifierResult) + + +def main() -> int: + if len(sys.argv) != 3: + return 2 + payload: str = sys.stdin.read() + try: + result = _RUN_ADAPTER.validate_json(payload) + except ValidationError: + return 2 + module = importlib.import_module(sys.argv[1]) + functions = tuple( + function + for name, function in inspect.getmembers(module, inspect.isfunction) + if name == sys.argv[2] + ) + if len(functions) != 1: + return 2 + function: Callable[[RunResult], VerifierResult] = cast( + Callable[[RunResult], VerifierResult], + functions[0], + ) + verified: VerifierResult = function(result) + sys.stdout.write(_VERIFIER_ADAPTER.dump_json(verified).decode()) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/ofw/contracts.py b/src/ofw/contracts.py index 3b2ea54..28f9741 100644 --- a/src/ofw/contracts.py +++ b/src/ofw/contracts.py @@ -50,6 +50,7 @@ class HarnessErrorCode(StrEnum): RUNTIME_INCOMPLETE = "runtime_incomplete" CANARY_FAILED = "canary_failed" DUPLICATE_VERIFIER = "duplicate_verifier" + RUNTIME_INVALID = "runtime_invalid" class HarnessValidationError(Exception): @@ -139,7 +140,6 @@ class HarnessRevisionContent: components: tuple[HarnessComponent, ...] observability: LangfuseConnectionManifest | None runtime: RuntimeConfiguration | None - canary_digest: Sha256Digest | None def canonical_json(self) -> str: return _render_content(self) @@ -217,7 +217,6 @@ def _render_content(content: HarnessRevisionContent) -> str: f'"repository":{_render_repository(content.repository)},' f'"observability":{_render_observability(content.observability)},' f'"runtime":{_render_runtime(content.runtime)},' - f'"canary_digest":{_render_digest(content.canary_digest)},' f'"components":[{components}]' "}" ) diff --git a/src/ofw/harness.py b/src/ofw/harness.py index 01e3b55..831da84 100644 --- a/src/ofw/harness.py +++ b/src/ofw/harness.py @@ -188,7 +188,6 @@ def process(self, *, canary: CanaryCase | None = None) -> HarnessRevision: components=components, observability=(None if self._observability is None else self._observability.manifest()), runtime=runtime, - canary_digest=None, ) revision = _revision_from_content(content, root) report: CanaryReport | None = None @@ -205,16 +204,7 @@ def process(self, *, canary: CanaryCase | None = None) -> HarnessRevision: if not report.passed: _write_canary(revision, report) raise HarnessValidationError(HarnessErrorCode.CANARY_FAILED, canary.id.value) - content = HarnessRevisionContent( - schema_version=content.schema_version, - harness_name=content.harness_name, - repository=content.repository, - components=content.components, - observability=content.observability, - runtime=content.runtime, - canary_digest=report.digest, - ) - revision = _revision_from_content(content, root) + revision = _revision_from_content(content, root, report.digest) _write_manifest(revision) if report is not None: _write_canary(revision, report) @@ -231,19 +221,26 @@ def _runtime(self, root: Path) -> RuntimeConfiguration | None: return None if not all(connections) or self._execution is None or self._lifecycle is None: raise HarnessValidationError(HarnessErrorCode.RUNTIME_INCOMPLETE, self.name) - return runtime_configuration( - root, - self._execution, - self._lifecycle, - tuple(self._verifiers), - ) + try: + return runtime_configuration( + root, + self._execution, + self._lifecycle, + tuple(self._verifiers), + ) + except ValueError as error: + raise HarnessValidationError(HarnessErrorCode.RUNTIME_INVALID, self.name) from error def _has_component(registrations: list[_FileRegistration], kind: ComponentKind) -> bool: return any(registration.component is kind for registration in registrations) -def _revision_from_content(content: HarnessRevisionContent, root: Path) -> HarnessRevision: +def _revision_from_content( + content: HarnessRevisionContent, + root: Path, + canary_digest: Sha256Digest | None = None, +) -> HarnessRevision: content_digest = _digest_text(content.canonical_json()) return HarnessRevision( schema_version=content.schema_version, @@ -254,7 +251,7 @@ def _revision_from_content(content: HarnessRevisionContent, root: Path) -> Harne components=content.components, observability=content.observability, runtime=content.runtime, - canary_digest=content.canary_digest, + canary_digest=canary_digest, ) diff --git a/src/ofw/runtime.py b/src/ofw/runtime.py index b240d9b..e5fe1f1 100644 --- a/src/ofw/runtime.py +++ b/src/ofw/runtime.py @@ -2,21 +2,20 @@ from __future__ import annotations +import ast import hashlib -import inspect import re import shutil import subprocess # nosec B404 import sys import tempfile import time -from collections.abc import Callable from dataclasses import dataclass from datetime import timedelta from enum import IntEnum, StrEnum from pathlib import Path -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError from ofw.contracts import HarnessRevision, RuntimeConfiguration, Sha256Digest @@ -323,11 +322,11 @@ class PythonLoop: models: tuple[ModelFingerprint, ...] = () def fingerprint(self, root: Path) -> Sha256Digest: - module_path = root / Path(*self.entrypoint.module.value.split(".")).with_suffix(".py") - source_digest = _digest_file(module_path) if module_path.is_file() else "missing" + module_path = _python_source(root, self.entrypoint) return _digest_text( f"python-loop\0{self.entrypoint.module.value}\0" - f"{self.entrypoint.function.value}\0{source_digest}\0{_models_text(self.models)}" + f"{self.entrypoint.function.value}\0{_digest_file(module_path)}\0" + f"{_models_text(self.models)}" ) def invoke( @@ -350,40 +349,57 @@ def invoke( LifecycleAdapter = CommandLoop | PythonLoop -VerifierFunction = Callable[[RunResult], VerifierResult] @dataclass(frozen=True, slots=True) class PythonVerifier: name: str - function: VerifierFunction + entrypoint: PythonEntrypoint def __post_init__(self) -> None: if _NAME_PATTERN.fullmatch(self.name) is None: raise ValueError("invalid verifier name") def fingerprint(self, root: Path) -> Sha256Digest: - del root - source = inspect.getsourcefile(self.function) - if source is None: - raise ValueError("verifier must be file-backed") - return _digest_text(f"python-verifier\0{self.name}\0{_digest_file(Path(source))}") + source = _python_source(root, self.entrypoint) + return _digest_text( + f"python-verifier\0{self.name}\0{self.entrypoint.module.value}\0" + f"{self.entrypoint.function.value}\0{_digest_file(source)}" + ) def verify( self, result: RunResult, prepared: PreparedEnvironment, ) -> VerifierResult: - del prepared - try: - return self.function(result) - except Exception: + command = ProcessCommand( + ( + sys.executable, + "-m", + "ofw._verifier_runner", + self.entrypoint.module.value, + self.entrypoint.function.value, + ) + ) + process = prepared.run(command, _RUN_ADAPTER.dump_json(result).decode()) + if process.timed_out: + return VerifierResult( + VerifierVerdict.ERROR, + None, + "python verifier timed out", + retryable=True, + ) + if process.exit_code != 0: return VerifierResult( VerifierVerdict.ERROR, None, "python verifier failed", retryable=True, ) + try: + return _VERIFIER_ADAPTER.validate_json(process.stdout) + except ValidationError: + return VerifierResult(VerifierVerdict.ERROR, None, "invalid verifier result") @dataclass(frozen=True, slots=True) @@ -422,6 +438,9 @@ def verify( VerifierAdapter = PythonVerifier | CommandVerifier +_RUN_ADAPTER: TypeAdapter[RunResult] = TypeAdapter(RunResult) +_VERIFIER_ADAPTER: TypeAdapter[VerifierResult] = TypeAdapter(VerifierResult) + @dataclass(frozen=True, slots=True) class CanaryReport: @@ -575,6 +594,22 @@ def _command_fingerprint( return _digest_text("\0".join((kind, *command.arguments, *sources, _models_text(models)))) +def _python_source(root: Path, entrypoint: PythonEntrypoint) -> Path: + path = root / Path(*entrypoint.module.value.split(".")).with_suffix(".py") + if not path.is_file(): + raise ValueError("python module is missing") + try: + module = ast.parse(path.read_text(encoding="utf-8")) + except (OSError, SyntaxError) as error: + raise ValueError("python module is invalid") from error + if not any( + isinstance(node, ast.FunctionDef) and node.name == entrypoint.function.value + for node in module.body + ): + raise ValueError("top-level python function is missing") + return path + + def _models_text(models: tuple[ModelFingerprint, ...]) -> str: return "\0".join(f"{model.provider}:{model.model}:{model.reasoning}" for model in models) diff --git a/tests/test_runtime.py b/tests/test_runtime.py index bdc2348..b626def 100644 --- a/tests/test_runtime.py +++ b/tests/test_runtime.py @@ -4,6 +4,7 @@ import subprocess import sys +import time from datetime import timedelta from pathlib import Path @@ -32,7 +33,6 @@ RunResult, RunStatus, ServiceName, - VerifierResult, VerifierVerdict, ) @@ -54,6 +54,25 @@ def _repository(tmp_path: Path) -> Path: "def run_case(value: str) -> str:\n return value.upper()\n", encoding="utf-8", ) + (root / "verifiers.py").write_text( + "from __future__ import annotations\n" + "import time\n" + "from ofw import RunResult, VerifierResult, VerifierVerdict\n" + "def uppercase(result: RunResult) -> VerifierResult:\n" + " verdict = VerifierVerdict.PASS if result.output == 'SHIP' else VerifierVerdict.FAIL\n" + " return VerifierResult(verdict, 1.0, 'uppercase output')\n" + "def broken(result: RunResult) -> VerifierResult:\n" + " del result\n" + " raise RuntimeError('fixture verifier failure')\n" + "def reject(result: RunResult) -> VerifierResult:\n" + " del result\n" + " return VerifierResult(VerifierVerdict.FAIL, 0.0, 'fixture rejection')\n" + "def slow(result: RunResult) -> VerifierResult:\n" + " del result\n" + " time.sleep(2)\n" + " return VerifierResult(VerifierVerdict.PASS, 1.0, 'late')\n", + encoding="utf-8", + ) (root / "compose.yaml").write_text( "services:\n agent:\n image: fixture-agent:latest\n", encoding="utf-8", @@ -72,19 +91,11 @@ def _revision(root: Path) -> HarnessRevision: return harness.process() -def _uppercase_verifier(result: RunResult) -> VerifierResult: - verdict = VerifierVerdict.PASS if result.output == "SHIP" else VerifierVerdict.FAIL - return VerifierResult(verdict=verdict, score=1.0, feedback="uppercase output") - - -def _broken_verifier(result: RunResult) -> VerifierResult: - del result - raise RuntimeError("fixture verifier failure") - - -def _failing_verifier(result: RunResult) -> VerifierResult: - del result - return VerifierResult(VerifierVerdict.FAIL, 0.0, "fixture rejection") +def _verifier(function: str, name: str = "verifier") -> PythonVerifier: + return PythonVerifier( + name, + PythonEntrypoint(ModuleName("verifiers"), FunctionName(function)), + ) def _python_loop() -> PythonLoop: @@ -94,13 +105,22 @@ def _python_loop() -> PythonLoop: ) +def _runtime_harness(root: Path) -> Harness: + harness = Harness("runtime-agent", root=root) + harness.connect_prompt(Path("prompt.md")) + harness.connect_execute(LocalProcess(ProcessLimits(timedelta(seconds=2)))) + harness.connect_lifecycle(_python_loop()) + harness.connect_verifiers(_verifier("uppercase", "uppercase")) + return harness + + def test_process_runs_local_python_canary_and_records_frozen_evidence(tmp_path: Path) -> None: root = _repository(tmp_path) harness = Harness("runtime-agent", root=root) harness.connect_prompt(Path("prompt.md")) harness.connect_execute(LocalProcess(ProcessLimits(timedelta(seconds=2)))) harness.connect_lifecycle(_python_loop()) - harness.connect_verifiers(PythonVerifier("uppercase", _uppercase_verifier)) + harness.connect_verifiers(_verifier("uppercase", "uppercase")) revision = harness.process(canary=CanaryCase(CaseId("smoke"), "ship")) @@ -110,6 +130,16 @@ def test_process_runs_local_python_canary_and_records_frozen_evidence(tmp_path: assert "pass" in revision.canary_path.read_text(encoding="utf-8") +def test_canary_evidence_does_not_change_runtime_revision_identity(tmp_path: Path) -> None: + root = _repository(tmp_path) + + without_canary = _runtime_harness(root).process() + with_canary = _runtime_harness(root).process(canary=CanaryCase(CaseId("identity"), "ship")) + + assert with_canary.id == without_canary.id + assert with_canary.canary_digest is not None + + def test_partial_runtime_configuration_is_rejected(tmp_path: Path) -> None: root = _repository(tmp_path) harness = Harness("runtime-agent", root=root) @@ -127,8 +157,8 @@ def test_duplicate_verifier_name_is_rejected(tmp_path: Path) -> None: with pytest.raises(HarnessValidationError) as raised: harness.connect_verifiers( - PythonVerifier("duplicate", _uppercase_verifier), - PythonVerifier("duplicate", _failing_verifier), + _verifier("uppercase", "duplicate"), + _verifier("reject", "duplicate"), ) assert raised.value.code is HarnessErrorCode.DUPLICATE_VERIFIER @@ -140,7 +170,7 @@ def test_failed_canary_blocks_revision_creation(tmp_path: Path) -> None: harness.connect_prompt(Path("prompt.md")) harness.connect_execute(LocalProcess(ProcessLimits(timedelta(seconds=2)))) harness.connect_lifecycle(_python_loop()) - harness.connect_verifiers(PythonVerifier("reject", _failing_verifier)) + harness.connect_verifiers(_verifier("reject", "reject")) with pytest.raises(HarnessValidationError) as raised: harness.process(canary=CanaryCase(CaseId("rejected"), "ship")) @@ -204,7 +234,7 @@ def test_python_and_command_verifiers_report_typed_outcomes(tmp_path: Path) -> N "command", ProcessCommand((sys.executable, "abstain.py")), ).verify(run, prepared) - errored = PythonVerifier("broken", _broken_verifier).verify(run, prepared) + errored = _verifier("broken", "broken").verify(run, prepared) finally: environment.destroy(prepared) @@ -257,15 +287,50 @@ def test_runtime_fingerprint_changes_without_changing_assets(tmp_path: Path) -> first.connect_prompt(Path("prompt.md")) first.connect_execute(LocalProcess(ProcessLimits(timedelta(seconds=1)))) first.connect_lifecycle(_python_loop()) - first.connect_verifiers(PythonVerifier("uppercase", _uppercase_verifier)) + first.connect_verifiers(_verifier("uppercase", "uppercase")) second = Harness("runtime-agent", root=root) second.connect_prompt(Path("prompt.md")) second.connect_execute(LocalProcess(ProcessLimits(timedelta(seconds=2)))) second.connect_lifecycle(_python_loop()) - second.connect_verifiers(PythonVerifier("uppercase", _uppercase_verifier)) + second.connect_verifiers(_verifier("uppercase", "uppercase")) first_revision = first.process() second_revision = second.process() assert first_revision.id != second_revision.id assert first_revision.components == second_revision.components + + +def test_missing_python_entrypoint_is_rejected_during_process(tmp_path: Path) -> None: + root = _repository(tmp_path) + harness = Harness("runtime-agent", root=root) + harness.connect_prompt(Path("prompt.md")) + harness.connect_execute(LocalProcess(ProcessLimits(timedelta(seconds=1)))) + harness.connect_lifecycle( + PythonLoop(PythonEntrypoint(ModuleName("missing_module"), FunctionName("run"))) + ) + harness.connect_verifiers(_verifier("uppercase")) + + with pytest.raises(HarnessValidationError) as raised: + harness.process() + + assert raised.value.code is HarnessErrorCode.RUNTIME_INVALID + + +def test_python_verifier_is_terminated_at_environment_timeout(tmp_path: Path) -> None: + root = _repository(tmp_path) + revision = _revision(root) + environment = LocalProcess(ProcessLimits(timedelta(milliseconds=50))) + prepared = environment.prepare(revision, CanaryCase(CaseId("slow"), "ship")) + started = time.monotonic() + try: + result = _verifier("slow", "slow").verify( + RunResult.success(CaseId("slow"), "SHIP"), + prepared, + ) + finally: + environment.destroy(prepared) + + assert result.verdict is VerifierVerdict.ERROR + assert result.retryable + assert time.monotonic() - started < 1 From dda329a1443bd32e40d05c3628b83e404ec34fea Mon Sep 17 00:00:00 2001 From: divo12 Date: Sat, 22 Aug 2026 16:56:00 +0530 Subject: [PATCH 04/18] stabilize process failure timing test --- tests/test_runtime.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/test_runtime.py b/tests/test_runtime.py index b626def..b3b5adb 100644 --- a/tests/test_runtime.py +++ b/tests/test_runtime.py @@ -185,17 +185,22 @@ def test_local_process_reports_timeout_and_nonzero_exit(tmp_path: Path) -> None: timeout_script.write_text("import time\ntime.sleep(2)\n", encoding="utf-8") crash_script = root / "crash.py" crash_script.write_text("raise SystemExit(7)\n", encoding="utf-8") - environment = LocalProcess(ProcessLimits(timedelta(milliseconds=50))) - prepared = environment.prepare(revision, CanaryCase(CaseId("failure"), "input")) + timeout_environment = LocalProcess(ProcessLimits(timedelta(milliseconds=50))) + prepared = timeout_environment.prepare(revision, CanaryCase(CaseId("failure"), "input")) try: timed_out = CommandLoop(ProcessCommand((sys.executable, "timeout.py"))).invoke( CanaryCase(CaseId("timeout"), "input"), prepared, revision ) + finally: + timeout_environment.destroy(prepared) + crash_environment = LocalProcess(ProcessLimits(timedelta(seconds=1))) + prepared = crash_environment.prepare(revision, CanaryCase(CaseId("failure"), "input")) + try: crashed = CommandLoop(ProcessCommand((sys.executable, "crash.py"))).invoke( CanaryCase(CaseId("crash"), "input"), prepared, revision ) finally: - environment.destroy(prepared) + crash_environment.destroy(prepared) assert timed_out.status is RunStatus.TIMEOUT assert timed_out.error_code is RunErrorCode.TIMEOUT From 4a57057df700a553781a14722aa506b9499c2e6f Mon Sep 17 00:00:00 2001 From: divo12 Date: Sat, 22 Aug 2026 16:57:37 +0530 Subject: [PATCH 05/18] implement Mine admission and snapshots --- src/ofw/__init__.py | 18 +++ src/ofw/harness.py | 24 +++ src/ofw/mine.py | 376 ++++++++++++++++++++++++++++++++++++++++++++ tests/test_mine.py | 306 +++++++++++++++++++++++++++++++++++ 4 files changed, 724 insertions(+) create mode 100644 src/ofw/mine.py create mode 100644 tests/test_mine.py diff --git a/src/ofw/__init__.py b/src/ofw/__init__.py index 83b36fc..b339c02 100644 --- a/src/ofw/__init__.py +++ b/src/ofw/__init__.py @@ -27,6 +27,15 @@ WorkspaceFile, ) from ofw.harness import EditableFile, Harness, Subagent, Tool, editable +from ofw.mine import ( + Mine, + MineError, + MineErrorCode, + MiningPolicy, + ScoreName, + TracePartition, + TraceQualityThreshold, +) from ofw.observability.langfuse import ( CollectionError, CollectionErrorCode, @@ -77,6 +86,8 @@ class _OfwNamespace: CanaryCase = CanaryCase CaseId = CaseId ServiceName = ServiceName + MiningPolicy = MiningPolicy + ScoreName = ScoreName def editable(self, path: Path) -> EditableFile: return editable(path) @@ -121,6 +132,10 @@ def collect( "LocalProcess", "ModelFingerprint", "ModuleName", + "Mine", + "MineError", + "MineErrorCode", + "MiningPolicy", "RepositorySnapshot", "ProcessCommand", "ProcessLimits", @@ -130,11 +145,14 @@ def collect( "RunErrorCode", "RunResult", "RunStatus", + "ScoreName", "Sha256Digest", "ServiceName", "Subagent", "Tool", "TraceWindow", + "TracePartition", + "TraceQualityThreshold", "VerifierResult", "VerifierVerdict", "WorkspaceFile", diff --git a/src/ofw/harness.py b/src/ofw/harness.py index 831da84..e59d07f 100644 --- a/src/ofw/harness.py +++ b/src/ofw/harness.py @@ -100,6 +100,7 @@ class Harness: _execution: ExecutionEnvironment | None = field(default=None, init=False, repr=False) _lifecycle: LifecycleAdapter | None = field(default=None, init=False, repr=False) _verifiers: list[VerifierAdapter] = field(default_factory=list, init=False, repr=False) + _current_revision: HarnessRevision | None = field(default=None, init=False, repr=False) def __post_init__(self) -> None: if _NAME_PATTERN.fullmatch(self.name) is None: @@ -112,6 +113,7 @@ def connect_prompt(self, *sources: Path | EditableFile) -> Harness: return self def connect_tools(self, *tools: Tool) -> Harness: + self._current_revision = None for tool in tools: if any( registration.component is ComponentKind.TOOL and registration.name == tool.name @@ -126,6 +128,7 @@ def connect_skills(self, *sources: Path | EditableFile) -> Harness: return self def connect_subagents(self, *subagents: Subagent) -> Harness: + self._current_revision = None for subagent in subagents: if any( registration.component is ComponentKind.SUBAGENT @@ -146,14 +149,17 @@ def connect_middleware(self, *sources: Path | EditableFile) -> Harness: return self def connect_observability(self, project: LangfuseProject) -> Harness: + self._current_revision = None self._observability = project return self def connect_execute(self, environment: ExecutionEnvironment) -> Harness: + self._current_revision = None self._execution = environment return self def connect_lifecycle(self, lifecycle: LifecycleAdapter) -> Harness: + self._current_revision = None self._lifecycle = lifecycle return self @@ -161,14 +167,31 @@ def connect_verifiers(self, *verifiers: VerifierAdapter) -> Harness: for verifier in verifiers: if any(existing.name == verifier.name for existing in self._verifiers): raise HarnessValidationError(HarnessErrorCode.DUPLICATE_VERIFIER, verifier.name) + self._current_revision = None self._verifiers.append(verifier) return self + @property + def current_revision(self) -> HarnessRevision | None: + revision = self._current_revision + if revision is None: + return None + try: + root = _resolve_root(self.root) + components = _compile_components(root, self._files) + repository = _snapshot_repository(root) + except HarnessValidationError: + return None + if components != revision.components or repository != revision.repository: + return None + return revision + def _register_files( self, component: ComponentKind, sources: tuple[Path | EditableFile, ...], ) -> None: + self._current_revision = None for source in sources: self._files.append(_registration(component, source, None)) @@ -208,6 +231,7 @@ def process(self, *, canary: CanaryCase | None = None) -> HarnessRevision: _write_manifest(revision) if report is not None: _write_canary(revision, report) + self._current_revision = revision logger.debug("Compiled harness revision %s", revision.id) return revision diff --git a/src/ofw/mine.py b/src/ofw/mine.py new file mode 100644 index 0000000..71b87ba --- /dev/null +++ b/src/ofw/mine.py @@ -0,0 +1,376 @@ +"""Deterministic trace admission and immutable Mine snapshots.""" + +from __future__ import annotations + +import hashlib +import math +import os +import tempfile +from dataclasses import dataclass +from datetime import datetime +from enum import IntEnum, StrEnum +from pathlib import Path + +from pydantic import TypeAdapter + +from ofw.contracts import HarnessRevision, HarnessRevisionId, Sha256Digest +from ofw.harness import Harness +from ofw.observability.langfuse.domain import ( + AttributionLevel, + CollectionResult, + ObservationRecord, + ScoreDataType, + ScoreId, + ScoreRecord, + ScoreSource, + TraceId, + TraceRecord, +) +from ofw.observability.langfuse.store import CollectionStore + + +class TracePartition(StrEnum): + VERIFIED_GOOD = "verified_good" + VERIFIED_FAILURE = "verified_failure" + AMBIGUOUS = "ambiguous" + INVALID = "invalid" + + +class MineSchemaVersion(IntEnum): + V1 = 1 + + +class TraceQualityThreshold(StrEnum): + COMPLETE = "complete" + DEGRADED = "degraded" + + +class AdmissionReason(StrEnum): + VERIFIED_PASS = "verified_pass" # nosec B105 + VERIFIED_FAIL = "verified_fail" + MISSING_EVIDENCE = "missing_evidence" + CONFLICTING_EVIDENCE = "conflicting_evidence" + REVISION_ATTRIBUTION = "revision_attribution" + TRACE_QUALITY = "trace_quality" + EXCLUDED_TRACE = "excluded_trace" + + +class MineErrorCode(StrEnum): + STALE_HARNESS = "stale_harness" + REVISION_MISMATCH = "revision_mismatch" + INVALID_POLICY = "invalid_policy" + ARTIFACT_WRITE_FAILED = "artifact_write_failed" + + +class MineError(Exception): + __slots__ = ("code", "subject") + + def __init__(self, code: MineErrorCode, subject: str) -> None: + self.code = code + self.subject = subject + super().__init__(f"{code.value}: {subject}") + + +@dataclass(frozen=True, slots=True) +class ScoreName: + value: str + + def __post_init__(self) -> None: + if not self.value: + raise MineError(MineErrorCode.INVALID_POLICY, "empty score name") + + +@dataclass(frozen=True, slots=True) +class TraceTag: + value: str + + def __post_init__(self) -> None: + if not self.value: + raise MineError(MineErrorCode.INVALID_POLICY, "empty trace tag") + + +@dataclass(frozen=True, slots=True) +class MiningPolicy: + critical_scores: tuple[ScoreName, ...] + trusted_sources: tuple[ScoreSource, ...] + quality: TraceQualityThreshold + numeric_pass_at: float = 0.5 + excluded_tags: tuple[TraceTag, ...] = (TraceTag("ofw-internal"),) + + def __post_init__(self) -> None: + if ( + not self.critical_scores + or not self.trusted_sources + or len(set(self.critical_scores)) != len(self.critical_scores) + or len(set(self.trusted_sources)) != len(self.trusted_sources) + or not math.isfinite(self.numeric_pass_at) + ): + raise MineError(MineErrorCode.INVALID_POLICY, "evidence policy is required") + + @property + def digest(self) -> Sha256Digest: + return _digest_text( + "\0".join( + ( + *(score.value for score in self.critical_scores), + *(source.value for source in self.trusted_sources), + self.quality.value, + str(self.numeric_pass_at), + *(tag.value for tag in self.excluded_tags), + ) + ) + ) + + +@dataclass(frozen=True, slots=True) +class MineRunId: + value: str + + def __str__(self) -> str: + return self.value + + +@dataclass(frozen=True, slots=True) +class TraceSnapshot: + schema_version: MineSchemaVersion + revision_id: HarnessRevisionId + collection_digest: Sha256Digest + trace: TraceRecord + observations: tuple[ObservationRecord, ...] + scores: tuple[ScoreRecord, ...] + + +@dataclass(frozen=True, slots=True) +class TraceAdmission: + trace_id: TraceId + partition: TracePartition + reason: AdmissionReason + evidence_score_ids: tuple[ScoreId, ...] + snapshot_digest: Sha256Digest | None + snapshot_path: Path | None + + +@dataclass(frozen=True, slots=True) +class MineResult: + schema_version: MineSchemaVersion + id: MineRunId + revision_id: HarnessRevisionId + created_at: datetime + collection_digest: Sha256Digest + policy_digest: Sha256Digest + admissions: tuple[TraceAdmission, ...] + root: Path + + @property + def manifest_path(self) -> Path: + return self.root / ".ofw" / "mine" / str(self.id) / "manifest.json" + + @property + def verified_good_count(self) -> int: + return self._count(TracePartition.VERIFIED_GOOD) + + @property + def verified_failure_count(self) -> int: + return self._count(TracePartition.VERIFIED_FAILURE) + + @property + def ambiguous_count(self) -> int: + return self._count(TracePartition.AMBIGUOUS) + + @property + def invalid_count(self) -> int: + return self._count(TracePartition.INVALID) + + def to_json(self) -> str: + return _MINE_RESULT_ADAPTER.dump_json(self).decode() + + def _count(self, partition: TracePartition) -> int: + return sum(admission.partition is partition for admission in self.admissions) + + +_SNAPSHOT_ADAPTER: TypeAdapter[TraceSnapshot] = TypeAdapter(TraceSnapshot) +_MINE_RESULT_ADAPTER: TypeAdapter[MineResult] = TypeAdapter(MineResult) + + +@dataclass(frozen=True, slots=True) +class Mine: + source: Harness | HarnessRevision + collection: CollectionResult + policy: MiningPolicy + + def run(self) -> MineResult: + revision = _resolve_revision(self.source) + if self.collection.revision_id != revision.id: + raise MineError(MineErrorCode.REVISION_MISMATCH, str(revision.id)) + run_id = MineRunId( + "mine_" + + hashlib.sha256( + "\0".join( + ( + str(revision.id), + str(self.collection.snapshot_digest), + str(self.policy.digest), + ) + ).encode() + ).hexdigest() + ) + store = CollectionStore(self.collection.store_path) + try: + observations = store.observations(self.collection.observation_sync_id) + scores = store.scores(self.collection.score_sync_id) + finally: + store.close() + admissions = tuple( + self._admit(revision, run_id, trace, observations, scores) + for trace in sorted(self.collection.traces, key=_trace_sort_key) + ) + result = MineResult( + MineSchemaVersion.V1, + run_id, + revision.id, + self.collection.window.end, + self.collection.snapshot_digest, + self.policy.digest, + admissions, + revision.root, + ) + _write_artifact(result.manifest_path, f"{result.to_json()}\n".encode()) + return result + + def _admit( + self, + revision: HarnessRevision, + run_id: MineRunId, + trace: TraceRecord, + observations: tuple[ObservationRecord, ...], + scores: tuple[ScoreRecord, ...], + ) -> TraceAdmission: + # ponytail: linear scans are simplest for local v0; index when profiling shows pressure. + trace_observations = tuple( + observation for observation in observations if observation.id in trace.observation_ids + ) + trace_scores = tuple(score for score in scores if score.id in trace.score_ids) + partition, reason, evidence = self._classify(trace, trace_observations, trace_scores) + if partition is TracePartition.INVALID: + return TraceAdmission(trace.id, partition, reason, evidence, None, None) + snapshot_scores = tuple(score for score in trace_scores if score.id in evidence) + snapshot = TraceSnapshot( + MineSchemaVersion.V1, + revision.id, + self.collection.snapshot_digest, + trace, + trace_observations, + snapshot_scores, + ) + payload = _SNAPSHOT_ADAPTER.dump_json(snapshot) + digest = _digest_bytes(payload) + path = revision.root / ".ofw" / "mine" / str(run_id) / "traces" / f"{digest.value[7:]}.json" + _write_artifact(path, payload + b"\n") + return TraceAdmission(trace.id, partition, reason, evidence, digest, path) + + def _classify( + self, + trace: TraceRecord, + observations: tuple[ObservationRecord, ...], + scores: tuple[ScoreRecord, ...], + ) -> tuple[TracePartition, AdmissionReason, tuple[ScoreId, ...]]: + if trace.attribution is not AttributionLevel.EXACT: + return TracePartition.INVALID, AdmissionReason.REVISION_ATTRIBUTION, () + if not observations or not all( + any(observation.id == observation_id for observation in observations) + for observation_id in trace.observation_ids + ): + return TracePartition.INVALID, AdmissionReason.TRACE_QUALITY, () + if self.policy.quality is TraceQualityThreshold.COMPLETE and trace.gaps: + return TracePartition.INVALID, AdmissionReason.TRACE_QUALITY, () + if any( + tag.value in observation.tags + for tag in self.policy.excluded_tags + for observation in observations + ): + return TracePartition.INVALID, AdmissionReason.EXCLUDED_TRACE, () + evidence = tuple( + score + for score in scores + if score.source in self.policy.trusted_sources + and any(score.name == name.value for name in self.policy.critical_scores) + ) + verdicts: list[bool] = [] + missing = False + conflicting = False + for name in self.policy.critical_scores: + matching = tuple(score for score in evidence if score.name == name.value) + if not matching: + missing = True + continue + resolved = tuple( + _score_passes(score, self.policy.numeric_pass_at) for score in matching + ) + if any(verdict is None for verdict in resolved) or len(set(resolved)) != 1: + conflicting = True + continue + verdict = resolved[0] + if verdict is not None: + verdicts.append(verdict) + evidence_ids = tuple(score.id for score in evidence) + if conflicting: + return ( + TracePartition.AMBIGUOUS, + AdmissionReason.CONFLICTING_EVIDENCE, + evidence_ids, + ) + if any(not verdict for verdict in verdicts): + return TracePartition.VERIFIED_FAILURE, AdmissionReason.VERIFIED_FAIL, evidence_ids + if missing: + return TracePartition.AMBIGUOUS, AdmissionReason.MISSING_EVIDENCE, evidence_ids + return TracePartition.VERIFIED_GOOD, AdmissionReason.VERIFIED_PASS, evidence_ids + + +def _score_passes(score: ScoreRecord, numeric_pass_at: float) -> bool | None: + if score.data_type is ScoreDataType.BOOLEAN and isinstance(score.value, bool): + return score.value + if score.data_type is ScoreDataType.NUMERIC and isinstance(score.value, float): + return score.value >= numeric_pass_at + return None + + +def _trace_sort_key(trace: TraceRecord) -> str: + return trace.id.value + + +def _resolve_revision(source: Harness | HarnessRevision) -> HarnessRevision: + if isinstance(source, HarnessRevision): + return source + if source.current_revision is None: + raise MineError(MineErrorCode.STALE_HARNESS, source.name) + return source.current_revision + + +def _write_artifact(path: Path, payload: bytes) -> None: + try: + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp( + dir=path.parent, + prefix=f".{path.stem}-", + suffix=".tmp", + ) + temporary = Path(temporary_name) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + temporary.chmod(0o600) + temporary.replace(path) + finally: + temporary.unlink(missing_ok=True) + except OSError as error: + raise MineError(MineErrorCode.ARTIFACT_WRITE_FAILED, str(path)) from error + + +def _digest_text(value: str) -> Sha256Digest: + return _digest_bytes(value.encode()) + + +def _digest_bytes(value: bytes) -> Sha256Digest: + return Sha256Digest(f"sha256:{hashlib.sha256(value).hexdigest()}") diff --git a/tests/test_mine.py b/tests/test_mine.py new file mode 100644 index 0000000..f69d798 --- /dev/null +++ b/tests/test_mine.py @@ -0,0 +1,306 @@ +"""Deterministic Mine admission and immutable snapshot behavior.""" + +from __future__ import annotations + +import subprocess +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest + +from ofw import ( + Harness, + Mine, + MineError, + MineErrorCode, + MiningPolicy, + ScoreName, + TracePartition, + TraceQualityThreshold, +) +from ofw.contracts import HarnessRevision, Sha256Digest +from ofw.observability.langfuse.contracts import LangfuseConnectionId, TraceWindow +from ofw.observability.langfuse.domain import ( + AttributionLevel, + CollectionCapabilityReason, + CollectionResult, + CollectionSyncId, + JsonDocument, + ObservationId, + ObservationPage, + ObservationRecord, + ObservationType, + ProjectId, + ScoreDataType, + ScoreId, + ScorePage, + ScoreRecord, + ScoreSource, + ScoreSubject, + ScoreSubjectKind, + TraceGap, + TraceId, + TraceRecord, +) +from ofw.observability.langfuse.store import CollectionStore + + +def _run_git(root: Path, *arguments: str) -> None: + subprocess.run( + ("git", "-C", str(root), *arguments), + check=True, + capture_output=True, + text=True, + ) + + +def _harness(tmp_path: Path) -> Harness: + root = tmp_path / "mine-agent" + root.mkdir() + (root / "prompt.md").write_text("Be accurate.\n", encoding="utf-8") + (root / "memory.md").write_text("Known facts.\n", encoding="utf-8") + _run_git(root, "init", "-q") + _run_git(root, "config", "user.email", "fixture@example.test") + _run_git(root, "config", "user.name", "FixtureCo") + _run_git(root, "add", ".") + _run_git(root, "commit", "-qm", "fixture baseline") + harness = Harness("mine-agent", root=root) + harness.connect_prompt(Path("prompt.md")) + return harness + + +def _observation( + trace: str, + revision: HarnessRevision, + *, + tags: tuple[str, ...] = (), +) -> ObservationRecord: + return ObservationRecord( + id=ObservationId(f"observation-{trace}"), + trace_id=TraceId(trace), + start_time=datetime(2026, 8, 22, tzinfo=UTC), + end_time=datetime(2026, 8, 22, 0, 1, tzinfo=UTC), + project_id=ProjectId("project-1"), + parent_observation_id=None, + type=ObservationType.AGENT, + is_root=True, + name="agent-run", + level=None, + version="v1", + environment="production", + user_id=None, + session_id=f"session-{trace}", + created_at=datetime(2026, 8, 22, 0, 1, tzinfo=UTC), + updated_at=datetime(2026, 8, 22, 0, 1, tzinfo=UTC), + metadata=JsonDocument(f'{{"ofw.harness.revision":"{revision.id}"}}'), + usage=None, + costs=None, + total_cost=None, + tags=tags, + release=None, + trace_name="agent-run", + digest=Sha256Digest(f"sha256:observation-{trace}"), + ) + + +def _score(trace: str, value: bool, suffix: str = "") -> ScoreRecord: + return ScoreRecord( + id=ScoreId(f"score-{trace}{suffix}"), + project_id=ProjectId("project-1"), + name="correctness", + value=value, + data_type=ScoreDataType.BOOLEAN, + source=ScoreSource.ANNOTATION, + timestamp=datetime(2026, 8, 22, 0, 2, tzinfo=UTC), + environment="production", + created_at=datetime(2026, 8, 22, 0, 2, tzinfo=UTC), + updated_at=datetime(2026, 8, 22, 0, 2, tzinfo=UTC), + comment="reviewed", + metadata=None, + subject=ScoreSubject(ScoreSubjectKind.TRACE, trace, None), + digest=Sha256Digest(f"sha256:score-{trace}{suffix}"), + ) + + +def _trace( + trace: str, + scores: tuple[ScoreRecord, ...], + *, + attribution: AttributionLevel = AttributionLevel.EXACT, + gaps: tuple[TraceGap, ...] = (), +) -> TraceRecord: + return TraceRecord( + id=TraceId(trace), + observation_ids=(ObservationId(f"observation-{trace}"),), + root_observation_ids=(ObservationId(f"observation-{trace}"),), + score_ids=tuple(score.id for score in scores), + session_id=f"session-{trace}", + environment="production", + release=None, + attribution=attribution, + gaps=gaps, + digest=Sha256Digest(f"sha256:trace-{trace}"), + ) + + +def _collection( + tmp_path: Path, + revision: HarnessRevision, + *, + conflict: bool = False, +) -> CollectionResult: + good_score = _score("good", True) + failed_score = _score("failed", False) + conflicting_scores: tuple[ScoreRecord, ...] = ( + (_score("ambiguous", True, "-pass"), _score("ambiguous", False, "-fail")) + if conflict + else () + ) + scores: tuple[ScoreRecord, ...] = (good_score, failed_score, *conflicting_scores) + observations: tuple[ObservationRecord, ...] = ( + _observation("good", revision), + _observation("failed", revision), + _observation("ambiguous", revision), + _observation("invalid", revision, tags=("ofw-internal",)), + ) + traces: tuple[TraceRecord, ...] = ( + _trace("good", (good_score,)), + _trace("failed", (failed_score,)), + _trace("ambiguous", conflicting_scores), + _trace("invalid", (), attribution=AttributionLevel.MISSING), + ) + observation_sync = CollectionSyncId("observations-mine") + score_sync = CollectionSyncId("scores-mine") + store_path = tmp_path / "collection.sqlite" + store = CollectionStore(store_path) + try: + store.commit_observation_page( + "connection-1", + observation_sync, + ObservationPage(observations, None), + ) + store.commit_score_page("connection-1", score_sync, ScorePage(scores, None)) + finally: + store.close() + start = datetime(2026, 8, 22, tzinfo=UTC) + return CollectionResult( + revision_id=revision.id, + connection_id=LangfuseConnectionId("connection-1"), + window=TraceWindow(start, start + timedelta(hours=1)), + observation_sync_id=observation_sync, + score_sync_id=score_sync, + traces=traces, + observation_count=len(observations), + score_count=len(scores), + gap_count=0, + snapshot_digest=Sha256Digest("sha256:collection"), + capability=CollectionCapabilityReason.READY, + store_path=store_path, + ) + + +def _policy() -> MiningPolicy: + return MiningPolicy( + critical_scores=(ScoreName("correctness"),), + trusted_sources=(ScoreSource.ANNOTATION,), + quality=TraceQualityThreshold.COMPLETE, + ) + + +def test_mine_partitions_only_from_trusted_independent_evidence(tmp_path: Path) -> None: + revision = _harness(tmp_path).process() + result = Mine(revision, _collection(tmp_path, revision), _policy()).run() + + partitions = tuple(admission.partition for admission in result.admissions) + assert partitions == ( + TracePartition.AMBIGUOUS, + TracePartition.VERIFIED_FAILURE, + TracePartition.VERIFIED_GOOD, + TracePartition.INVALID, + ) + assert result.verified_good_count == 1 + assert result.verified_failure_count == 1 + assert result.ambiguous_count == 1 + assert result.invalid_count == 1 + + +def test_conflicting_trusted_scores_remain_ambiguous(tmp_path: Path) -> None: + revision = _harness(tmp_path).process() + result = Mine( + revision, + _collection(tmp_path, revision, conflict=True), + _policy(), + ).run() + ambiguous = next( + admission for admission in result.admissions if admission.trace_id == TraceId("ambiguous") + ) + + assert ambiguous.partition is TracePartition.AMBIGUOUS + + +def test_known_critical_failure_wins_over_other_missing_evidence(tmp_path: Path) -> None: + revision = _harness(tmp_path).process() + policy = MiningPolicy( + critical_scores=(ScoreName("correctness"), ScoreName("safety")), + trusted_sources=(ScoreSource.ANNOTATION,), + quality=TraceQualityThreshold.COMPLETE, + ) + result = Mine(revision, _collection(tmp_path, revision), policy).run() + failed = next( + admission for admission in result.admissions if admission.trace_id == TraceId("failed") + ) + + assert failed.partition is TracePartition.VERIFIED_FAILURE + + +def test_mine_is_content_addressed_and_idempotent(tmp_path: Path) -> None: + revision = _harness(tmp_path).process() + collection = _collection(tmp_path, revision) + + first = Mine(revision, collection, _policy()).run() + second = Mine(revision, collection, _policy()).run() + + assert first == second + assert first.manifest_path.read_text(encoding="utf-8") == f"{first.to_json()}\n" + assert all(admission.snapshot_path is not None for admission in first.admissions[:-1]) + assert first.admissions[-1].snapshot_path is None + + +def test_processed_harness_is_accepted_and_later_connection_makes_it_stale( + tmp_path: Path, +) -> None: + harness = _harness(tmp_path) + revision = harness.process() + collection = _collection(tmp_path, revision) + assert Mine(harness, collection, _policy()).run().revision_id == revision.id + + harness.connect_skills(Path("memory.md")) + + with pytest.raises(MineError) as raised: + Mine(harness, collection, _policy()).run() + assert raised.value.code is MineErrorCode.STALE_HARNESS + + +def test_processed_harness_is_stale_after_external_file_change(tmp_path: Path) -> None: + harness = _harness(tmp_path) + revision = harness.process() + collection = _collection(tmp_path, revision) + (harness.root / "prompt.md").write_text("Changed externally.\n", encoding="utf-8") + + with pytest.raises(MineError) as raised: + Mine(harness, collection, _policy()).run() + + assert raised.value.code is MineErrorCode.STALE_HARNESS + + +def test_collection_from_another_revision_is_rejected(tmp_path: Path) -> None: + first = _harness(tmp_path) + first_revision = first.process() + collection = _collection(tmp_path, first_revision) + (first.root / "prompt.md").write_text("Changed.\n", encoding="utf-8") + second_revision = first.process() + + with pytest.raises(MineError) as raised: + Mine(second_revision, collection, _policy()).run() + + assert raised.value.code is MineErrorCode.REVISION_MISMATCH From c0310a02e74b6c1715b0fe2b9cc9a99c3aa29724 Mon Sep 17 00:00:00 2001 From: divo12 Date: Sat, 22 Aug 2026 17:00:52 +0530 Subject: [PATCH 06/18] tighten Mine revision lineage --- src/ofw/harness.py | 7 ++++++- src/ofw/mine.py | 12 +++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/ofw/harness.py b/src/ofw/harness.py index e59d07f..e6fa65a 100644 --- a/src/ofw/harness.py +++ b/src/ofw/harness.py @@ -180,9 +180,14 @@ def current_revision(self) -> HarnessRevision | None: root = _resolve_root(self.root) components = _compile_components(root, self._files) repository = _snapshot_repository(root) + runtime = self._runtime(root) except HarnessValidationError: return None - if components != revision.components or repository != revision.repository: + if ( + components != revision.components + or repository != revision.repository + or runtime != revision.runtime + ): return None return revision diff --git a/src/ofw/mine.py b/src/ofw/mine.py index 71b87ba..24c56bb 100644 --- a/src/ofw/mine.py +++ b/src/ofw/mine.py @@ -76,7 +76,7 @@ class ScoreName: value: str def __post_init__(self) -> None: - if not self.value: + if not self.value or "\0" in self.value: raise MineError(MineErrorCode.INVALID_POLICY, "empty score name") @@ -85,7 +85,7 @@ class TraceTag: value: str def __post_init__(self) -> None: - if not self.value: + if not self.value or "\0" in self.value: raise MineError(MineErrorCode.INVALID_POLICY, "empty trace tag") @@ -155,7 +155,7 @@ class MineResult: schema_version: MineSchemaVersion id: MineRunId revision_id: HarnessRevisionId - created_at: datetime + source_watermark: datetime collection_digest: Sha256Digest policy_digest: Sha256Digest admissions: tuple[TraceAdmission, ...] @@ -210,6 +210,7 @@ def run(self) -> MineResult: str(revision.id), str(self.collection.snapshot_digest), str(self.policy.digest), + str(int(MineSchemaVersion.V1)), ) ).encode() ).hexdigest() @@ -341,9 +342,10 @@ def _trace_sort_key(trace: TraceRecord) -> str: def _resolve_revision(source: Harness | HarnessRevision) -> HarnessRevision: if isinstance(source, HarnessRevision): return source - if source.current_revision is None: + revision = source.current_revision + if revision is None: raise MineError(MineErrorCode.STALE_HARNESS, source.name) - return source.current_revision + return revision def _write_artifact(path: Path, payload: bytes) -> None: From 5ce7dc884d6190565979e4545b37d2b7115cbcfe Mon Sep 17 00:00:00 2001 From: divo12 Date: Sat, 22 Aug 2026 17:05:51 +0530 Subject: [PATCH 07/18] enforce Mine privacy and score correlation --- src/ofw/mine.py | 129 +++++++++++++++++++++++++++++++++++++++++---- tests/test_mine.py | 47 ++++++++++++++++- 2 files changed, 166 insertions(+), 10 deletions(-) diff --git a/src/ofw/mine.py b/src/ofw/mine.py index 24c56bb..2a7eb59 100644 --- a/src/ofw/mine.py +++ b/src/ofw/mine.py @@ -15,14 +15,21 @@ from ofw.contracts import HarnessRevision, HarnessRevisionId, Sha256Digest from ofw.harness import Harness +from ofw.observability.langfuse.contracts import TraceWindow from ofw.observability.langfuse.domain import ( AttributionLevel, CollectionResult, + ObservationId, + ObservationLevel, ObservationRecord, + ObservationType, ScoreDataType, ScoreId, ScoreRecord, ScoreSource, + ScoreSubject, + ScoreSubjectKind, + TraceGap, TraceId, TraceRecord, ) @@ -135,9 +142,47 @@ class TraceSnapshot: schema_version: MineSchemaVersion revision_id: HarnessRevisionId collection_digest: Sha256Digest - trace: TraceRecord - observations: tuple[ObservationRecord, ...] - scores: tuple[ScoreRecord, ...] + trace: SnapshotTrace + observations: tuple[SnapshotObservation, ...] + scores: tuple[SnapshotScore, ...] + + +@dataclass(frozen=True, slots=True) +class SnapshotTrace: + id: TraceId + observation_ids: tuple[ObservationId, ...] + root_observation_ids: tuple[ObservationId, ...] + evidence_score_ids: tuple[ScoreId, ...] + attribution: AttributionLevel + gaps: tuple[TraceGap, ...] + digest: Sha256Digest + + +@dataclass(frozen=True, slots=True) +class SnapshotObservation: + id: ObservationId + trace_id: TraceId | None + start_time: datetime + end_time: datetime | None + parent_observation_id: ObservationId | None + type: ObservationType + is_root: bool | None + name: str | None + level: ObservationLevel | None + status_message: str | None + digest: Sha256Digest + + +@dataclass(frozen=True, slots=True) +class SnapshotScore: + id: ScoreId + name: str + value: bool | float | str + data_type: ScoreDataType + source: ScoreSource + timestamp: datetime + subject: ScoreSubject | None + digest: Sha256Digest @dataclass(frozen=True, slots=True) @@ -155,7 +200,7 @@ class MineResult: schema_version: MineSchemaVersion id: MineRunId revision_id: HarnessRevisionId - source_watermark: datetime + window: TraceWindow collection_digest: Sha256Digest policy_digest: Sha256Digest admissions: tuple[TraceAdmission, ...] @@ -211,6 +256,8 @@ def run(self) -> MineResult: str(self.collection.snapshot_digest), str(self.policy.digest), str(int(MineSchemaVersion.V1)), + self.collection.window.start.isoformat(), + self.collection.window.end.isoformat(), ) ).encode() ).hexdigest() @@ -229,7 +276,7 @@ def run(self) -> MineResult: MineSchemaVersion.V1, run_id, revision.id, - self.collection.window.end, + self.collection.window, self.collection.snapshot_digest, self.policy.digest, admissions, @@ -250,7 +297,11 @@ def _admit( trace_observations = tuple( observation for observation in observations if observation.id in trace.observation_ids ) - trace_scores = tuple(score for score in scores if score.id in trace.score_ids) + trace_scores = tuple( + score + for score in scores + if score.id in trace.score_ids and _score_belongs(score, trace, trace_observations) + ) partition, reason, evidence = self._classify(trace, trace_observations, trace_scores) if partition is TracePartition.INVALID: return TraceAdmission(trace.id, partition, reason, evidence, None, None) @@ -259,9 +310,9 @@ def _admit( MineSchemaVersion.V1, revision.id, self.collection.snapshot_digest, - trace, - trace_observations, - snapshot_scores, + _snapshot_trace(trace, evidence), + tuple(_snapshot_observation(observation) for observation in trace_observations), + tuple(_snapshot_score(score) for score in snapshot_scores), ) payload = _SNAPSHOT_ADAPTER.dump_json(snapshot) digest = _digest_bytes(payload) @@ -335,6 +386,66 @@ def _score_passes(score: ScoreRecord, numeric_pass_at: float) -> bool | None: return None +def _score_belongs( + score: ScoreRecord, + trace: TraceRecord, + observations: tuple[ObservationRecord, ...], +) -> bool: + subject = score.subject + if subject is None: + return False + if subject.kind is ScoreSubjectKind.TRACE: + return subject.id == trace.id.value + if subject.kind is ScoreSubjectKind.OBSERVATION: + return any(observation.id.value == subject.id for observation in observations) and ( + subject.trace_id is None or subject.trace_id == trace.id + ) + if subject.kind is ScoreSubjectKind.SESSION: + return trace.session_id is not None and subject.id == trace.session_id + return False + + +def _snapshot_trace(trace: TraceRecord, evidence: tuple[ScoreId, ...]) -> SnapshotTrace: + return SnapshotTrace( + trace.id, + trace.observation_ids, + trace.root_observation_ids, + evidence, + trace.attribution, + trace.gaps, + trace.digest, + ) + + +def _snapshot_observation(observation: ObservationRecord) -> SnapshotObservation: + return SnapshotObservation( + observation.id, + observation.trace_id, + observation.start_time, + observation.end_time, + observation.parent_observation_id, + observation.type, + observation.is_root, + observation.name, + observation.level, + observation.status_message, + observation.digest, + ) + + +def _snapshot_score(score: ScoreRecord) -> SnapshotScore: + return SnapshotScore( + score.id, + score.name, + score.value, + score.data_type, + score.source, + score.timestamp, + score.subject, + score.digest, + ) + + def _trace_sort_key(trace: TraceRecord) -> str: return trace.id.value diff --git a/tests/test_mine.py b/tests/test_mine.py index f69d798..677c6aa 100644 --- a/tests/test_mine.py +++ b/tests/test_mine.py @@ -3,6 +3,7 @@ from __future__ import annotations import subprocess +from dataclasses import replace from datetime import UTC, datetime, timedelta from pathlib import Path @@ -92,7 +93,7 @@ def _observation( session_id=f"session-{trace}", created_at=datetime(2026, 8, 22, 0, 1, tzinfo=UTC), updated_at=datetime(2026, 8, 22, 0, 1, tzinfo=UTC), - metadata=JsonDocument(f'{{"ofw.harness.revision":"{revision.id}"}}'), + metadata=JsonDocument(f'{{"ofw.harness.revision":"{revision.id}","secret":"token"}}'), usage=None, costs=None, total_cost=None, @@ -148,8 +149,14 @@ def _collection( revision: HarnessRevision, *, conflict: bool = False, + foreign_good_score: bool = False, ) -> CollectionResult: good_score = _score("good", True) + if foreign_good_score: + good_score = replace( + good_score, + subject=ScoreSubject(ScoreSubjectKind.TRACE, "other-trace", None), + ) failed_score = _score("failed", False) conflicting_scores: tuple[ScoreRecord, ...] = ( (_score("ambiguous", True, "-pass"), _score("ambiguous", False, "-fail")) @@ -264,6 +271,44 @@ def test_mine_is_content_addressed_and_idempotent(tmp_path: Path) -> None: assert first.manifest_path.read_text(encoding="utf-8") == f"{first.to_json()}\n" assert all(admission.snapshot_path is not None for admission in first.admissions[:-1]) assert first.admissions[-1].snapshot_path is None + good = next( + admission for admission in first.admissions if admission.trace_id == TraceId("good") + ) + assert good.snapshot_path is not None + snapshot = good.snapshot_path.read_text(encoding="utf-8") + assert "token" not in snapshot + assert "reviewed" not in snapshot + + +def test_foreign_score_subject_cannot_label_trace(tmp_path: Path) -> None: + revision = _harness(tmp_path).process() + result = Mine( + revision, + _collection(tmp_path, revision, foreign_good_score=True), + _policy(), + ).run() + good = next( + admission for admission in result.admissions if admission.trace_id == TraceId("good") + ) + + assert good.partition is TracePartition.AMBIGUOUS + + +def test_source_window_is_part_of_mine_identity(tmp_path: Path) -> None: + revision = _harness(tmp_path).process() + collection = _collection(tmp_path, revision) + shifted = replace( + collection, + window=TraceWindow( + collection.window.start + timedelta(hours=1), + collection.window.end + timedelta(hours=1), + ), + ) + + first = Mine(revision, collection, _policy()).run() + second = Mine(revision, shifted, _policy()).run() + + assert first.id != second.id def test_processed_harness_is_accepted_and_later_connection_makes_it_stale( From 99df9359492d2089faecefb4d85e55349ab39c27 Mon Sep 17 00:00:00 2001 From: divo12 Date: Sun, 23 Aug 2026 12:53:11 +0530 Subject: [PATCH 08/18] freeze redacted Mine content references --- src/ofw/__init__.py | 12 ++++++ src/ofw/mine.py | 90 +++++++++++++++++++++++++++++++++++++++++---- tests/test_mine.py | 79 +++++++++++++++++++++++++++++++++++++-- 3 files changed, 171 insertions(+), 10 deletions(-) diff --git a/src/ofw/__init__.py b/src/ofw/__init__.py index af95060..bda3cea 100644 --- a/src/ofw/__init__.py +++ b/src/ofw/__init__.py @@ -31,10 +31,13 @@ Mine, MineError, MineErrorCode, + MineResult, MiningPolicy, ScoreName, + SnapshotContentReference, TracePartition, TraceQualityThreshold, + read_snapshot_content, ) from ofw.observability.langfuse import ( CollectionError, @@ -141,6 +144,13 @@ def read_observation_content( ) -> ObservationContent: return read_observation_content(collection, reference) + def read_snapshot_content( + self, + result: MineResult, + reference: SnapshotContentReference, + ) -> ObservationContent: + return read_snapshot_content(result, reference) + ofw = _OfwNamespace() @@ -197,6 +207,7 @@ def read_observation_content( "Sha256Digest", "ServiceName", "SecretEnvironmentVariable", + "SnapshotContentReference", "Subagent", "Tool", "TraceWindow", @@ -213,6 +224,7 @@ def read_observation_content( "ofw", "propagate_attributes", "read_observation_content", + "read_snapshot_content", "read_trace_observations", "search_observation_content", ] diff --git a/src/ofw/mine.py b/src/ofw/mine.py index 2a7eb59..5a3b080 100644 --- a/src/ofw/mine.py +++ b/src/ofw/mine.py @@ -15,10 +15,12 @@ from ofw.contracts import HarnessRevision, HarnessRevisionId, Sha256Digest from ofw.harness import Harness -from ofw.observability.langfuse.contracts import TraceWindow +from ofw.observability.langfuse.contracts import CollectionError, TraceWindow from ofw.observability.langfuse.domain import ( AttributionLevel, CollectionResult, + ObservationContent, + ObservationContentReference, ObservationId, ObservationLevel, ObservationRecord, @@ -67,6 +69,7 @@ class MineErrorCode(StrEnum): REVISION_MISMATCH = "revision_mismatch" INVALID_POLICY = "invalid_policy" ARTIFACT_WRITE_FAILED = "artifact_write_failed" + CONTENT_INVALID = "content_invalid" class MineError(Exception): @@ -171,6 +174,14 @@ class SnapshotObservation: level: ObservationLevel | None status_message: str | None digest: Sha256Digest + input_content: SnapshotContentReference | None = None + output_content: SnapshotContentReference | None = None + + +@dataclass(frozen=True, slots=True) +class SnapshotContentReference: + content: ObservationContentReference + path: Path @dataclass(frozen=True, slots=True) @@ -266,12 +277,12 @@ def run(self) -> MineResult: try: observations = store.observations(self.collection.observation_sync_id) scores = store.scores(self.collection.score_sync_id) + admissions = tuple( + self._admit(revision, run_id, trace, observations, scores, store) + for trace in sorted(self.collection.traces, key=_trace_sort_key) + ) finally: store.close() - admissions = tuple( - self._admit(revision, run_id, trace, observations, scores) - for trace in sorted(self.collection.traces, key=_trace_sort_key) - ) result = MineResult( MineSchemaVersion.V1, run_id, @@ -292,6 +303,7 @@ def _admit( trace: TraceRecord, observations: tuple[ObservationRecord, ...], scores: tuple[ScoreRecord, ...], + store: CollectionStore, ) -> TraceAdmission: # ponytail: linear scans are simplest for local v0; index when profiling shows pressure. trace_observations = tuple( @@ -311,7 +323,10 @@ def _admit( revision.id, self.collection.snapshot_digest, _snapshot_trace(trace, evidence), - tuple(_snapshot_observation(observation) for observation in trace_observations), + tuple( + self._snapshot_observation(revision, run_id, store, observation) + for observation in trace_observations + ), tuple(_snapshot_score(score) for score in snapshot_scores), ) payload = _SNAPSHOT_ADAPTER.dump_json(snapshot) @@ -320,6 +335,43 @@ def _admit( _write_artifact(path, payload + b"\n") return TraceAdmission(trace.id, partition, reason, evidence, digest, path) + def _snapshot_observation( + self, + revision: HarnessRevision, + run_id: MineRunId, + store: CollectionStore, + observation: ObservationRecord, + ) -> SnapshotObservation: + return _snapshot_observation( + observation, + self._snapshot_content(revision, run_id, store, observation.input_content), + self._snapshot_content(revision, run_id, store, observation.output_content), + ) + + def _snapshot_content( + self, + revision: HarnessRevision, + run_id: MineRunId, + store: CollectionStore, + reference: ObservationContentReference | None, + ) -> SnapshotContentReference | None: + if reference is None: + return None + try: + content = store.read_content(self.collection.observation_sync_id, reference) + except CollectionError as error: + raise MineError(MineErrorCode.CONTENT_INVALID, str(reference.digest)) from error + path = ( + revision.root + / ".ofw" + / "mine" + / str(run_id) + / "content" + / f"{reference.digest.value[7:]}.txt" + ) + _write_artifact(path, content.text.encode()) + return SnapshotContentReference(reference, path) + def _classify( self, trace: TraceRecord, @@ -417,7 +469,11 @@ def _snapshot_trace(trace: TraceRecord, evidence: tuple[ScoreId, ...]) -> Snapsh ) -def _snapshot_observation(observation: ObservationRecord) -> SnapshotObservation: +def _snapshot_observation( + observation: ObservationRecord, + input_content: SnapshotContentReference | None, + output_content: SnapshotContentReference | None, +) -> SnapshotObservation: return SnapshotObservation( observation.id, observation.trace_id, @@ -430,6 +486,8 @@ def _snapshot_observation(observation: ObservationRecord) -> SnapshotObservation observation.level, observation.status_message, observation.digest, + input_content, + output_content, ) @@ -487,3 +545,21 @@ def _digest_text(value: str) -> Sha256Digest: def _digest_bytes(value: bytes) -> Sha256Digest: return Sha256Digest(f"sha256:{hashlib.sha256(value).hexdigest()}") + + +def read_snapshot_content( + result: MineResult, + reference: SnapshotContentReference, +) -> ObservationContent: + try: + allowed = ( + result.root / ".ofw" / "mine" / str(result.id) / "content" + ).resolve(strict=True) + resolved = reference.path.resolve(strict=True) + resolved.relative_to(allowed) + expected_name = f"{reference.content.digest.value[7:]}.txt" + if resolved.name != expected_name: + raise ValueError("content path does not match digest") + return ObservationContent(reference.content, resolved.read_text(encoding="utf-8")) + except (OSError, ValueError) as error: + raise MineError(MineErrorCode.CONTENT_INVALID, str(reference.path)) from error diff --git a/tests/test_mine.py b/tests/test_mine.py index 0424e36..f6f5c4a 100644 --- a/tests/test_mine.py +++ b/tests/test_mine.py @@ -8,6 +8,7 @@ from pathlib import Path import pytest +from pydantic import TypeAdapter from ofw import ( Harness, @@ -18,8 +19,10 @@ ScoreName, TracePartition, TraceQualityThreshold, + read_snapshot_content, ) from ofw.contracts import HarnessRevision, Sha256Digest +from ofw.mine import TraceSnapshot from ofw.observability.langfuse.contracts import ( LangfuseConnectionId, ObservationContentPolicy, @@ -31,6 +34,8 @@ CollectionResult, CollectionSyncId, JsonDocument, + ObservationContent, + ObservationContentReference, ObservationId, ObservationPage, ObservationRecord, @@ -154,6 +159,7 @@ def _collection( *, conflict: bool = False, foreign_good_score: bool = False, + content: bool = False, ) -> CollectionResult: good_score = _score("good", True) if foreign_good_score: @@ -168,12 +174,43 @@ def _collection( else () ) scores: tuple[ScoreRecord, ...] = (good_score, failed_score, *conflicting_scores) - observations: tuple[ObservationRecord, ...] = ( + plain_observations: tuple[ObservationRecord, ...] = ( _observation("good", revision), _observation("failed", revision), _observation("ambiguous", revision), _observation("invalid", revision, tags=("ofw-internal",)), ) + contents: tuple[ObservationContent, ...] = () + observations = plain_observations + if content: + captured: tuple[ObservationRecord, ...] = () + for observation in plain_observations: + trace_id = observation.trace_id + assert trace_id is not None + input_text = f"request {trace_id.value} [REDACTED_EMAIL]" + output_text = f"result {trace_id.value}" + input_reference = ObservationContentReference.for_text( + input_text, + truncated=False, + ) + output_reference = ObservationContentReference.for_text( + output_text, + truncated=False, + ) + captured = ( + *captured, + replace( + observation, + input_content=input_reference, + output_content=output_reference, + ), + ) + contents = ( + *contents, + ObservationContent(input_reference, input_text), + ObservationContent(output_reference, output_text), + ) + observations = captured traces: tuple[TraceRecord, ...] = ( _trace("good", (good_score,)), _trace("failed", (failed_score,)), @@ -188,7 +225,7 @@ def _collection( store.commit_observation_page( "connection-1", observation_sync, - ObservationPage(observations, None), + ObservationPage(observations, None, contents), ) store.commit_score_page("connection-1", score_sync, ScorePage(scores, None)) finally: @@ -207,7 +244,14 @@ def _collection( snapshot_digest=Sha256Digest("sha256:collection"), capability=CollectionCapabilityReason.READY, store_path=store_path, - content_policy=ObservationContentPolicy.metadata_only(), + content_policy=( + ObservationContentPolicy.redacted( + maximum_bytes_per_field=4096, + secret_environment_variables=(), + ) + if content + else ObservationContentPolicy.metadata_only() + ), ) @@ -299,6 +343,35 @@ def test_foreign_score_subject_cannot_label_trace(tmp_path: Path) -> None: assert good.partition is TracePartition.AMBIGUOUS +def test_mine_freezes_redacted_content_as_verified_artifact_references( + tmp_path: Path, +) -> None: + revision = _harness(tmp_path).process() + collection = _collection(tmp_path, revision, content=True) + + result = Mine(revision, collection, _policy()).run() + + failed = next( + admission for admission in result.admissions if admission.trace_id == TraceId("failed") + ) + assert failed.snapshot_path is not None + snapshot = TypeAdapter(TraceSnapshot).validate_json(failed.snapshot_path.read_bytes()) + observation = snapshot.observations[0] + assert observation.input_content is not None + assert observation.output_content is not None + assert "request failed" not in failed.snapshot_path.read_text(encoding="utf-8") + input_content = read_snapshot_content(result, observation.input_content) + assert input_content.text == "request failed [REDACTED_EMAIL]" + assert input_content.reference == observation.input_content.content + collection.store_path.unlink() + assert read_snapshot_content(result, observation.input_content) == input_content + + observation.input_content.path.write_text("tampered", encoding="utf-8") + with pytest.raises(MineError) as raised: + read_snapshot_content(result, observation.input_content) + assert raised.value.code is MineErrorCode.CONTENT_INVALID + + def test_source_window_is_part_of_mine_identity(tmp_path: Path) -> None: revision = _harness(tmp_path).process() collection = _collection(tmp_path, revision) From 0cd892848ebcda25ee281be259027e2401fda772 Mon Sep 17 00:00:00 2001 From: divo12 Date: Tue, 25 Aug 2026 08:07:02 +0530 Subject: [PATCH 09/18] refactor mine around evidence-backed failure mining --- src/ofw/__init__.py | 62 ++- src/ofw/harness.py | 4 - src/ofw/mine.py | 1018 ++++++++++++++++++++++++------------------- tests/test_mine.py | 797 +++++++++++++++++++-------------- 4 files changed, 1085 insertions(+), 796 deletions(-) diff --git a/src/ofw/__init__.py b/src/ofw/__init__.py index b370d84..5c8c0e2 100644 --- a/src/ofw/__init__.py +++ b/src/ofw/__init__.py @@ -28,16 +28,28 @@ ) from ofw.harness import EditableFile, Harness, Subagent, Tool, editable from ofw.mine import ( + CompletionCheck, + CompletionStatus, + EnvironmentCheck, + EnvironmentCheckId, + EnvironmentCheckRequest, + EnvironmentSource, + EnvironmentSourceId, + EnvironmentSourceKind, + EnvironmentVerification, + EvidenceKind, + EvidenceRecordId, + EvidenceReference, + FailureMiningResult, + FailureMiningRun, + FailureSource, + FailureSourceId, + FailureSourceKind, Mine, - MineError, - MineErrorCode, - MineResult, - MiningPolicy, - ScoreName, - SnapshotContentReference, - TracePartition, - TraceQualityThreshold, - read_snapshot_content, + MiningInvalidReason, + MiningNomination, + MiningVerdict, + TraceMiningCase, ) from ofw.observability.langfuse import ( CollectionError, @@ -125,14 +137,6 @@ def read_observation_content( ) -> ObservationContent: return read_observation_content(collection, reference) - def read_snapshot_content( - self, - result: MineResult, - reference: SnapshotContentReference, - ) -> ObservationContent: - return read_snapshot_content(result, reference) - - ofw = _OfwNamespace() __all__ = [ @@ -145,8 +149,25 @@ def read_snapshot_content( "CollectionResult", "CommandLoop", "CommandVerifier", + "CompletionCheck", + "CompletionStatus", "E2BSandbox", "EditableFile", + "EnvironmentCheck", + "EnvironmentCheckId", + "EnvironmentCheckRequest", + "EnvironmentSource", + "EnvironmentSourceId", + "EnvironmentSourceKind", + "EnvironmentVerification", + "EvidenceKind", + "EvidenceRecordId", + "EvidenceReference", + "FailureMiningResult", + "FailureMiningRun", + "FailureSource", + "FailureSourceId", + "FailureSourceKind", "GitCommit", "Harness", "HarnessAsset", @@ -155,12 +176,15 @@ def read_snapshot_content( "HarnessRevision", "HarnessRevisionId", "HarnessValidationError", - "FunctionName", "Langfuse", "LangfuseOtelSpanAttributes", "LangfuseProject", "LangfuseSpan", "ModelFingerprint", + "Mine", + "MiningInvalidReason", + "MiningNomination", + "MiningVerdict", "ObservationContent", "ObservationContentField", "ObservationContentHit", @@ -177,6 +201,7 @@ def read_snapshot_content( "Subagent", "Tool", "TraceWindow", + "TraceMiningCase", "VerifierResult", "VerifierVerdict", "WorkspaceFile", @@ -188,7 +213,6 @@ def read_snapshot_content( "ofw", "propagate_attributes", "read_observation_content", - "read_snapshot_content", "read_trace_observations", "search_observation_content", ] diff --git a/src/ofw/harness.py b/src/ofw/harness.py index 454775a..831da84 100644 --- a/src/ofw/harness.py +++ b/src/ofw/harness.py @@ -112,7 +112,6 @@ def connect_prompt(self, *sources: Path | EditableFile) -> Harness: return self def connect_tools(self, *tools: Tool) -> Harness: - self._current_revision = None for tool in tools: if any( registration.component is ComponentKind.TOOL and registration.name == tool.name @@ -127,7 +126,6 @@ def connect_skills(self, *sources: Path | EditableFile) -> Harness: return self def connect_subagents(self, *subagents: Subagent) -> Harness: - self._current_revision = None for subagent in subagents: if any( registration.component is ComponentKind.SUBAGENT @@ -148,7 +146,6 @@ def connect_middleware(self, *sources: Path | EditableFile) -> Harness: return self def connect_observability(self, project: LangfuseProject) -> Harness: - self._current_revision = None self._observability = project return self @@ -172,7 +169,6 @@ def _register_files( component: ComponentKind, sources: tuple[Path | EditableFile, ...], ) -> None: - self._current_revision = None for source in sources: self._files.append(_registration(component, source, None)) diff --git a/src/ofw/mine.py b/src/ofw/mine.py index 5a3b080..13e6f43 100644 --- a/src/ofw/mine.py +++ b/src/ofw/mine.py @@ -1,565 +1,707 @@ -"""Deterministic trace admission and immutable Mine snapshots.""" +"""Evidence-backed failure mining over full Langfuse trajectories.""" from __future__ import annotations -import hashlib import math -import os -import tempfile -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import datetime -from enum import IntEnum, StrEnum -from pathlib import Path - -from pydantic import TypeAdapter +from enum import StrEnum +from typing import Protocol from ofw.contracts import HarnessRevision, HarnessRevisionId, Sha256Digest -from ofw.harness import Harness -from ofw.observability.langfuse.contracts import CollectionError, TraceWindow +from ofw.observability.langfuse.contracts import CollectionError from ofw.observability.langfuse.domain import ( AttributionLevel, CollectionResult, ObservationContent, - ObservationContentReference, + ObservationContentField, + ObservationContentHit, + ObservationContentMatch, + ObservationContentQuery, ObservationId, - ObservationLevel, ObservationRecord, - ObservationType, - ScoreDataType, - ScoreId, - ScoreRecord, - ScoreSource, - ScoreSubject, - ScoreSubjectKind, - TraceGap, TraceId, TraceRecord, ) from ofw.observability.langfuse.store import CollectionStore -class TracePartition(StrEnum): - VERIFIED_GOOD = "verified_good" - VERIFIED_FAILURE = "verified_failure" +class FailureSourceKind(StrEnum): + HUMAN_FEEDBACK = "human_feedback" + USER_CORRECTION = "user_correction" + TRUSTED_SCORE = "trusted_score" + DOWNSTREAM_FAILURE = "downstream_failure" + INCIDENT = "incident" + ROLLBACK = "rollback" + REOPENED_WORK = "reopened_work" + ENVIRONMENT_MISMATCH = "environment_mismatch" + AGENT_ERROR = "agent_error" + + +class EnvironmentSourceKind(StrEnum): + RECORDED_STATE = "recorded_state" + AUDIT_LOG = "audit_log" + PRODUCTION_API = "production_api" + DETERMINISTIC_CHECK = "deterministic_check" + + +class EvidenceKind(StrEnum): + TRAJECTORY = "trajectory" + ENVIRONMENT = "environment" + PRODUCTION_SIGNAL = "production_signal" + + +class CompletionStatus(StrEnum): + COMPLETED = "completed" + NOT_COMPLETED = "not_completed" + UNKNOWN = "unknown" + + +class MiningVerdict(StrEnum): + CONFIRMED_FAILURE = "confirmed_failure" + NO_FAILURE = "no_failure" AMBIGUOUS = "ambiguous" INVALID = "invalid" -class MineSchemaVersion(IntEnum): - V1 = 1 +class MiningInvalidReason(StrEnum): + REVISION_MISMATCH = "revision_mismatch" + TRACE_NOT_FOUND = "trace_not_found" + CORRUPT_TRACE = "corrupt_trace" + JUDGE_OUTPUT = "judge_output" -class TraceQualityThreshold(StrEnum): - COMPLETE = "complete" - DEGRADED = "degraded" +class ToolStatus(StrEnum): + OK = "ok" + NOT_FOUND = "not_found" + UNAVAILABLE = "unavailable" + BLOCKED = "blocked" + ERROR = "error" -class AdmissionReason(StrEnum): - VERIFIED_PASS = "verified_pass" # nosec B105 - VERIFIED_FAIL = "verified_fail" - MISSING_EVIDENCE = "missing_evidence" - CONFLICTING_EVIDENCE = "conflicting_evidence" - REVISION_ATTRIBUTION = "revision_attribution" - TRACE_QUALITY = "trace_quality" - EXCLUDED_TRACE = "excluded_trace" +class ToolAction(StrEnum): + SEARCH_TRAJECTORY = "search_trajectory" + READ_TRAJECTORY = "read_trajectory" + VERIFY_ENVIRONMENT = "verify_environment" + RETURN_VERDICT = "return_verdict" -class MineErrorCode(StrEnum): - STALE_HARNESS = "stale_harness" - REVISION_MISMATCH = "revision_mismatch" - INVALID_POLICY = "invalid_policy" - ARTIFACT_WRITE_FAILED = "artifact_write_failed" - CONTENT_INVALID = "content_invalid" +@dataclass(frozen=True, slots=True) +class FailureSourceId: + value: str + def __post_init__(self) -> None: + _require_identifier(self.value) -class MineError(Exception): - __slots__ = ("code", "subject") - def __init__(self, code: MineErrorCode, subject: str) -> None: - self.code = code - self.subject = subject - super().__init__(f"{code.value}: {subject}") +@dataclass(frozen=True, slots=True) +class EnvironmentSourceId: + value: str + + def __post_init__(self) -> None: + _require_identifier(self.value) @dataclass(frozen=True, slots=True) -class ScoreName: +class EnvironmentCheckId: value: str def __post_init__(self) -> None: - if not self.value or "\0" in self.value: - raise MineError(MineErrorCode.INVALID_POLICY, "empty score name") + _require_identifier(self.value) @dataclass(frozen=True, slots=True) -class TraceTag: +class EvidenceRecordId: value: str def __post_init__(self) -> None: - if not self.value or "\0" in self.value: - raise MineError(MineErrorCode.INVALID_POLICY, "empty trace tag") + _require_identifier(self.value) @dataclass(frozen=True, slots=True) -class MiningPolicy: - critical_scores: tuple[ScoreName, ...] - trusted_sources: tuple[ScoreSource, ...] - quality: TraceQualityThreshold - numeric_pass_at: float = 0.5 - excluded_tags: tuple[TraceTag, ...] = (TraceTag("ofw-internal"),) +class Confidence: + value: float def __post_init__(self) -> None: - if ( - not self.critical_scores - or not self.trusted_sources - or len(set(self.critical_scores)) != len(self.critical_scores) - or len(set(self.trusted_sources)) != len(self.trusted_sources) - or not math.isfinite(self.numeric_pass_at) - ): - raise MineError(MineErrorCode.INVALID_POLICY, "evidence policy is required") + if not math.isfinite(self.value) or not 0.0 <= self.value <= 1.0: + raise ValueError("confidence must be finite and between zero and one") - @property - def digest(self) -> Sha256Digest: - return _digest_text( - "\0".join( - ( - *(score.value for score in self.critical_scores), - *(source.value for source in self.trusted_sources), - self.quality.value, - str(self.numeric_pass_at), - *(tag.value for tag in self.excluded_tags), - ) - ) - ) + +@dataclass(frozen=True, slots=True) +class EvidenceReference: + kind: EvidenceKind + record_id: EvidenceRecordId + digest: Sha256Digest @dataclass(frozen=True, slots=True) -class MineRunId: - value: str +class FailureSource: + id: FailureSourceId + kind: FailureSourceKind + trace_id: TraceId + observed_at: datetime + summary: str + evidence: tuple[EvidenceReference, ...] - def __str__(self) -> str: - return self.value + def __post_init__(self) -> None: + _require_text(self.summary, "failure source summary") + if not self.evidence: + raise ValueError("failure source requires evidence") @dataclass(frozen=True, slots=True) -class TraceSnapshot: - schema_version: MineSchemaVersion - revision_id: HarnessRevisionId - collection_digest: Sha256Digest - trace: SnapshotTrace - observations: tuple[SnapshotObservation, ...] - scores: tuple[SnapshotScore, ...] +class EnvironmentCheck: + id: EnvironmentCheckId + required_outcome: str + + def __post_init__(self) -> None: + _require_text(self.required_outcome, "required outcome") + + +@dataclass(frozen=True, slots=True) +class EnvironmentSource: + id: EnvironmentSourceId + kind: EnvironmentSourceKind + summary: str + checks: tuple[EnvironmentCheck, ...] + + def __post_init__(self) -> None: + _require_text(self.summary, "environment source summary") + if not self.checks or len({check.id for check in self.checks}) != len(self.checks): + raise ValueError("environment source requires unique checks") @dataclass(frozen=True, slots=True) -class SnapshotTrace: - id: TraceId +class MiningNomination: + trace_id: TraceId + user_job: str + sources: tuple[FailureSource, ...] + environment_sources: tuple[EnvironmentSource, ...] + + def __post_init__(self) -> None: + _require_text(self.user_job, "user job") + if not self.sources: + raise ValueError("mining nomination requires a failure source") + if any(source.trace_id != self.trace_id for source in self.sources): + raise ValueError("failure source trace does not match nomination") + if len({source.id for source in self.sources}) != len(self.sources): + raise ValueError("failure source ids must be unique") + if len({source.id for source in self.environment_sources}) != len( + self.environment_sources + ): + raise ValueError("environment source ids must be unique") + + +@dataclass(frozen=True, slots=True) +class TraceMiningCase: + revision_id: HarnessRevisionId + trace_id: TraceId + trace_digest: Sha256Digest observation_ids: tuple[ObservationId, ...] - root_observation_ids: tuple[ObservationId, ...] - evidence_score_ids: tuple[ScoreId, ...] - attribution: AttributionLevel - gaps: tuple[TraceGap, ...] - digest: Sha256Digest + user_job: str + sources: tuple[FailureSource, ...] + environment_sources: tuple[EnvironmentSource, ...] @dataclass(frozen=True, slots=True) -class SnapshotObservation: - id: ObservationId - trace_id: TraceId | None - start_time: datetime - end_time: datetime | None - parent_observation_id: ObservationId | None - type: ObservationType - is_root: bool | None - name: str | None - level: ObservationLevel | None - status_message: str | None - digest: Sha256Digest - input_content: SnapshotContentReference | None = None - output_content: SnapshotContentReference | None = None +class EnvironmentCheckRequest: + source_id: EnvironmentSourceId + check_id: EnvironmentCheckId @dataclass(frozen=True, slots=True) -class SnapshotContentReference: - content: ObservationContentReference - path: Path +class EnvironmentVerification: + status: CompletionStatus + observed_state: str | None + evidence: tuple[EvidenceReference, ...] + + def __post_init__(self) -> None: + if self.status is CompletionStatus.UNKNOWN: + return + if self.observed_state is None or not self.observed_state.strip() or not self.evidence: + raise ValueError("known environment state requires observation and evidence") + if any(item.kind is not EvidenceKind.ENVIRONMENT for item in self.evidence): + raise ValueError("environment verification requires environment evidence") @dataclass(frozen=True, slots=True) -class SnapshotScore: - id: ScoreId - name: str - value: bool | float | str - data_type: ScoreDataType - source: ScoreSource - timestamp: datetime - subject: ScoreSubject | None - digest: Sha256Digest +class CompletionCheck: + check_id: EnvironmentCheckId + required_outcome: str + agent_claim: str | None + observed_state: str | None + status: CompletionStatus + evidence: tuple[EvidenceReference, ...] @dataclass(frozen=True, slots=True) -class TraceAdmission: +class FailureMiningResult: + revision_id: HarnessRevisionId trace_id: TraceId - partition: TracePartition - reason: AdmissionReason - evidence_score_ids: tuple[ScoreId, ...] - snapshot_digest: Sha256Digest | None - snapshot_path: Path | None + trace_digest: Sha256Digest | None + verdict: MiningVerdict + user_job: str + source_ids: tuple[FailureSourceId, ...] + completion_checks: tuple[CompletionCheck, ...] + trajectory_evidence: tuple[EvidenceReference, ...] + environment_evidence: tuple[EvidenceReference, ...] + confidence: Confidence + unresolved_questions: tuple[str, ...] + invalid_reason: MiningInvalidReason | None + + def __post_init__(self) -> None: + if self.verdict is MiningVerdict.INVALID: + if self.invalid_reason is None: + raise ValueError("invalid result requires a reason") + return + if self.invalid_reason is not None or self.trace_digest is None: + raise ValueError("valid result cannot carry an invalid reason") + if not self.completion_checks: + raise ValueError("mining result requires completion checks") + if self.verdict is MiningVerdict.CONFIRMED_FAILURE: + if ( + not any( + check.status is CompletionStatus.NOT_COMPLETED + for check in self.completion_checks + ) + or not self.trajectory_evidence + or not self.environment_evidence + ): + raise ValueError( + "confirmed failure requires failed completion, trajectory, " + "and environment evidence" + ) + elif self.verdict is MiningVerdict.NO_FAILURE: + if ( + any( + check.status is not CompletionStatus.COMPLETED + for check in self.completion_checks + ) + or not self.trajectory_evidence + or not self.environment_evidence + ): + raise ValueError( + "no-failure verdict requires completed checks and supporting evidence" + ) + elif not self.unresolved_questions and not any( + check.status is CompletionStatus.UNKNOWN for check in self.completion_checks + ): + raise ValueError("ambiguous verdict requires an unresolved question") @dataclass(frozen=True, slots=True) -class MineResult: - schema_version: MineSchemaVersion - id: MineRunId +class FailureMiningRun: revision_id: HarnessRevisionId - window: TraceWindow collection_digest: Sha256Digest - policy_digest: Sha256Digest - admissions: tuple[TraceAdmission, ...] - root: Path + results: tuple[FailureMiningResult, ...] - @property - def manifest_path(self) -> Path: - return self.root / ".ofw" / "mine" / str(self.id) / "manifest.json" - @property - def verified_good_count(self) -> int: - return self._count(TracePartition.VERIFIED_GOOD) +@dataclass(frozen=True, slots=True) +class TrajectorySearchRequest: + text: str + field: ObservationContentField + limit: int - @property - def verified_failure_count(self) -> int: - return self._count(TracePartition.VERIFIED_FAILURE) + def __post_init__(self) -> None: + _require_text(self.text, "trajectory search text") + if not 1 <= self.limit <= 100: + raise ValueError("trajectory search limit must be between 1 and 100") - @property - def ambiguous_count(self) -> int: - return self._count(TracePartition.AMBIGUOUS) - @property - def invalid_count(self) -> int: - return self._count(TracePartition.INVALID) +@dataclass(frozen=True, slots=True) +class TrajectorySearchResult: + status: ToolStatus + summary: str + hits: tuple[ObservationContentHit, ...] + next_actions: tuple[ToolAction, ...] + artifacts: tuple[EvidenceReference, ...] - def to_json(self) -> str: - return _MINE_RESULT_ADAPTER.dump_json(self).decode() - def _count(self, partition: TracePartition) -> int: - return sum(admission.partition is partition for admission in self.admissions) +@dataclass(frozen=True, slots=True) +class TrajectoryPageRequest: + cursor: ObservationId | None + limit: int + def __post_init__(self) -> None: + if not 1 <= self.limit <= 100: + raise ValueError("trajectory page limit must be between 1 and 100") -_SNAPSHOT_ADAPTER: TypeAdapter[TraceSnapshot] = TypeAdapter(TraceSnapshot) -_MINE_RESULT_ADAPTER: TypeAdapter[MineResult] = TypeAdapter(MineResult) + +@dataclass(frozen=True, slots=True) +class TrajectoryObservation: + record: ObservationRecord + input_content: ObservationContent | None + output_content: ObservationContent | None @dataclass(frozen=True, slots=True) -class Mine: - source: Harness | HarnessRevision +class TrajectoryPageResult: + status: ToolStatus + summary: str + observations: tuple[TrajectoryObservation, ...] + next_cursor: ObservationId | None + next_actions: tuple[ToolAction, ...] + artifacts: tuple[EvidenceReference, ...] + + +@dataclass(frozen=True, slots=True) +class EnvironmentCheckResult: + status: ToolStatus + summary: str + verification: EnvironmentVerification | None + next_actions: tuple[ToolAction, ...] + artifacts: tuple[EvidenceReference, ...] + + +class EnvironmentVerifier(Protocol): + def verify( + self, + request: EnvironmentCheckRequest, + source: EnvironmentSource, + check: EnvironmentCheck, + ) -> EnvironmentVerification: ... + + +class HermesJudge(Protocol): + def investigate( + self, + case: TraceMiningCase, + tools: MiningTools, + ) -> FailureMiningResult: ... + + +@dataclass(slots=True) +class MiningTools: + case: TraceMiningCase collection: CollectionResult - policy: MiningPolicy - - def run(self) -> MineResult: - revision = _resolve_revision(self.source) - if self.collection.revision_id != revision.id: - raise MineError(MineErrorCode.REVISION_MISMATCH, str(revision.id)) - run_id = MineRunId( - "mine_" - + hashlib.sha256( - "\0".join( - ( - str(revision.id), - str(self.collection.snapshot_digest), - str(self.policy.digest), - str(int(MineSchemaVersion.V1)), - self.collection.window.start.isoformat(), - self.collection.window.end.isoformat(), - ) - ).encode() - ).hexdigest() - ) + environment: EnvironmentVerifier + _issued_evidence: list[EvidenceReference] = field( + default_factory=list, + init=False, + repr=False, + ) + _read_trajectory_evidence: list[EvidenceReference] = field( + default_factory=list, + init=False, + repr=False, + ) + + @property + def issued_evidence(self) -> tuple[EvidenceReference, ...]: + return tuple(self._issued_evidence) + + @property + def read_trajectory_evidence(self) -> tuple[EvidenceReference, ...]: + return tuple(self._read_trajectory_evidence) + + def search_trajectory(self, request: TrajectorySearchRequest) -> TrajectorySearchResult: store = CollectionStore(self.collection.store_path) try: - observations = store.observations(self.collection.observation_sync_id) - scores = store.scores(self.collection.score_sync_id) - admissions = tuple( - self._admit(revision, run_id, trace, observations, scores, store) - for trace in sorted(self.collection.traces, key=_trace_sort_key) + hits = store.search_content( + self.collection.observation_sync_id, + ObservationContentQuery( + text=request.text, + match=ObservationContentMatch.TOKEN_PHRASE, + field=request.field, + trace_id=self.case.trace_id, + limit=request.limit, + maximum_excerpt_characters=1000, + ), + ) + except CollectionError: + return TrajectorySearchResult( + ToolStatus.ERROR, + "Trajectory search failed.", + (), + (ToolAction.RETURN_VERDICT,), + (), ) finally: store.close() - result = MineResult( - MineSchemaVersion.V1, - run_id, - revision.id, - self.collection.window, - self.collection.snapshot_digest, - self.policy.digest, - admissions, - revision.root, + artifacts = tuple( + EvidenceReference( + EvidenceKind.TRAJECTORY, + EvidenceRecordId(hit.observation_id.value), + hit.reference.digest, + ) + for hit in hits ) - _write_artifact(result.manifest_path, f"{result.to_json()}\n".encode()) - return result - - def _admit( - self, - revision: HarnessRevision, - run_id: MineRunId, - trace: TraceRecord, - observations: tuple[ObservationRecord, ...], - scores: tuple[ScoreRecord, ...], - store: CollectionStore, - ) -> TraceAdmission: - # ponytail: linear scans are simplest for local v0; index when profiling shows pressure. - trace_observations = tuple( - observation for observation in observations if observation.id in trace.observation_ids + self._issued_evidence.extend(artifacts) + return TrajectorySearchResult( + ToolStatus.OK if hits else ToolStatus.NOT_FOUND, + f"Found {len(hits)} matching trajectory segments.", + hits, + (ToolAction.READ_TRAJECTORY,), + artifacts, ) - trace_scores = tuple( - score - for score in scores - if score.id in trace.score_ids and _score_belongs(score, trace, trace_observations) + + def read_trajectory(self, request: TrajectoryPageRequest) -> TrajectoryPageResult: + store = CollectionStore(self.collection.store_path) + try: + # ponytail: collection-wide scan is simplest for local v0; add a paged + # trace SQL query if production profiles show this O(pages * records) path. + observations = tuple( + observation + for observation in store.observations(self.collection.observation_sync_id) + if observation.trace_id == self.case.trace_id + ) + start = _page_start(observations, request.cursor) + if start is None: + return TrajectoryPageResult( + ToolStatus.BLOCKED, + "Cursor is outside the nominated trace.", + (), + None, + (ToolAction.RETURN_VERDICT,), + (), + ) + selected = observations[start : start + request.limit] + views = tuple(_read_observation(store, self.collection, item) for item in selected) + except CollectionError: + return TrajectoryPageResult( + ToolStatus.ERROR, + "Trajectory content could not be read.", + (), + None, + (ToolAction.RETURN_VERDICT,), + (), + ) + finally: + store.close() + has_more = start + len(selected) < len(observations) + next_cursor = observations[start + len(selected)].id if has_more else None + artifacts = tuple( + EvidenceReference( + EvidenceKind.TRAJECTORY, + EvidenceRecordId(observation.record.id.value), + observation.record.digest, + ) + for observation in views ) - partition, reason, evidence = self._classify(trace, trace_observations, trace_scores) - if partition is TracePartition.INVALID: - return TraceAdmission(trace.id, partition, reason, evidence, None, None) - snapshot_scores = tuple(score for score in trace_scores if score.id in evidence) - snapshot = TraceSnapshot( - MineSchemaVersion.V1, - revision.id, - self.collection.snapshot_digest, - _snapshot_trace(trace, evidence), - tuple( - self._snapshot_observation(revision, run_id, store, observation) - for observation in trace_observations + self._issued_evidence.extend(artifacts) + self._read_trajectory_evidence.extend(artifacts) + return TrajectoryPageResult( + ToolStatus.OK, + f"Read {len(views)} ordered trajectory observations.", + views, + next_cursor, + ( + (ToolAction.READ_TRAJECTORY,) + if next_cursor is not None + else (ToolAction.VERIFY_ENVIRONMENT, ToolAction.RETURN_VERDICT) ), - tuple(_snapshot_score(score) for score in snapshot_scores), + artifacts, ) - payload = _SNAPSHOT_ADAPTER.dump_json(snapshot) - digest = _digest_bytes(payload) - path = revision.root / ".ofw" / "mine" / str(run_id) / "traces" / f"{digest.value[7:]}.json" - _write_artifact(path, payload + b"\n") - return TraceAdmission(trace.id, partition, reason, evidence, digest, path) - def _snapshot_observation( - self, - revision: HarnessRevision, - run_id: MineRunId, - store: CollectionStore, - observation: ObservationRecord, - ) -> SnapshotObservation: - return _snapshot_observation( - observation, - self._snapshot_content(revision, run_id, store, observation.input_content), - self._snapshot_content(revision, run_id, store, observation.output_content), + def verify_environment(self, request: EnvironmentCheckRequest) -> EnvironmentCheckResult: + selected = _find_environment_check(self.case, request) + if selected is None: + return EnvironmentCheckResult( + ToolStatus.BLOCKED, + "Environment check is not declared for this mining case.", + None, + (ToolAction.RETURN_VERDICT,), + (), + ) + source, check = selected + verification = self.environment.verify(request, source, check) + self._issued_evidence.extend(verification.evidence) + return EnvironmentCheckResult( + ( + ToolStatus.UNAVAILABLE + if verification.status is CompletionStatus.UNKNOWN + else ToolStatus.OK + ), + "Environment state is unavailable." + if verification.status is CompletionStatus.UNKNOWN + else "Environment state verified.", + verification, + (ToolAction.RETURN_VERDICT,), + verification.evidence, ) - def _snapshot_content( - self, - revision: HarnessRevision, - run_id: MineRunId, - store: CollectionStore, - reference: ObservationContentReference | None, - ) -> SnapshotContentReference | None: - if reference is None: - return None - try: - content = store.read_content(self.collection.observation_sync_id, reference) - except CollectionError as error: - raise MineError(MineErrorCode.CONTENT_INVALID, str(reference.digest)) from error - path = ( - revision.root - / ".ofw" - / "mine" - / str(run_id) - / "content" - / f"{reference.digest.value[7:]}.txt" - ) - _write_artifact(path, content.text.encode()) - return SnapshotContentReference(reference, path) - def _classify( - self, - trace: TraceRecord, - observations: tuple[ObservationRecord, ...], - scores: tuple[ScoreRecord, ...], - ) -> tuple[TracePartition, AdmissionReason, tuple[ScoreId, ...]]: - if trace.attribution is not AttributionLevel.EXACT: - return TracePartition.INVALID, AdmissionReason.REVISION_ATTRIBUTION, () - if not observations or not all( - any(observation.id == observation_id for observation in observations) - for observation_id in trace.observation_ids - ): - return TracePartition.INVALID, AdmissionReason.TRACE_QUALITY, () - if self.policy.quality is TraceQualityThreshold.COMPLETE and trace.gaps: - return TracePartition.INVALID, AdmissionReason.TRACE_QUALITY, () - if any( - tag.value in observation.tags - for tag in self.policy.excluded_tags - for observation in observations - ): - return TracePartition.INVALID, AdmissionReason.EXCLUDED_TRACE, () - evidence = tuple( - score - for score in scores - if score.source in self.policy.trusted_sources - and any(score.name == name.value for name in self.policy.critical_scores) +@dataclass(frozen=True, slots=True) +class Mine: + revision: HarnessRevision + collection: CollectionResult + nominations: tuple[MiningNomination, ...] + judge: HermesJudge + environment: EnvironmentVerifier + + def __post_init__(self) -> None: + if not self.nominations: + raise ValueError("mine requires at least one nomination") + + def run(self) -> FailureMiningRun: + results = tuple(self._mine(nomination) for nomination in self.nominations) + return FailureMiningRun(self.revision.id, self.collection.snapshot_digest, results) + + def _mine(self, nomination: MiningNomination) -> FailureMiningResult: + trace = next( + (item for item in self.collection.traces if item.id == nomination.trace_id), + None, ) - verdicts: list[bool] = [] - missing = False - conflicting = False - for name in self.policy.critical_scores: - matching = tuple(score for score in evidence if score.name == name.value) - if not matching: - missing = True - continue - resolved = tuple( - _score_passes(score, self.policy.numeric_pass_at) for score in matching + if self.collection.revision_id != self.revision.id: + return _invalid( + self.revision.id, + nomination, + trace, + MiningInvalidReason.REVISION_MISMATCH, ) - if any(verdict is None for verdict in resolved) or len(set(resolved)) != 1: - conflicting = True - continue - verdict = resolved[0] - if verdict is not None: - verdicts.append(verdict) - evidence_ids = tuple(score.id for score in evidence) - if conflicting: - return ( - TracePartition.AMBIGUOUS, - AdmissionReason.CONFLICTING_EVIDENCE, - evidence_ids, - ) - if any(not verdict for verdict in verdicts): - return TracePartition.VERIFIED_FAILURE, AdmissionReason.VERIFIED_FAIL, evidence_ids - if missing: - return TracePartition.AMBIGUOUS, AdmissionReason.MISSING_EVIDENCE, evidence_ids - return TracePartition.VERIFIED_GOOD, AdmissionReason.VERIFIED_PASS, evidence_ids - - -def _score_passes(score: ScoreRecord, numeric_pass_at: float) -> bool | None: - if score.data_type is ScoreDataType.BOOLEAN and isinstance(score.value, bool): - return score.value - if score.data_type is ScoreDataType.NUMERIC and isinstance(score.value, float): - return score.value >= numeric_pass_at - return None + if trace is None: + return _invalid(self.revision.id, nomination, None, MiningInvalidReason.TRACE_NOT_FOUND) + if not _trace_is_complete(self.collection, trace): + return _invalid(self.revision.id, nomination, trace, MiningInvalidReason.CORRUPT_TRACE) + case = TraceMiningCase( + self.revision.id, + trace.id, + trace.digest, + trace.observation_ids, + nomination.user_job, + nomination.sources, + nomination.environment_sources, + ) + tools = MiningTools(case, self.collection, self.environment) + result = self.judge.investigate(case, tools) + if not _judge_result_matches( + case, + result, + tools.issued_evidence, + tools.read_trajectory_evidence, + ): + return _invalid(self.revision.id, nomination, trace, MiningInvalidReason.JUDGE_OUTPUT) + return result -def _score_belongs( - score: ScoreRecord, - trace: TraceRecord, - observations: tuple[ObservationRecord, ...], -) -> bool: - subject = score.subject - if subject is None: +def _trace_is_complete(collection: CollectionResult, trace: TraceRecord) -> bool: + if trace.attribution is not AttributionLevel.EXACT or trace.gaps: return False - if subject.kind is ScoreSubjectKind.TRACE: - return subject.id == trace.id.value - if subject.kind is ScoreSubjectKind.OBSERVATION: - return any(observation.id.value == subject.id for observation in observations) and ( - subject.trace_id is None or subject.trace_id == trace.id + store = CollectionStore(collection.store_path) + try: + observations = tuple( + observation + for observation in store.observations(collection.observation_sync_id) + if observation.trace_id == trace.id ) - if subject.kind is ScoreSubjectKind.SESSION: - return trace.session_id is not None and subject.id == trace.session_id - return False - - -def _snapshot_trace(trace: TraceRecord, evidence: tuple[ScoreId, ...]) -> SnapshotTrace: - return SnapshotTrace( - trace.id, - trace.observation_ids, - trace.root_observation_ids, - evidence, - trace.attribution, - trace.gaps, - trace.digest, + for observation in observations: + _read_observation(store, collection, observation) + except CollectionError: + return False + finally: + store.close() + return ( + bool(observations) + and tuple(observation.id for observation in observations) == trace.observation_ids ) -def _snapshot_observation( - observation: ObservationRecord, - input_content: SnapshotContentReference | None, - output_content: SnapshotContentReference | None, -) -> SnapshotObservation: - return SnapshotObservation( - observation.id, - observation.trace_id, - observation.start_time, - observation.end_time, - observation.parent_observation_id, - observation.type, - observation.is_root, - observation.name, - observation.level, - observation.status_message, - observation.digest, - input_content, - output_content, +def _judge_result_matches( + case: TraceMiningCase, + result: FailureMiningResult, + issued_evidence: tuple[EvidenceReference, ...], + read_trajectory_evidence: tuple[EvidenceReference, ...], +) -> bool: + observation_ids = {evidence.record_id.value for evidence in result.trajectory_evidence} + read_observation_ids = { + ObservationId(evidence.record_id.value) for evidence in read_trajectory_evidence + } + allowed_checks = { + check.id + for source in case.environment_sources + for check in source.checks + } + completion_evidence = tuple( + evidence for check in result.completion_checks for evidence in check.evidence ) - - -def _snapshot_score(score: ScoreRecord) -> SnapshotScore: - return SnapshotScore( - score.id, - score.name, - score.value, - score.data_type, - score.source, - score.timestamp, - score.subject, - score.digest, + return ( + result.revision_id == case.revision_id + and result.trace_id == case.trace_id + and result.trace_digest == case.trace_digest + and result.source_ids == tuple(source.id for source in case.sources) + and read_observation_ids == set(case.observation_ids) + and all(item.kind is EvidenceKind.TRAJECTORY for item in result.trajectory_evidence) + and all(item in result.trajectory_evidence for item in read_trajectory_evidence) + and observation_ids.issubset({item.value for item in case.observation_ids}) + and all(item.kind is EvidenceKind.ENVIRONMENT for item in result.environment_evidence) + and all(item.kind is EvidenceKind.ENVIRONMENT for item in completion_evidence) + and all(item in issued_evidence for item in result.trajectory_evidence) + and all(item in issued_evidence for item in result.environment_evidence) + and all(item in issued_evidence for item in completion_evidence) + and all(check.check_id in allowed_checks for check in result.completion_checks) ) -def _trace_sort_key(trace: TraceRecord) -> str: - return trace.id.value - - -def _resolve_revision(source: Harness | HarnessRevision) -> HarnessRevision: - if isinstance(source, HarnessRevision): - return source - revision = source.current_revision - if revision is None: - raise MineError(MineErrorCode.STALE_HARNESS, source.name) - return revision +def _find_environment_check( + case: TraceMiningCase, + request: EnvironmentCheckRequest, +) -> tuple[EnvironmentSource, EnvironmentCheck] | None: + for source in case.environment_sources: + if source.id != request.source_id: + continue + for check in source.checks: + if check.id == request.check_id: + return source, check + return None -def _write_artifact(path: Path, payload: bytes) -> None: - try: - path.parent.mkdir(parents=True, exist_ok=True) - descriptor, temporary_name = tempfile.mkstemp( - dir=path.parent, - prefix=f".{path.stem}-", - suffix=".tmp", - ) - temporary = Path(temporary_name) - try: - with os.fdopen(descriptor, "wb") as stream: - stream.write(payload) - stream.flush() - os.fsync(stream.fileno()) - temporary.chmod(0o600) - temporary.replace(path) - finally: - temporary.unlink(missing_ok=True) - except OSError as error: - raise MineError(MineErrorCode.ARTIFACT_WRITE_FAILED, str(path)) from error +def _page_start( + observations: tuple[ObservationRecord, ...], + cursor: ObservationId | None, +) -> int | None: + if cursor is None: + return 0 + for index, observation in enumerate(observations): + if observation.id == cursor: + return index + return None -def _digest_text(value: str) -> Sha256Digest: - return _digest_bytes(value.encode()) +def _read_observation( + store: CollectionStore, + collection: CollectionResult, + observation: ObservationRecord, +) -> TrajectoryObservation: + input_content = ( + None + if observation.input_content is None + else store.read_content(collection.observation_sync_id, observation.input_content) + ) + output_content = ( + None + if observation.output_content is None + else store.read_content(collection.observation_sync_id, observation.output_content) + ) + return TrajectoryObservation(observation, input_content, output_content) + + +def _invalid( + revision_id: HarnessRevisionId, + nomination: MiningNomination, + trace: TraceRecord | None, + reason: MiningInvalidReason, +) -> FailureMiningResult: + return FailureMiningResult( + revision_id=revision_id, + trace_id=nomination.trace_id, + trace_digest=None if trace is None else trace.digest, + verdict=MiningVerdict.INVALID, + user_job=nomination.user_job, + source_ids=tuple(source.id for source in nomination.sources), + completion_checks=(), + trajectory_evidence=(), + environment_evidence=(), + confidence=Confidence(1.0), + unresolved_questions=(), + invalid_reason=reason, + ) -def _digest_bytes(value: bytes) -> Sha256Digest: - return Sha256Digest(f"sha256:{hashlib.sha256(value).hexdigest()}") +def _require_identifier(value: str) -> None: + if not value or not value.isascii() or any(character.isspace() for character in value): + raise ValueError("identifier must be non-empty ASCII without whitespace") -def read_snapshot_content( - result: MineResult, - reference: SnapshotContentReference, -) -> ObservationContent: - try: - allowed = ( - result.root / ".ofw" / "mine" / str(result.id) / "content" - ).resolve(strict=True) - resolved = reference.path.resolve(strict=True) - resolved.relative_to(allowed) - expected_name = f"{reference.content.digest.value[7:]}.txt" - if resolved.name != expected_name: - raise ValueError("content path does not match digest") - return ObservationContent(reference.content, resolved.read_text(encoding="utf-8")) - except (OSError, ValueError) as error: - raise MineError(MineErrorCode.CONTENT_INVALID, str(reference.path)) from error +def _require_text(value: str, name: str) -> None: + if not value.strip() or "\0" in value: + raise ValueError(f"{name} must be non-empty text") diff --git a/tests/test_mine.py b/tests/test_mine.py index f6f5c4a..2f9774b 100644 --- a/tests/test_mine.py +++ b/tests/test_mine.py @@ -1,33 +1,50 @@ -"""Deterministic Mine admission and immutable snapshot behavior.""" +"""Failure mining over complete Langfuse trajectories and verified final state.""" from __future__ import annotations -import subprocess -from dataclasses import replace +from dataclasses import dataclass, replace from datetime import UTC, datetime, timedelta from pathlib import Path import pytest -from pydantic import TypeAdapter -from ofw import ( - Harness, - Mine, - MineError, - MineErrorCode, - MiningPolicy, - ScoreName, - TracePartition, - TraceQualityThreshold, - read_snapshot_content, +from ofw.contracts import ( + GitCommit, + HarnessRevision, + HarnessRevisionId, + HarnessSchemaVersion, + RepositorySnapshot, + Sha256Digest, ) -from ofw.contracts import HarnessRevision, Sha256Digest -from ofw.mine import TraceSnapshot -from ofw.observability.langfuse.contracts import ( - LangfuseConnectionId, - ObservationContentPolicy, - TraceWindow, +from ofw.mine import ( + CompletionCheck, + CompletionStatus, + Confidence, + EnvironmentCheck, + EnvironmentCheckId, + EnvironmentCheckRequest, + EnvironmentSource, + EnvironmentSourceId, + EnvironmentSourceKind, + EnvironmentVerification, + EvidenceKind, + EvidenceRecordId, + EvidenceReference, + FailureMiningResult, + FailureSource, + FailureSourceId, + FailureSourceKind, + Mine, + MiningInvalidReason, + MiningNomination, + MiningTools, + MiningVerdict, + ToolStatus, + TraceMiningCase, + TrajectoryPageRequest, + TrajectorySearchRequest, ) +from ofw.observability.langfuse.contracts import LangfuseConnectionId, TraceWindow from ofw.observability.langfuse.domain import ( AttributionLevel, CollectionCapabilityReason, @@ -35,395 +52,505 @@ CollectionSyncId, JsonDocument, ObservationContent, + ObservationContentField, ObservationContentReference, ObservationId, ObservationPage, ObservationRecord, ObservationType, ProjectId, - ScoreDataType, - ScoreId, ScorePage, - ScoreRecord, - ScoreSource, - ScoreSubject, - ScoreSubjectKind, TraceGap, TraceId, TraceRecord, ) from ofw.observability.langfuse.store import CollectionStore - -def _run_git(root: Path, *arguments: str) -> None: - subprocess.run( - ("git", "-C", str(root), *arguments), - check=True, - capture_output=True, - text=True, - ) - - -def _harness(tmp_path: Path) -> Harness: - root = tmp_path / "mine-agent" - root.mkdir() - (root / "prompt.md").write_text("Be accurate.\n", encoding="utf-8") - (root / "memory.md").write_text("Known facts.\n", encoding="utf-8") - _run_git(root, "init", "-q") - _run_git(root, "config", "user.email", "fixture@example.test") - _run_git(root, "config", "user.name", "FixtureCo") - _run_git(root, "add", ".") - _run_git(root, "commit", "-qm", "fixture baseline") - harness = Harness("mine-agent", root=root) - harness.connect_prompt(Path("prompt.md")) - return harness +NOW = datetime(2026, 8, 25, tzinfo=UTC) +TRACE_ID = TraceId("trace-1") +CHECK_ID = EnvironmentCheckId("ticket-closed") +SOURCE_ID = EnvironmentSourceId("itsm-production") -def _observation( - trace: str, - revision: HarnessRevision, - *, - tags: tuple[str, ...] = (), -) -> ObservationRecord: - return ObservationRecord( - id=ObservationId(f"observation-{trace}"), - trace_id=TraceId(trace), - start_time=datetime(2026, 8, 22, tzinfo=UTC), - end_time=datetime(2026, 8, 22, 0, 1, tzinfo=UTC), - project_id=ProjectId("project-1"), - parent_observation_id=None, - type=ObservationType.AGENT, - is_root=True, - name="agent-run", - level=None, - version="v1", - environment="production", - user_id=None, - session_id=f"session-{trace}", - created_at=datetime(2026, 8, 22, 0, 1, tzinfo=UTC), - updated_at=datetime(2026, 8, 22, 0, 1, tzinfo=UTC), - metadata=JsonDocument(f'{{"ofw.harness.revision":"{revision.id}","secret":"token"}}'), - usage=None, - costs=None, - total_cost=None, - tags=tags, - release=None, - trace_name="agent-run", - digest=Sha256Digest(f"sha256:observation-{trace}"), - ) +def _digest(value: str) -> Sha256Digest: + return Sha256Digest(f"sha256:{value}") -def _score(trace: str, value: bool, suffix: str = "") -> ScoreRecord: - return ScoreRecord( - id=ScoreId(f"score-{trace}{suffix}"), - project_id=ProjectId("project-1"), - name="correctness", - value=value, - data_type=ScoreDataType.BOOLEAN, - source=ScoreSource.ANNOTATION, - timestamp=datetime(2026, 8, 22, 0, 2, tzinfo=UTC), - environment="production", - created_at=datetime(2026, 8, 22, 0, 2, tzinfo=UTC), - updated_at=datetime(2026, 8, 22, 0, 2, tzinfo=UTC), - comment="reviewed", - metadata=None, - subject=ScoreSubject(ScoreSubjectKind.TRACE, trace, None), - digest=Sha256Digest(f"sha256:score-{trace}{suffix}"), +def _revision(tmp_path: Path, value: str = "revision-1") -> HarnessRevision: + return HarnessRevision( + schema_version=HarnessSchemaVersion.V1, + id=HarnessRevisionId(value), + harness_name="support-agent", + root=tmp_path, + repository=RepositorySnapshot(GitCommit("abc123"), False, None), + components=(), + observability=None, + runtime=None, + canary_digest=None, ) -def _trace( - trace: str, - scores: tuple[ScoreRecord, ...], +def _observation( + revision: HarnessRevision, + index: int, + name: str, + output: str, *, - attribution: AttributionLevel = AttributionLevel.EXACT, - gaps: tuple[TraceGap, ...] = (), -) -> TraceRecord: - return TraceRecord( - id=TraceId(trace), - observation_ids=(ObservationId(f"observation-{trace}"),), - root_observation_ids=(ObservationId(f"observation-{trace}"),), - score_ids=tuple(score.id for score in scores), - session_id=f"session-{trace}", - environment="production", - release=None, - attribution=attribution, - gaps=gaps, - digest=Sha256Digest(f"sha256:trace-{trace}"), + level: str | None = None, +) -> tuple[ObservationRecord, ObservationContent]: + reference = ObservationContentReference.for_text(output) + observation_id = ObservationId(f"observation-{index}") + return ( + ObservationRecord( + id=observation_id, + trace_id=TRACE_ID, + start_time=NOW + timedelta(seconds=index), + end_time=NOW + timedelta(seconds=index, milliseconds=100), + project_id=ProjectId("project-1"), + parent_observation_id=None if index == 0 else ObservationId("observation-0"), + type=ObservationType.AGENT if index in (0, 3) else ObservationType.TOOL, + is_root=index == 0, + name=name, + level=None, + version="v1", + environment="production", + user_id="user-1", + session_id="session-1", + created_at=NOW + timedelta(seconds=index), + updated_at=NOW + timedelta(seconds=index, milliseconds=100), + metadata=JsonDocument(f'{{"revision":"{revision.id}"}}'), + usage=None, + costs=None, + total_cost=None, + tags=(), + release=str(revision.id), + trace_name="close-ticket", + raw=JsonDocument(f'{{"name":"{name}","output":"{output}"}}'), + digest=_digest(f"observation-{index}"), + status_message=level, + output_content=reference, + ), + ObservationContent(reference, output), ) def _collection( tmp_path: Path, revision: HarnessRevision, + outputs: tuple[str, ...], *, - conflict: bool = False, - foreign_good_score: bool = False, - content: bool = False, + attribution: AttributionLevel = AttributionLevel.EXACT, + gaps: tuple[TraceGap, ...] = (), ) -> CollectionResult: - good_score = _score("good", True) - if foreign_good_score: - good_score = replace( - good_score, - subject=ScoreSubject(ScoreSubjectKind.TRACE, "other-trace", None), + named = tuple( + _observation( + revision, + index, + "agent" if index in (0, len(outputs) - 1) else "update-ticket", + output, + level="failed" if "failed" in output else None, ) - failed_score = _score("failed", False) - conflicting_scores: tuple[ScoreRecord, ...] = ( - (_score("ambiguous", True, "-pass"), _score("ambiguous", False, "-fail")) - if conflict - else () - ) - scores: tuple[ScoreRecord, ...] = (good_score, failed_score, *conflicting_scores) - plain_observations: tuple[ObservationRecord, ...] = ( - _observation("good", revision), - _observation("failed", revision), - _observation("ambiguous", revision), - _observation("invalid", revision, tags=("ofw-internal",)), + for index, output in enumerate(outputs) ) - contents: tuple[ObservationContent, ...] = () - observations = plain_observations - if content: - captured: tuple[ObservationRecord, ...] = () - for observation in plain_observations: - trace_id = observation.trace_id - assert trace_id is not None - input_text = f"request {trace_id.value} [REDACTED_EMAIL]" - output_text = f"result {trace_id.value}" - input_reference = ObservationContentReference.for_text( - input_text, - truncated=False, - ) - output_reference = ObservationContentReference.for_text( - output_text, - truncated=False, - ) - captured = ( - *captured, - replace( - observation, - input_content=input_reference, - output_content=output_reference, - ), - ) - contents = ( - *contents, - ObservationContent(input_reference, input_text), - ObservationContent(output_reference, output_text), - ) - observations = captured - traces: tuple[TraceRecord, ...] = ( - _trace("good", (good_score,)), - _trace("failed", (failed_score,)), - _trace("ambiguous", conflicting_scores), - _trace("invalid", (), attribution=AttributionLevel.MISSING), - ) - observation_sync = CollectionSyncId("observations-mine") - score_sync = CollectionSyncId("scores-mine") + observations = tuple(item[0] for item in named) + contents = tuple(item[1] for item in named) + observation_sync_id = CollectionSyncId("observations-mine") + score_sync_id = CollectionSyncId("scores-mine") store_path = tmp_path / "collection.sqlite" store = CollectionStore(store_path) try: store.commit_observation_page( "connection-1", - observation_sync, + observation_sync_id, ObservationPage(observations, None, contents), ) - store.commit_score_page("connection-1", score_sync, ScorePage(scores, None)) + store.commit_score_page("connection-1", score_sync_id, ScorePage((), None)) finally: store.close() - start = datetime(2026, 8, 22, tzinfo=UTC) + trace = TraceRecord( + id=TRACE_ID, + observation_ids=tuple(observation.id for observation in observations), + root_observation_ids=(observations[0].id,), + score_ids=(), + session_id="session-1", + environment="production", + release=str(revision.id), + attribution=attribution, + gaps=gaps, + digest=_digest("trace-1"), + ) return CollectionResult( revision_id=revision.id, connection_id=LangfuseConnectionId("connection-1"), - window=TraceWindow(start, start + timedelta(hours=1)), - observation_sync_id=observation_sync, - score_sync_id=score_sync, - traces=traces, + window=TraceWindow(NOW - timedelta(minutes=1), NOW + timedelta(minutes=1)), + observation_sync_id=observation_sync_id, + score_sync_id=score_sync_id, + traces=(trace,), observation_count=len(observations), - score_count=len(scores), - gap_count=0, - snapshot_digest=Sha256Digest("sha256:collection"), - capability=CollectionCapabilityReason.READY, - store_path=store_path, - content_policy=( - ObservationContentPolicy.redacted( - maximum_bytes_per_field=4096, - secret_environment_variables=(), - ) - if content - else ObservationContentPolicy.metadata_only() + score_count=0, + gap_count=len(gaps), + snapshot_digest=_digest("collection"), + capability=( + CollectionCapabilityReason.READY + if attribution is AttributionLevel.EXACT and not gaps + else CollectionCapabilityReason.INCOMPLETE_TRACE ), + store_path=store_path, ) -def _policy() -> MiningPolicy: - return MiningPolicy( - critical_scores=(ScoreName("correctness"),), - trusted_sources=(ScoreSource.ANNOTATION,), - quality=TraceQualityThreshold.COMPLETE, - ) - +def _evidence(kind: EvidenceKind, record_id: str, digest: str) -> EvidenceReference: + return EvidenceReference(kind, EvidenceRecordId(record_id), _digest(digest)) -def test_mine_partitions_only_from_trusted_independent_evidence(tmp_path: Path) -> None: - revision = _harness(tmp_path).process() - result = Mine(revision, _collection(tmp_path, revision), _policy()).run() - partitions = tuple(admission.partition for admission in result.admissions) - assert partitions == ( - TracePartition.AMBIGUOUS, - TracePartition.VERIFIED_FAILURE, - TracePartition.VERIFIED_GOOD, - TracePartition.INVALID, +def _nomination(kind: FailureSourceKind) -> MiningNomination: + source = FailureSource( + id=FailureSourceId("signal-1"), + kind=kind, + trace_id=TRACE_ID, + observed_at=NOW, + summary="The ticket may still be open.", + evidence=(_evidence(EvidenceKind.PRODUCTION_SIGNAL, "signal-1", "signal-1"),), ) - assert result.verified_good_count == 1 - assert result.verified_failure_count == 1 - assert result.ambiguous_count == 1 - assert result.invalid_count == 1 - - -def test_conflicting_trusted_scores_remain_ambiguous(tmp_path: Path) -> None: - revision = _harness(tmp_path).process() - result = Mine( - revision, - _collection(tmp_path, revision, conflict=True), - _policy(), - ).run() - ambiguous = next( - admission for admission in result.admissions if admission.trace_id == TraceId("ambiguous") - ) - - assert ambiguous.partition is TracePartition.AMBIGUOUS - - -def test_known_critical_failure_wins_over_other_missing_evidence(tmp_path: Path) -> None: - revision = _harness(tmp_path).process() - policy = MiningPolicy( - critical_scores=(ScoreName("correctness"), ScoreName("safety")), - trusted_sources=(ScoreSource.ANNOTATION,), - quality=TraceQualityThreshold.COMPLETE, + environment = EnvironmentSource( + id=SOURCE_ID, + kind=EnvironmentSourceKind.PRODUCTION_API, + summary="Read-only ITSM production state.", + checks=(EnvironmentCheck(CHECK_ID, "The ticket is closed."),), ) - result = Mine(revision, _collection(tmp_path, revision), policy).run() - failed = next( - admission for admission in result.admissions if admission.trace_id == TraceId("failed") + return MiningNomination( + trace_id=TRACE_ID, + user_job="Close the customer ticket.", + sources=(source,), + environment_sources=(environment,), ) - assert failed.partition is TracePartition.VERIFIED_FAILURE +@dataclass(frozen=True, slots=True) +class RecordedEnvironmentVerifier: + status: CompletionStatus + observed_state: str | None + + def verify( + self, + request: EnvironmentCheckRequest, + source: EnvironmentSource, + check: EnvironmentCheck, + ) -> EnvironmentVerification: + assert request.source_id == source.id + assert request.check_id == check.id + evidence = ( + () + if self.status is CompletionStatus.UNKNOWN + else (_evidence(EvidenceKind.ENVIRONMENT, "ticket-123", "state-1"),) + ) + return EnvironmentVerification( + status=self.status, + observed_state=self.observed_state, + evidence=evidence, + ) -def test_mine_is_content_addressed_and_idempotent(tmp_path: Path) -> None: - revision = _harness(tmp_path).process() - collection = _collection(tmp_path, revision) - first = Mine(revision, collection, _policy()).run() - second = Mine(revision, collection, _policy()).run() +@dataclass(frozen=True, slots=True) +class FakeHermesJudge: + search_text: str + + def investigate( + self, + case: TraceMiningCase, + tools: MiningTools, + ) -> FailureMiningResult: + assert case == tools.case + search = tools.search_trajectory( + TrajectorySearchRequest( + text=self.search_text, + field=ObservationContentField.ANY, + limit=10, + ) + ) + assert search.status is ToolStatus.OK + focused = tools.read_trajectory( + TrajectoryPageRequest(cursor=search.hits[0].observation_id, limit=1) + ) + assert focused.observations[0].record.id == search.hits[0].observation_id + + trajectory_evidence: tuple[EvidenceReference, ...] = () + cursor: ObservationId | None = None + while True: + page = tools.read_trajectory(TrajectoryPageRequest(cursor=cursor, limit=2)) + assert page.status is ToolStatus.OK + trajectory_evidence = (*trajectory_evidence, *page.artifacts) + if page.next_cursor is None: + break + cursor = page.next_cursor + + verification = tools.verify_environment( + EnvironmentCheckRequest(SOURCE_ID, CHECK_ID) + ) + unresolved: tuple[str, ...] + if verification.status is ToolStatus.UNAVAILABLE: + completion = CompletionStatus.UNKNOWN + verdict = MiningVerdict.AMBIGUOUS + unresolved = ("Production state was unavailable.",) + environment_evidence: tuple[EvidenceReference, ...] = () + observed_state = None + else: + assert verification.verification is not None + completion = verification.verification.status + verdict = ( + MiningVerdict.CONFIRMED_FAILURE + if completion is CompletionStatus.NOT_COMPLETED + else MiningVerdict.NO_FAILURE + ) + unresolved = () + environment_evidence = verification.artifacts + observed_state = verification.verification.observed_state + + return FailureMiningResult( + revision_id=tools.case.revision_id, + trace_id=tools.case.trace_id, + trace_digest=tools.case.trace_digest, + verdict=verdict, + user_job=tools.case.user_job, + source_ids=tuple(source.id for source in tools.case.sources), + completion_checks=( + CompletionCheck( + check_id=CHECK_ID, + required_outcome="The ticket is closed.", + agent_claim="Ticket closed successfully.", + observed_state=observed_state, + status=completion, + evidence=environment_evidence, + ), + ), + trajectory_evidence=trajectory_evidence, + environment_evidence=environment_evidence, + confidence=Confidence(0.95 if completion is not CompletionStatus.UNKNOWN else 0.4), + unresolved_questions=unresolved, + invalid_reason=None, + ) - assert first == second - assert first.manifest_path.read_text(encoding="utf-8") == f"{first.to_json()}\n" - assert all(admission.snapshot_path is not None for admission in first.admissions[:-1]) - assert first.admissions[-1].snapshot_path is None - good = next( - admission for admission in first.admissions if admission.trace_id == TraceId("good") - ) - assert good.snapshot_path is not None - snapshot = good.snapshot_path.read_text(encoding="utf-8") - assert "token" not in snapshot - assert "reviewed" not in snapshot +@dataclass(frozen=True, slots=True) +class ForgingHermesJudge: + delegate: FakeHermesJudge + + def investigate( + self, + case: TraceMiningCase, + tools: MiningTools, + ) -> FailureMiningResult: + result = self.delegate.investigate(case, tools) + forged = _evidence(EvidenceKind.ENVIRONMENT, "invented-state", "invented") + check = replace(result.completion_checks[0], evidence=(forged,)) + return replace( + result, + completion_checks=(check,), + environment_evidence=(forged,), + ) -def test_foreign_score_subject_cannot_label_trace(tmp_path: Path) -> None: - revision = _harness(tmp_path).process() - result = Mine( - revision, - _collection(tmp_path, revision, foreign_good_score=True), - _policy(), - ).run() - good = next( - admission for admission in result.admissions if admission.trace_id == TraceId("good") - ) - assert good.partition is TracePartition.AMBIGUOUS +@dataclass(frozen=True, slots=True) +class PartialTraceHermesJudge: + def investigate( + self, + case: TraceMiningCase, + tools: MiningTools, + ) -> FailureMiningResult: + page = tools.read_trajectory(TrajectoryPageRequest(cursor=None, limit=1)) + verification = tools.verify_environment( + EnvironmentCheckRequest(SOURCE_ID, CHECK_ID) + ) + assert verification.verification is not None + return FailureMiningResult( + revision_id=case.revision_id, + trace_id=case.trace_id, + trace_digest=case.trace_digest, + verdict=MiningVerdict.CONFIRMED_FAILURE, + user_job=case.user_job, + source_ids=tuple(source.id for source in case.sources), + completion_checks=( + CompletionCheck( + check_id=CHECK_ID, + required_outcome="The ticket is closed.", + agent_claim="Ticket closed successfully.", + observed_state=verification.verification.observed_state, + status=CompletionStatus.NOT_COMPLETED, + evidence=verification.artifacts, + ), + ), + trajectory_evidence=page.artifacts, + environment_evidence=verification.artifacts, + confidence=Confidence(0.9), + unresolved_questions=(), + invalid_reason=None, + ) -def test_mine_freezes_redacted_content_as_verified_artifact_references( +@pytest.mark.parametrize( + ("outputs", "source_kind", "state", "observed", "search", "expected"), + ( + ( + ("Close ticket", "update failed", "continuing", "Ticket closed successfully"), + FailureSourceKind.DOWNSTREAM_FAILURE, + CompletionStatus.NOT_COMPLETED, + "Ticket remains open", + "failed", + MiningVerdict.CONFIRMED_FAILURE, + ), + ( + ("Close ticket", "update failed", "retry succeeded", "Ticket closed successfully"), + FailureSourceKind.AGENT_ERROR, + CompletionStatus.COMPLETED, + "Ticket is closed", + "failed", + MiningVerdict.NO_FAILURE, + ), + ( + ("Close ticket", "update accepted", "Ticket closed successfully"), + FailureSourceKind.USER_CORRECTION, + CompletionStatus.NOT_COMPLETED, + "Ticket remains open", + "successfully", + MiningVerdict.CONFIRMED_FAILURE, + ), + ( + ("Close ticket", "update accepted", "Ticket closed successfully"), + FailureSourceKind.HUMAN_FEEDBACK, + CompletionStatus.UNKNOWN, + None, + "successfully", + MiningVerdict.AMBIGUOUS, + ), + ), +) +def test_mine_uses_complete_trajectory_and_verified_state( tmp_path: Path, + outputs: tuple[str, ...], + source_kind: FailureSourceKind, + state: CompletionStatus, + observed: str | None, + search: str, + expected: MiningVerdict, ) -> None: - revision = _harness(tmp_path).process() - collection = _collection(tmp_path, revision, content=True) - - result = Mine(revision, collection, _policy()).run() - - failed = next( - admission for admission in result.admissions if admission.trace_id == TraceId("failed") - ) - assert failed.snapshot_path is not None - snapshot = TypeAdapter(TraceSnapshot).validate_json(failed.snapshot_path.read_bytes()) - observation = snapshot.observations[0] - assert observation.input_content is not None - assert observation.output_content is not None - assert "request failed" not in failed.snapshot_path.read_text(encoding="utf-8") - input_content = read_snapshot_content(result, observation.input_content) - assert input_content.text == "request failed [REDACTED_EMAIL]" - assert input_content.reference == observation.input_content.content - collection.store_path.unlink() - assert read_snapshot_content(result, observation.input_content) == input_content - - observation.input_content.path.write_text("tampered", encoding="utf-8") - with pytest.raises(MineError) as raised: - read_snapshot_content(result, observation.input_content) - assert raised.value.code is MineErrorCode.CONTENT_INVALID - - -def test_source_window_is_part_of_mine_identity(tmp_path: Path) -> None: - revision = _harness(tmp_path).process() - collection = _collection(tmp_path, revision) - shifted = replace( - collection, - window=TraceWindow( - collection.window.start + timedelta(hours=1), - collection.window.end + timedelta(hours=1), - ), - ) - - first = Mine(revision, collection, _policy()).run() - second = Mine(revision, shifted, _policy()).run() + revision = _revision(tmp_path) + run = Mine( + revision=revision, + collection=_collection(tmp_path, revision, outputs), + nominations=(_nomination(source_kind),), + judge=FakeHermesJudge(search), + environment=RecordedEnvironmentVerifier(state, observed), + ).run() - assert first.id != second.id + assert run.results[0].verdict is expected + assert len(run.results[0].trajectory_evidence) == len(outputs) -def test_processed_harness_is_accepted_and_later_connection_makes_it_stale( +@pytest.mark.parametrize( + ("collection_revision", "attribution", "gaps", "reason"), + ( + ("other-revision", AttributionLevel.EXACT, (), MiningInvalidReason.REVISION_MISMATCH), + ( + "revision-1", + AttributionLevel.MISSING, + (TraceGap.MISSING_ROOT,), + MiningInvalidReason.CORRUPT_TRACE, + ), + ), +) +def test_wrong_revision_or_corrupt_trace_is_invalid( tmp_path: Path, + collection_revision: str, + attribution: AttributionLevel, + gaps: tuple[TraceGap, ...], + reason: MiningInvalidReason, ) -> None: - harness = _harness(tmp_path) - revision = harness.process() - collection = _collection(tmp_path, revision) - assert Mine(harness, collection, _policy()).run().revision_id == revision.id - - harness.connect_skills(Path("memory.md")) - - with pytest.raises(MineError) as raised: - Mine(harness, collection, _policy()).run() - assert raised.value.code is MineErrorCode.STALE_HARNESS - + revision = _revision(tmp_path) + foreign_revision = _revision(tmp_path, collection_revision) + collection = _collection( + tmp_path, + foreign_revision, + ("Close ticket", "Ticket closed successfully"), + attribution=attribution, + gaps=gaps, + ) -def test_processed_harness_is_stale_after_external_file_change(tmp_path: Path) -> None: - harness = _harness(tmp_path) - revision = harness.process() - collection = _collection(tmp_path, revision) - (harness.root / "prompt.md").write_text("Changed externally.\n", encoding="utf-8") + result = Mine( + revision=revision, + collection=collection, + nominations=(_nomination(FailureSourceKind.AGENT_ERROR),), + judge=FakeHermesJudge("successfully"), + environment=RecordedEnvironmentVerifier(CompletionStatus.COMPLETED, "closed"), + ).run().results[0] + + assert result.verdict is MiningVerdict.INVALID + assert result.invalid_reason is reason + + +def test_confirmed_failure_requires_trajectory_and_environment_evidence() -> None: + with pytest.raises(ValueError, match="confirmed failure requires"): + FailureMiningResult( + revision_id=HarnessRevisionId("revision-1"), + trace_id=TRACE_ID, + trace_digest=_digest("trace-1"), + verdict=MiningVerdict.CONFIRMED_FAILURE, + user_job="Close the customer ticket.", + source_ids=(FailureSourceId("signal-1"),), + completion_checks=( + CompletionCheck( + check_id=CHECK_ID, + required_outcome="The ticket is closed.", + agent_claim="Ticket closed successfully.", + observed_state="Ticket remains open.", + status=CompletionStatus.NOT_COMPLETED, + evidence=(), + ), + ), + trajectory_evidence=(), + environment_evidence=(), + confidence=Confidence(0.9), + unresolved_questions=(), + invalid_reason=None, + ) - with pytest.raises(MineError) as raised: - Mine(harness, collection, _policy()).run() - assert raised.value.code is MineErrorCode.STALE_HARNESS +def test_judge_cannot_invent_evidence_that_no_tool_returned(tmp_path: Path) -> None: + revision = _revision(tmp_path) + result = Mine( + revision=revision, + collection=_collection( + tmp_path, + revision, + ("Close ticket", "update failed", "Ticket closed successfully"), + ), + nominations=(_nomination(FailureSourceKind.DOWNSTREAM_FAILURE),), + judge=ForgingHermesJudge(FakeHermesJudge("failed")), + environment=RecordedEnvironmentVerifier( + CompletionStatus.NOT_COMPLETED, + "Ticket remains open", + ), + ).run().results[0] + assert result.verdict is MiningVerdict.INVALID + assert result.invalid_reason is MiningInvalidReason.JUDGE_OUTPUT -def test_collection_from_another_revision_is_rejected(tmp_path: Path) -> None: - first = _harness(tmp_path) - first_revision = first.process() - collection = _collection(tmp_path, first_revision) - (first.root / "prompt.md").write_text("Changed.\n", encoding="utf-8") - second_revision = first.process() - with pytest.raises(MineError) as raised: - Mine(second_revision, collection, _policy()).run() +def test_judge_must_read_the_full_trace_before_returning_verdict(tmp_path: Path) -> None: + revision = _revision(tmp_path) + result = Mine( + revision=revision, + collection=_collection( + tmp_path, + revision, + ("Close ticket", "update failed", "Ticket closed successfully"), + ), + nominations=(_nomination(FailureSourceKind.DOWNSTREAM_FAILURE),), + judge=PartialTraceHermesJudge(), + environment=RecordedEnvironmentVerifier( + CompletionStatus.NOT_COMPLETED, + "Ticket remains open", + ), + ).run().results[0] - assert raised.value.code is MineErrorCode.REVISION_MISMATCH + assert result.verdict is MiningVerdict.INVALID + assert result.invalid_reason is MiningInvalidReason.JUDGE_OUTPUT From 333c6e72c88de46b3aa76e784bb0376fbb20e4d8 Mon Sep 17 00:00:00 2001 From: divo12 Date: Tue, 25 Aug 2026 09:02:11 +0530 Subject: [PATCH 10/18] remove stale mining runner leftovers --- src/ofw/_runner.py | 30 ------------------------- src/ofw/_verifier_runner.py | 45 ------------------------------------- 2 files changed, 75 deletions(-) delete mode 100644 src/ofw/_runner.py delete mode 100644 src/ofw/_verifier_runner.py diff --git a/src/ofw/_runner.py b/src/ofw/_runner.py deleted file mode 100644 index 9da394a..0000000 --- a/src/ofw/_runner.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Child-process entrypoint for a file-backed Python lifecycle function.""" - -from __future__ import annotations - -import importlib -import inspect -import sys -from collections.abc import Callable -from typing import cast - - -def main() -> int: - if len(sys.argv) != 3: - return 2 - module = importlib.import_module(sys.argv[1]) - functions = tuple( - function - for name, function in inspect.getmembers(module, inspect.isfunction) - if name == sys.argv[2] - ) - if len(functions) != 1: - return 2 - function: Callable[[str], str] = cast(Callable[[str], str], functions[0]) - output: str = function(sys.stdin.read()) # type: ignore[misc] - sys.stdout.write(output) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/src/ofw/_verifier_runner.py b/src/ofw/_verifier_runner.py deleted file mode 100644 index a08121f..0000000 --- a/src/ofw/_verifier_runner.py +++ /dev/null @@ -1,45 +0,0 @@ -"""Timed child-process entrypoint for a file-backed Python verifier.""" - -from __future__ import annotations - -import importlib -import inspect -import sys -from collections.abc import Callable -from typing import cast - -from pydantic import TypeAdapter, ValidationError - -from ofw.runtime import RunResult, VerifierResult - -_RUN_ADAPTER: TypeAdapter[RunResult] = TypeAdapter(RunResult) -_VERIFIER_ADAPTER: TypeAdapter[VerifierResult] = TypeAdapter(VerifierResult) - - -def main() -> int: - if len(sys.argv) != 3: - return 2 - payload: str = sys.stdin.read() - try: - result = _RUN_ADAPTER.validate_json(payload) - except ValidationError: - return 2 - module = importlib.import_module(sys.argv[1]) - functions = tuple( - function - for name, function in inspect.getmembers(module, inspect.isfunction) - if name == sys.argv[2] - ) - if len(functions) != 1: - return 2 - function: Callable[[RunResult], VerifierResult] = cast( - Callable[[RunResult], VerifierResult], - functions[0], - ) - verified: VerifierResult = function(result) - sys.stdout.write(_VERIFIER_ADAPTER.dump_json(verified).decode()) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) From bedd2779f1a8424086813994cc235e3c1ac4fcb1 Mon Sep 17 00:00:00 2001 From: divo12 Date: Tue, 25 Aug 2026 14:07:58 +0530 Subject: [PATCH 11/18] align failure judge contracts and tools --- src/ofw/__init__.py | 36 ++- src/ofw/mine.py | 545 +++++++++++++++++++++++++++++++++----------- tests/test_mine.py | 353 ++++++++++++++++++++++++++-- 3 files changed, 787 insertions(+), 147 deletions(-) diff --git a/src/ofw/__init__.py b/src/ofw/__init__.py index 5c8c0e2..28d6548 100644 --- a/src/ofw/__init__.py +++ b/src/ofw/__init__.py @@ -28,9 +28,13 @@ ) from ofw.harness import EditableFile, Harness, Subagent, Tool, editable from ofw.mine import ( + AdaptationRequest, + AdaptationResult, + BehaviorObservation, CompletionCheck, CompletionStatus, - EnvironmentCheck, + Confidence, + ConstraintKind, EnvironmentCheckId, EnvironmentCheckRequest, EnvironmentSource, @@ -40,15 +44,27 @@ EvidenceKind, EvidenceRecordId, EvidenceReference, + FailureBehavior, + FailureBehaviorKind, FailureMiningResult, FailureMiningRun, + FailurePhase, FailureSource, FailureSourceId, FailureSourceKind, Mine, + MiningContext, MiningInvalidReason, MiningNomination, + MiningTask, MiningVerdict, + RecoveryStatus, + RequiredOutcome, + TaskConstraint, + TaskId, + ToolAccess, + ToolCapability, + ToolName, TraceMiningCase, ) from ofw.observability.langfuse import ( @@ -141,6 +157,9 @@ def read_observation_content( __all__ = [ "AssetAccess", + "AdaptationRequest", + "AdaptationResult", + "BehaviorObservation", "CanaryCase", "CaseId", "ComponentKind", @@ -151,9 +170,10 @@ def read_observation_content( "CommandVerifier", "CompletionCheck", "CompletionStatus", + "Confidence", + "ConstraintKind", "E2BSandbox", "EditableFile", - "EnvironmentCheck", "EnvironmentCheckId", "EnvironmentCheckRequest", "EnvironmentSource", @@ -165,6 +185,9 @@ def read_observation_content( "EvidenceReference", "FailureMiningResult", "FailureMiningRun", + "FailureBehavior", + "FailureBehaviorKind", + "FailurePhase", "FailureSource", "FailureSourceId", "FailureSourceKind", @@ -182,8 +205,10 @@ def read_observation_content( "LangfuseSpan", "ModelFingerprint", "Mine", + "MiningContext", "MiningInvalidReason", "MiningNomination", + "MiningTask", "MiningVerdict", "ObservationContent", "ObservationContentField", @@ -192,6 +217,8 @@ def read_observation_content( "ObservationContentQuery", "ObservationContentReference", "RepositorySnapshot", + "RecoveryStatus", + "RequiredOutcome", "ProcessCommand", "ProcessLimits", "RunErrorCode", @@ -199,7 +226,12 @@ def read_observation_content( "RunStatus", "Sha256Digest", "Subagent", + "TaskConstraint", + "TaskId", "Tool", + "ToolAccess", + "ToolCapability", + "ToolName", "TraceWindow", "TraceMiningCase", "VerifierResult", diff --git a/src/ofw/mine.py b/src/ofw/mine.py index 13e6f43..e58c10c 100644 --- a/src/ofw/mine.py +++ b/src/ofw/mine.py @@ -81,11 +81,49 @@ class ToolStatus(StrEnum): class ToolAction(StrEnum): SEARCH_TRAJECTORY = "search_trajectory" + SEARCH_PRIOR_TRAJECTORIES = "search_prior_trajectories" READ_TRAJECTORY = "read_trajectory" VERIFY_ENVIRONMENT = "verify_environment" + ADAPT = "adapt" RETURN_VERDICT = "return_verdict" +class ConstraintKind(StrEnum): + TIME = "time" + RESOURCE = "resource" + NETWORK = "network" + ACCESS = "access" + POLICY = "policy" + + +class ToolAccess(StrEnum): + READ_ONLY = "read_only" + MUTATING = "mutating" + + +class FailureBehaviorKind(StrEnum): + OUTCOME_MISMATCH = "outcome_mismatch" + FALSE_COMPLETION = "false_completion" + REQUIRED_ACTION_OMITTED = "required_action_omitted" + FORBIDDEN_STATE_CHANGE = "forbidden_state_change" + UNRECOVERED_ACTION_FAILURE = "unrecovered_action_failure" + NO_PROGRESS_LOOP = "no_progress_loop" + ABANDONED_BEFORE_COMPLETION = "abandoned_before_completion" + + +class FailurePhase(StrEnum): + ACTION = "action" + RECOVERY = "recovery" + COMPLETION = "completion" + VERIFICATION = "verification" + + +class RecoveryStatus(StrEnum): + RECOVERED = "recovered" + NOT_RECOVERED = "not_recovered" + UNKNOWN = "unknown" + + @dataclass(frozen=True, slots=True) class FailureSourceId: value: str @@ -110,6 +148,22 @@ def __post_init__(self) -> None: _require_identifier(self.value) +@dataclass(frozen=True, slots=True) +class TaskId: + value: str + + def __post_init__(self) -> None: + _require_identifier(self.value) + + +@dataclass(frozen=True, slots=True) +class ToolName: + value: str + + def __post_init__(self) -> None: + _require_identifier(self.value) + + @dataclass(frozen=True, slots=True) class EvidenceRecordId: value: str @@ -134,6 +188,48 @@ class EvidenceReference: digest: Sha256Digest +@dataclass(frozen=True, slots=True) +class RequiredOutcome: + check_id: EnvironmentCheckId + source_id: EnvironmentSourceId + description: str + + def __post_init__(self) -> None: + _require_text(self.description, "required outcome description") + + +@dataclass(frozen=True, slots=True) +class TaskConstraint: + kind: ConstraintKind + description: str + + def __post_init__(self) -> None: + _require_text(self.description, "task constraint description") + + +@dataclass(frozen=True, slots=True) +class MiningTask: + id: TaskId + intent: str + required_outcomes: tuple[RequiredOutcome, ...] + constraints: tuple[TaskConstraint, ...] = () + + def __post_init__(self) -> None: + _require_text(self.intent, "task intent") + if not self.required_outcomes: + raise ValueError("mining task requires a required outcome") + if len({item.check_id for item in self.required_outcomes}) != len( + self.required_outcomes + ): + raise ValueError("required outcome ids must be unique") + + +@dataclass(frozen=True, slots=True) +class ToolCapability: + name: ToolName + access: ToolAccess + + @dataclass(frozen=True, slots=True) class FailureSource: id: FailureSourceId @@ -145,17 +241,10 @@ class FailureSource: def __post_init__(self) -> None: _require_text(self.summary, "failure source summary") - if not self.evidence: - raise ValueError("failure source requires evidence") - - -@dataclass(frozen=True, slots=True) -class EnvironmentCheck: - id: EnvironmentCheckId - required_outcome: str - - def __post_init__(self) -> None: - _require_text(self.required_outcome, "required outcome") + if not self.evidence or any( + item.kind is not EvidenceKind.PRODUCTION_SIGNAL for item in self.evidence + ): + raise ValueError("failure source requires production-signal evidence") @dataclass(frozen=True, slots=True) @@ -163,44 +252,100 @@ class EnvironmentSource: id: EnvironmentSourceId kind: EnvironmentSourceKind summary: str - checks: tuple[EnvironmentCheck, ...] def __post_init__(self) -> None: _require_text(self.summary, "environment source summary") - if not self.checks or len({check.id for check in self.checks}) != len(self.checks): - raise ValueError("environment source requires unique checks") @dataclass(frozen=True, slots=True) class MiningNomination: trace_id: TraceId - user_job: str + task: MiningTask sources: tuple[FailureSource, ...] environment_sources: tuple[EnvironmentSource, ...] + available_tools: tuple[ToolCapability, ...] = () + initial_state_evidence: tuple[EvidenceReference, ...] = () def __post_init__(self) -> None: - _require_text(self.user_job, "user job") if not self.sources: raise ValueError("mining nomination requires a failure source") if any(source.trace_id != self.trace_id for source in self.sources): raise ValueError("failure source trace does not match nomination") if len({source.id for source in self.sources}) != len(self.sources): raise ValueError("failure source ids must be unique") - if len({source.id for source in self.environment_sources}) != len( - self.environment_sources - ): + source_ids = {source.id for source in self.environment_sources} + if len(source_ids) != len(self.environment_sources): raise ValueError("environment source ids must be unique") + if any(outcome.source_id not in source_ids for outcome in self.task.required_outcomes): + raise ValueError("required outcome references an undeclared environment source") + if len({tool.name for tool in self.available_tools}) != len(self.available_tools): + raise ValueError("tool capabilities must be unique") + if any( + item.kind is not EvidenceKind.ENVIRONMENT + for item in self.initial_state_evidence + ): + raise ValueError("initial state requires environment evidence") @dataclass(frozen=True, slots=True) -class TraceMiningCase: +class MiningContext: revision_id: HarnessRevisionId trace_id: TraceId trace_digest: Sha256Digest observation_ids: tuple[ObservationId, ...] - user_job: str - sources: tuple[FailureSource, ...] + session_id: str | None + environment_name: str | None + release: str | None + available_tools: tuple[ToolCapability, ...] environment_sources: tuple[EnvironmentSource, ...] + initial_state_evidence: tuple[EvidenceReference, ...] + + def __post_init__(self) -> None: + if not self.observation_ids or len(set(self.observation_ids)) != len( + self.observation_ids + ): + raise ValueError("mining context requires unique observation ids") + + +@dataclass(frozen=True, slots=True) +class BehaviorObservation: + kind: FailureBehaviorKind + phase: FailurePhase + first_observation_id: ObservationId + last_observation_id: ObservationId | None + recovery_status: RecoveryStatus + evidence: tuple[EvidenceReference, ...] + + def __post_init__(self) -> None: + if not self.evidence or any( + item.kind is not EvidenceKind.TRAJECTORY for item in self.evidence + ): + raise ValueError("failure behavior observation requires trajectory evidence") + + +@dataclass(frozen=True, slots=True) +class FailureBehavior: + primary: FailureBehaviorKind + summary: str + observations: tuple[BehaviorObservation, ...] + + def __post_init__(self) -> None: + _require_text(self.summary, "failure behavior summary") + if not self.observations: + raise ValueError("failure behavior requires observations") + if not any(item.kind is self.primary for item in self.observations): + raise ValueError("primary failure behavior must appear in observations") + + +@dataclass(frozen=True, slots=True) +class TraceMiningCase: + task: MiningTask + context: MiningContext + sources: tuple[FailureSource, ...] + + @property + def trace_id(self) -> TraceId: + return self.context.trace_id @dataclass(frozen=True, slots=True) @@ -217,6 +362,8 @@ class EnvironmentVerification: def __post_init__(self) -> None: if self.status is CompletionStatus.UNKNOWN: + if self.observed_state is not None or self.evidence: + raise ValueError("unknown environment state cannot carry evidence") return if self.observed_state is None or not self.observed_state.strip() or not self.evidence: raise ValueError("known environment state requires observation and evidence") @@ -236,13 +383,12 @@ class CompletionCheck: @dataclass(frozen=True, slots=True) class FailureMiningResult: - revision_id: HarnessRevisionId - trace_id: TraceId - trace_digest: Sha256Digest | None + task: MiningTask + context: MiningContext | None verdict: MiningVerdict - user_job: str source_ids: tuple[FailureSourceId, ...] completion_checks: tuple[CompletionCheck, ...] + failure_behavior: FailureBehavior | None trajectory_evidence: tuple[EvidenceReference, ...] environment_evidence: tuple[EvidenceReference, ...] confidence: Confidence @@ -251,16 +397,19 @@ class FailureMiningResult: def __post_init__(self) -> None: if self.verdict is MiningVerdict.INVALID: - if self.invalid_reason is None: - raise ValueError("invalid result requires a reason") + if self.invalid_reason is None or self.failure_behavior is not None: + raise ValueError("invalid result requires a reason and no failure behavior") return - if self.invalid_reason is not None or self.trace_digest is None: - raise ValueError("valid result cannot carry an invalid reason") + if self.context is None or self.invalid_reason is not None: + raise ValueError("valid result requires context and no invalid reason") if not self.completion_checks: raise ValueError("mining result requires completion checks") + self._validate_checks() + self._validate_behavior() if self.verdict is MiningVerdict.CONFIRMED_FAILURE: if ( - not any( + self.failure_behavior is None + or not any( check.status is CompletionStatus.NOT_COMPLETED for check in self.completion_checks ) @@ -268,12 +417,13 @@ def __post_init__(self) -> None: or not self.environment_evidence ): raise ValueError( - "confirmed failure requires failed completion, trajectory, " + "confirmed failure requires behavior, failed completion, trajectory, " "and environment evidence" ) elif self.verdict is MiningVerdict.NO_FAILURE: if ( - any( + self.failure_behavior is not None + or any( check.status is not CompletionStatus.COMPLETED for check in self.completion_checks ) @@ -281,12 +431,40 @@ def __post_init__(self) -> None: or not self.environment_evidence ): raise ValueError( - "no-failure verdict requires completed checks and supporting evidence" + "no-failure verdict requires completed checks, no failure behavior, " + "and supporting evidence" ) - elif not self.unresolved_questions and not any( - check.status is CompletionStatus.UNKNOWN for check in self.completion_checks + elif ( + self.failure_behavior is not None + or not self.unresolved_questions + and not any( + check.status is CompletionStatus.UNKNOWN for check in self.completion_checks + ) ): - raise ValueError("ambiguous verdict requires an unresolved question") + raise ValueError("ambiguous verdict requires no behavior and an unresolved question") + + def _validate_checks(self) -> None: + outcomes = {item.check_id: item for item in self.task.required_outcomes} + if len(self.completion_checks) != len(outcomes): + raise ValueError("completion checks must cover every required outcome") + for check in self.completion_checks: + outcome = outcomes.get(check.check_id) + if outcome is None or check.required_outcome != outcome.description: + raise ValueError("completion check does not match the task") + + def _validate_behavior(self) -> None: + if self.context is None or self.failure_behavior is None: + return + ids = set(self.context.observation_ids) + evidence = set(self.trajectory_evidence) + for observation in self.failure_behavior.observations: + if ( + observation.first_observation_id not in ids + or observation.last_observation_id is not None + and observation.last_observation_id not in ids + or not set(observation.evidence).issubset(evidence) + ): + raise ValueError("failure behavior observation is outside the grounded context") @dataclass(frozen=True, slots=True) @@ -353,12 +531,32 @@ class EnvironmentCheckResult: artifacts: tuple[EvidenceReference, ...] +@dataclass(frozen=True, slots=True) +class AdaptationRequest: + kinds: tuple[FailureSourceKind, ...] + limit: int + + def __post_init__(self) -> None: + if not self.kinds or len(set(self.kinds)) != len(self.kinds): + raise ValueError("adaptation requires unique signal kinds") + if not 1 <= self.limit <= 100: + raise ValueError("adaptation limit must be between 1 and 100") + + +@dataclass(frozen=True, slots=True) +class AdaptationResult: + status: ToolStatus + summary: str + signals: tuple[FailureSource, ...] + next_actions: tuple[ToolAction, ...] + + class EnvironmentVerifier(Protocol): def verify( self, request: EnvironmentCheckRequest, source: EnvironmentSource, - check: EnvironmentCheck, + outcome: RequiredOutcome, ) -> EnvironmentVerification: ... @@ -375,15 +573,10 @@ class MiningTools: case: TraceMiningCase collection: CollectionResult environment: EnvironmentVerifier - _issued_evidence: list[EvidenceReference] = field( - default_factory=list, - init=False, - repr=False, - ) + production_signals: tuple[FailureSource, ...] + _issued_evidence: list[EvidenceReference] = field(default_factory=list, init=False) _read_trajectory_evidence: list[EvidenceReference] = field( - default_factory=list, - init=False, - repr=False, + default_factory=list, init=False ) @property @@ -395,19 +588,33 @@ def read_trajectory_evidence(self) -> tuple[EvidenceReference, ...]: return tuple(self._read_trajectory_evidence) def search_trajectory(self, request: TrajectorySearchRequest) -> TrajectorySearchResult: + return self._search(request, self.case.trace_id, False) + + def search_prior_trajectories( + self, request: TrajectorySearchRequest + ) -> TrajectorySearchResult: + return self._search(request, None, True) + + def _search( + self, + request: TrajectorySearchRequest, + trace_id: TraceId | None, + prior_only: bool, + ) -> TrajectorySearchResult: store = CollectionStore(self.collection.store_path) try: - hits = store.search_content( - self.collection.observation_sync_id, - ObservationContentQuery( - text=request.text, - match=ObservationContentMatch.TOKEN_PHRASE, - field=request.field, - trace_id=self.case.trace_id, - limit=request.limit, - maximum_excerpt_characters=1000, - ), - ) + limit = 100 if prior_only else request.limit + hits = self._search_phrase(store, request, trace_id, request.text, limit) + if not hits: + found: list[ObservationContentHit] = [] + for token in _search_tokens(request.text): + for hit in self._search_phrase(store, request, trace_id, token, limit): + if hit not in found: + found.append(hit) + if len(found) >= limit: + break + hits = tuple(found[:limit]) + hits = tuple(_focus_hit(store, self.collection, hit, request.text) for hit in hits) except CollectionError: return TrajectorySearchResult( ToolStatus.ERROR, @@ -418,6 +625,10 @@ def search_trajectory(self, request: TrajectorySearchRequest) -> TrajectorySearc ) finally: store.close() + if prior_only: + hits = tuple(hit for hit in hits if hit.trace_id != self.case.trace_id)[ + : request.limit + ] artifacts = tuple( EvidenceReference( EvidenceKind.TRAJECTORY, @@ -435,15 +646,35 @@ def search_trajectory(self, request: TrajectorySearchRequest) -> TrajectorySearc artifacts, ) + def _search_phrase( + self, + store: CollectionStore, + request: TrajectorySearchRequest, + trace_id: TraceId | None, + text: str, + limit: int, + ) -> tuple[ObservationContentHit, ...]: + return store.search_content( + self.collection.observation_sync_id, + ObservationContentQuery( + text=text, + match=ObservationContentMatch.TOKEN_PHRASE, + field=request.field, + trace_id=trace_id, + limit=limit, + maximum_excerpt_characters=1000, + ), + ) + def read_trajectory(self, request: TrajectoryPageRequest) -> TrajectoryPageResult: store = CollectionStore(self.collection.store_path) try: - # ponytail: collection-wide scan is simplest for local v0; add a paged - # trace SQL query if production profiles show this O(pages * records) path. + # ponytail: collection scan is simplest for local v0; add a trace SQL query + # if production profiles show this O(pages * records) path matters. observations = tuple( - observation - for observation in store.observations(self.collection.observation_sync_id) - if observation.trace_id == self.case.trace_id + item + for item in store.observations(self.collection.observation_sync_id) + if item.trace_id == self.case.trace_id ) start = _page_start(observations, request.cursor) if start is None: @@ -488,7 +719,12 @@ def read_trajectory(self, request: TrajectoryPageRequest) -> TrajectoryPageResul ( (ToolAction.READ_TRAJECTORY,) if next_cursor is not None - else (ToolAction.VERIFY_ENVIRONMENT, ToolAction.RETURN_VERDICT) + else ( + ToolAction.SEARCH_PRIOR_TRAJECTORIES, + ToolAction.VERIFY_ENVIRONMENT, + ToolAction.ADAPT, + ToolAction.RETURN_VERDICT, + ) ), artifacts, ) @@ -503,23 +739,37 @@ def verify_environment(self, request: EnvironmentCheckRequest) -> EnvironmentChe (ToolAction.RETURN_VERDICT,), (), ) - source, check = selected - verification = self.environment.verify(request, source, check) + source, outcome = selected + verification = self.environment.verify(request, source, outcome) self._issued_evidence.extend(verification.evidence) return EnvironmentCheckResult( - ( - ToolStatus.UNAVAILABLE - if verification.status is CompletionStatus.UNKNOWN - else ToolStatus.OK - ), + ToolStatus.UNAVAILABLE + if verification.status is CompletionStatus.UNKNOWN + else ToolStatus.OK, "Environment state is unavailable." if verification.status is CompletionStatus.UNKNOWN else "Environment state verified.", verification, - (ToolAction.RETURN_VERDICT,), + (ToolAction.ADAPT, ToolAction.RETURN_VERDICT), verification.evidence, ) + def adapt(self, request: AdaptationRequest) -> AdaptationResult: + kinds = set(request.kinds) + signals = tuple( + signal for signal in self.production_signals if signal.kind in kinds + )[: request.limit] + return AdaptationResult( + ToolStatus.OK if signals else ToolStatus.NOT_FOUND, + f"Found {len(signals)} human or production calibration signals.", + signals, + ( + ToolAction.SEARCH_PRIOR_TRAJECTORIES + if signals + else ToolAction.RETURN_VERDICT, + ), + ) + @dataclass(frozen=True, slots=True) class Mine: @@ -534,35 +784,39 @@ def __post_init__(self) -> None: raise ValueError("mine requires at least one nomination") def run(self) -> FailureMiningRun: - results = tuple(self._mine(nomination) for nomination in self.nominations) + signals = tuple(source for item in self.nominations for source in item.sources) + results = tuple(self._mine(nomination, signals) for nomination in self.nominations) return FailureMiningRun(self.revision.id, self.collection.snapshot_digest, results) - def _mine(self, nomination: MiningNomination) -> FailureMiningResult: + def _mine( + self, + nomination: MiningNomination, + production_signals: tuple[FailureSource, ...], + ) -> FailureMiningResult: trace = next( (item for item in self.collection.traces if item.id == nomination.trace_id), None, ) if self.collection.revision_id != self.revision.id: - return _invalid( - self.revision.id, - nomination, - trace, - MiningInvalidReason.REVISION_MISMATCH, - ) + return _invalid(nomination, MiningInvalidReason.REVISION_MISMATCH) if trace is None: - return _invalid(self.revision.id, nomination, None, MiningInvalidReason.TRACE_NOT_FOUND) + return _invalid(nomination, MiningInvalidReason.TRACE_NOT_FOUND) if not _trace_is_complete(self.collection, trace): - return _invalid(self.revision.id, nomination, trace, MiningInvalidReason.CORRUPT_TRACE) - case = TraceMiningCase( - self.revision.id, - trace.id, - trace.digest, - trace.observation_ids, - nomination.user_job, - nomination.sources, - nomination.environment_sources, + return _invalid(nomination, MiningInvalidReason.CORRUPT_TRACE) + context = MiningContext( + revision_id=self.revision.id, + trace_id=trace.id, + trace_digest=trace.digest, + observation_ids=trace.observation_ids, + session_id=trace.session_id, + environment_name=trace.environment, + release=trace.release, + available_tools=nomination.available_tools, + environment_sources=nomination.environment_sources, + initial_state_evidence=nomination.initial_state_evidence, ) - tools = MiningTools(case, self.collection, self.environment) + case = TraceMiningCase(nomination.task, context, nomination.sources) + tools = MiningTools(case, self.collection, self.environment, production_signals) result = self.judge.investigate(case, tools) if not _judge_result_matches( case, @@ -570,7 +824,7 @@ def _mine(self, nomination: MiningNomination) -> FailureMiningResult: tools.issued_evidence, tools.read_trajectory_evidence, ): - return _invalid(self.revision.id, nomination, trace, MiningInvalidReason.JUDGE_OUTPUT) + return _invalid(nomination, MiningInvalidReason.JUDGE_OUTPUT) return result @@ -580,9 +834,9 @@ def _trace_is_complete(collection: CollectionResult, trace: TraceRecord) -> bool store = CollectionStore(collection.store_path) try: observations = tuple( - observation - for observation in store.observations(collection.observation_sync_id) - if observation.trace_id == trace.id + item + for item in store.observations(collection.observation_sync_id) + if item.trace_id == trace.id ) for observation in observations: _read_observation(store, collection, observation) @@ -590,10 +844,7 @@ def _trace_is_complete(collection: CollectionResult, trace: TraceRecord) -> bool return False finally: store.close() - return ( - bool(observations) - and tuple(observation.id for observation in observations) == trace.observation_ids - ) + return bool(observations) and tuple(item.id for item in observations) == trace.observation_ids def _judge_result_matches( @@ -602,47 +853,58 @@ def _judge_result_matches( issued_evidence: tuple[EvidenceReference, ...], read_trajectory_evidence: tuple[EvidenceReference, ...], ) -> bool: - observation_ids = {evidence.record_id.value for evidence in result.trajectory_evidence} - read_observation_ids = { - ObservationId(evidence.record_id.value) for evidence in read_trajectory_evidence - } - allowed_checks = { - check.id - for source in case.environment_sources - for check in source.checks - } + if result.context is None: + return False + read_ids = {ObservationId(item.record_id.value) for item in read_trajectory_evidence} + behavior_evidence = ( + () + if result.failure_behavior is None + else tuple( + evidence + for observation in result.failure_behavior.observations + for evidence in observation.evidence + ) + ) completion_evidence = tuple( evidence for check in result.completion_checks for evidence in check.evidence ) return ( - result.revision_id == case.revision_id - and result.trace_id == case.trace_id - and result.trace_digest == case.trace_digest + result.task == case.task + and result.context == case.context and result.source_ids == tuple(source.id for source in case.sources) - and read_observation_ids == set(case.observation_ids) + and read_ids == set(case.context.observation_ids) and all(item.kind is EvidenceKind.TRAJECTORY for item in result.trajectory_evidence) and all(item in result.trajectory_evidence for item in read_trajectory_evidence) - and observation_ids.issubset({item.value for item in case.observation_ids}) and all(item.kind is EvidenceKind.ENVIRONMENT for item in result.environment_evidence) and all(item.kind is EvidenceKind.ENVIRONMENT for item in completion_evidence) and all(item in issued_evidence for item in result.trajectory_evidence) and all(item in issued_evidence for item in result.environment_evidence) and all(item in issued_evidence for item in completion_evidence) - and all(check.check_id in allowed_checks for check in result.completion_checks) + and all(item in issued_evidence for item in behavior_evidence) ) def _find_environment_check( case: TraceMiningCase, request: EnvironmentCheckRequest, -) -> tuple[EnvironmentSource, EnvironmentCheck] | None: - for source in case.environment_sources: - if source.id != request.source_id: - continue - for check in source.checks: - if check.id == request.check_id: - return source, check - return None +) -> tuple[EnvironmentSource, RequiredOutcome] | None: + source = next( + ( + item + for item in case.context.environment_sources + if item.id == request.source_id + ), + None, + ) + outcome = next( + ( + item + for item in case.task.required_outcomes + if item.source_id == request.source_id and item.check_id == request.check_id + ), + None, + ) + return None if source is None or outcome is None else (source, outcome) def _page_start( @@ -675,20 +937,49 @@ def _read_observation( return TrajectoryObservation(observation, input_content, output_content) +def _search_tokens(text: str) -> tuple[str, ...]: + tokens: list[str] = [] + for word in text.split(): + token = "".join(character for character in word if character.isalnum() or character in "-_") + if len(token) >= 3 and token.casefold() not in {item.casefold() for item in tokens}: + tokens.append(token) + return tuple(tokens) + + +def _focus_hit( + store: CollectionStore, + collection: CollectionResult, + hit: ObservationContentHit, + query: str, +) -> ObservationContentHit: + content = store.read_content(collection.observation_sync_id, hit.reference).text + lowered = content.casefold() + positions = tuple( + lowered.find(candidate.casefold()) + for candidate in (query, *_search_tokens(query)) + ) + position = next((item for item in positions if item >= 0), 0) + start = max(0, position - 300) + return ObservationContentHit( + hit.observation_id, + hit.trace_id, + hit.field, + hit.reference, + content[start : start + 1000], + ) + + def _invalid( - revision_id: HarnessRevisionId, nomination: MiningNomination, - trace: TraceRecord | None, reason: MiningInvalidReason, ) -> FailureMiningResult: return FailureMiningResult( - revision_id=revision_id, - trace_id=nomination.trace_id, - trace_digest=None if trace is None else trace.digest, + task=nomination.task, + context=None, verdict=MiningVerdict.INVALID, - user_job=nomination.user_job, source_ids=tuple(source.id for source in nomination.sources), completion_checks=(), + failure_behavior=None, trajectory_evidence=(), environment_evidence=(), confidence=Confidence(1.0), diff --git a/tests/test_mine.py b/tests/test_mine.py index 2f9774b..c98e1f5 100644 --- a/tests/test_mine.py +++ b/tests/test_mine.py @@ -17,10 +17,12 @@ Sha256Digest, ) from ofw.mine import ( + AdaptationRequest, + BehaviorObservation, CompletionCheck, CompletionStatus, Confidence, - EnvironmentCheck, + ConstraintKind, EnvironmentCheckId, EnvironmentCheckRequest, EnvironmentSource, @@ -30,15 +32,27 @@ EvidenceKind, EvidenceRecordId, EvidenceReference, + FailureBehavior, + FailureBehaviorKind, FailureMiningResult, + FailurePhase, FailureSource, FailureSourceId, FailureSourceKind, Mine, + MiningContext, MiningInvalidReason, MiningNomination, + MiningTask, MiningTools, MiningVerdict, + RecoveryStatus, + RequiredOutcome, + TaskConstraint, + TaskId, + ToolAccess, + ToolCapability, + ToolName, ToolStatus, TraceMiningCase, TrajectoryPageRequest, @@ -216,13 +230,60 @@ def _nomination(kind: FailureSourceKind) -> MiningNomination: id=SOURCE_ID, kind=EnvironmentSourceKind.PRODUCTION_API, summary="Read-only ITSM production state.", - checks=(EnvironmentCheck(CHECK_ID, "The ticket is closed."),), ) return MiningNomination( trace_id=TRACE_ID, - user_job="Close the customer ticket.", + task=_task(), sources=(source,), environment_sources=(environment,), + available_tools=( + ToolCapability(ToolName("update-ticket"), ToolAccess.MUTATING), + ), + ) + + +def _task() -> MiningTask: + return MiningTask( + id=TaskId("close-ticket"), + intent="Close the customer ticket.", + required_outcomes=( + RequiredOutcome( + check_id=CHECK_ID, + source_id=SOURCE_ID, + description="The ticket is closed.", + ), + ), + constraints=( + TaskConstraint( + kind=ConstraintKind.POLICY, + description="Do not claim completion without verifying the ticket state.", + ), + ), + ) + + +def _context(observation_ids: tuple[ObservationId, ...]) -> MiningContext: + return MiningContext( + revision_id=HarnessRevisionId("revision-1"), + trace_id=TRACE_ID, + trace_digest=_digest("trace-1"), + observation_ids=observation_ids, + session_id="session-1", + environment_name="production", + release="revision-1", + available_tools=( + ToolCapability(ToolName("search_trajectory"), ToolAccess.READ_ONLY), + ToolCapability(ToolName("read_trajectory"), ToolAccess.READ_ONLY), + ToolCapability(ToolName("verify_environment"), ToolAccess.READ_ONLY), + ), + environment_sources=( + EnvironmentSource( + id=SOURCE_ID, + kind=EnvironmentSourceKind.PRODUCTION_API, + summary="Read-only ITSM production state.", + ), + ), + initial_state_evidence=(), ) @@ -235,10 +296,10 @@ def verify( self, request: EnvironmentCheckRequest, source: EnvironmentSource, - check: EnvironmentCheck, + outcome: RequiredOutcome, ) -> EnvironmentVerification: assert request.source_id == source.id - assert request.check_id == check.id + assert request.check_id == outcome.check_id evidence = ( () if self.status is CompletionStatus.UNKNOWN @@ -306,12 +367,64 @@ def investigate( environment_evidence = verification.artifacts observed_state = verification.verification.observed_state + adapted = tools.adapt( + AdaptationRequest( + ( + FailureSourceKind.HUMAN_FEEDBACK, + FailureSourceKind.USER_CORRECTION, + FailureSourceKind.DOWNSTREAM_FAILURE, + FailureSourceKind.AGENT_ERROR, + ), + 10, + ) + ) + assert adapted.status is ToolStatus.OK + + behavior_evidence = next( + item + for item in trajectory_evidence + if item.record_id.value == search.hits[0].observation_id.value + ) + + failure_behavior = ( + None + if verdict is not MiningVerdict.CONFIRMED_FAILURE + else FailureBehavior( + primary=( + FailureBehaviorKind.FALSE_COMPLETION + if "successfully" in search.hits[0].excerpt.lower() + else FailureBehaviorKind.UNRECOVERED_ACTION_FAILURE + ), + summary=( + "The agent claimed success while the ticket stayed open." + if "successfully" in search.hits[0].excerpt.lower() + else "The tool failure was not recovered before completion." + ), + observations=( + BehaviorObservation( + kind=( + FailureBehaviorKind.FALSE_COMPLETION + if "successfully" in search.hits[0].excerpt.lower() + else FailureBehaviorKind.UNRECOVERED_ACTION_FAILURE + ), + phase=( + FailurePhase.COMPLETION + if "successfully" in search.hits[0].excerpt.lower() + else FailurePhase.RECOVERY + ), + first_observation_id=search.hits[0].observation_id, + last_observation_id=None, + recovery_status=RecoveryStatus.NOT_RECOVERED, + evidence=(behavior_evidence,), + ), + ), + ) + ) + return FailureMiningResult( - revision_id=tools.case.revision_id, - trace_id=tools.case.trace_id, - trace_digest=tools.case.trace_digest, + task=tools.case.task, + context=tools.case.context, verdict=verdict, - user_job=tools.case.user_job, source_ids=tuple(source.id for source in tools.case.sources), completion_checks=( CompletionCheck( @@ -323,6 +436,7 @@ def investigate( evidence=environment_evidence, ), ), + failure_behavior=failure_behavior, trajectory_evidence=trajectory_evidence, environment_evidence=environment_evidence, confidence=Confidence(0.95 if completion is not CompletionStatus.UNKNOWN else 0.4), @@ -363,11 +477,9 @@ def investigate( ) assert verification.verification is not None return FailureMiningResult( - revision_id=case.revision_id, - trace_id=case.trace_id, - trace_digest=case.trace_digest, + task=case.task, + context=case.context, verdict=MiningVerdict.CONFIRMED_FAILURE, - user_job=case.user_job, source_ids=tuple(source.id for source in case.sources), completion_checks=( CompletionCheck( @@ -379,6 +491,20 @@ def investigate( evidence=verification.artifacts, ), ), + failure_behavior=FailureBehavior( + primary=FailureBehaviorKind.UNRECOVERED_ACTION_FAILURE, + summary="The tool failure was not recovered before completion.", + observations=( + BehaviorObservation( + kind=FailureBehaviorKind.UNRECOVERED_ACTION_FAILURE, + phase=FailurePhase.RECOVERY, + first_observation_id=page.observations[0].record.id, + last_observation_id=None, + recovery_status=RecoveryStatus.NOT_RECOVERED, + evidence=page.artifacts, + ), + ), + ), trajectory_evidence=page.artifacts, environment_evidence=verification.artifacts, confidence=Confidence(0.9), @@ -490,11 +616,9 @@ def test_wrong_revision_or_corrupt_trace_is_invalid( def test_confirmed_failure_requires_trajectory_and_environment_evidence() -> None: with pytest.raises(ValueError, match="confirmed failure requires"): FailureMiningResult( - revision_id=HarnessRevisionId("revision-1"), - trace_id=TRACE_ID, - trace_digest=_digest("trace-1"), + task=_task(), + context=_context((ObservationId("observation-1"),)), verdict=MiningVerdict.CONFIRMED_FAILURE, - user_job="Close the customer ticket.", source_ids=(FailureSourceId("signal-1"),), completion_checks=( CompletionCheck( @@ -506,7 +630,29 @@ def test_confirmed_failure_requires_trajectory_and_environment_evidence() -> Non evidence=(), ), ), - trajectory_evidence=(), + failure_behavior=FailureBehavior( + primary=FailureBehaviorKind.FALSE_COMPLETION, + summary="The agent claimed success while the ticket stayed open.", + observations=( + BehaviorObservation( + kind=FailureBehaviorKind.FALSE_COMPLETION, + phase=FailurePhase.COMPLETION, + first_observation_id=ObservationId("observation-1"), + last_observation_id=None, + recovery_status=RecoveryStatus.NOT_RECOVERED, + evidence=( + _evidence( + EvidenceKind.TRAJECTORY, + "observation-1", + "observation-1", + ), + ), + ), + ), + ), + trajectory_evidence=( + _evidence(EvidenceKind.TRAJECTORY, "observation-1", "observation-1"), + ), environment_evidence=(), confidence=Confidence(0.9), unresolved_questions=(), @@ -554,3 +700,174 @@ def test_judge_must_read_the_full_trace_before_returning_verdict(tmp_path: Path) assert result.verdict is MiningVerdict.INVALID assert result.invalid_reason is MiningInvalidReason.JUDGE_OUTPUT + + +def test_confirmed_failure_requires_failure_behavior() -> None: + with pytest.raises(ValueError, match="confirmed failure requires"): + FailureMiningResult( + task=_task(), + context=_context((ObservationId("observation-1"),)), + verdict=MiningVerdict.CONFIRMED_FAILURE, + source_ids=(FailureSourceId("signal-1"),), + completion_checks=( + CompletionCheck( + check_id=CHECK_ID, + required_outcome="The ticket is closed.", + agent_claim="Ticket closed successfully.", + observed_state="Ticket remains open.", + status=CompletionStatus.NOT_COMPLETED, + evidence=(_evidence(EvidenceKind.ENVIRONMENT, "ticket-123", "state-1"),), + ), + ), + failure_behavior=None, + trajectory_evidence=( + _evidence(EvidenceKind.TRAJECTORY, "observation-1", "observation-1"), + ), + environment_evidence=( + _evidence(EvidenceKind.ENVIRONMENT, "ticket-123", "state-1"), + ), + confidence=Confidence(0.9), + unresolved_questions=(), + invalid_reason=None, + ) + + +def test_no_failure_rejects_failure_behavior() -> None: + with pytest.raises(ValueError, match="no-failure verdict"): + FailureMiningResult( + task=_task(), + context=_context((ObservationId("observation-1"),)), + verdict=MiningVerdict.NO_FAILURE, + source_ids=(FailureSourceId("signal-1"),), + completion_checks=( + CompletionCheck( + check_id=CHECK_ID, + required_outcome="The ticket is closed.", + agent_claim="Retry succeeded.", + observed_state="Ticket is closed.", + status=CompletionStatus.COMPLETED, + evidence=(_evidence(EvidenceKind.ENVIRONMENT, "ticket-123", "state-1"),), + ), + ), + failure_behavior=FailureBehavior( + primary=FailureBehaviorKind.UNRECOVERED_ACTION_FAILURE, + summary="A failure was incorrectly retained after recovery.", + observations=( + BehaviorObservation( + kind=FailureBehaviorKind.UNRECOVERED_ACTION_FAILURE, + phase=FailurePhase.RECOVERY, + first_observation_id=ObservationId("observation-1"), + last_observation_id=None, + recovery_status=RecoveryStatus.RECOVERED, + evidence=( + _evidence( + EvidenceKind.TRAJECTORY, + "observation-1", + "observation-1", + ), + ), + ), + ), + ), + trajectory_evidence=( + _evidence(EvidenceKind.TRAJECTORY, "observation-1", "observation-1"), + ), + environment_evidence=( + _evidence(EvidenceKind.ENVIRONMENT, "ticket-123", "state-1"), + ), + confidence=Confidence(0.9), + unresolved_questions=(), + invalid_reason=None, + ) + + +def test_failure_behavior_observation_must_belong_to_context() -> None: + with pytest.raises(ValueError, match="failure behavior observation"): + FailureMiningResult( + task=_task(), + context=_context((ObservationId("observation-1"),)), + verdict=MiningVerdict.CONFIRMED_FAILURE, + source_ids=(FailureSourceId("signal-1"),), + completion_checks=( + CompletionCheck( + check_id=CHECK_ID, + required_outcome="The ticket is closed.", + agent_claim="Ticket closed successfully.", + observed_state="Ticket remains open.", + status=CompletionStatus.NOT_COMPLETED, + evidence=(_evidence(EvidenceKind.ENVIRONMENT, "ticket-123", "state-1"),), + ), + ), + failure_behavior=FailureBehavior( + primary=FailureBehaviorKind.FALSE_COMPLETION, + summary="The agent claimed success while the ticket stayed open.", + observations=( + BehaviorObservation( + kind=FailureBehaviorKind.FALSE_COMPLETION, + phase=FailurePhase.COMPLETION, + first_observation_id=ObservationId("observation-2"), + last_observation_id=None, + recovery_status=RecoveryStatus.NOT_RECOVERED, + evidence=( + _evidence( + EvidenceKind.TRAJECTORY, + "observation-2", + "observation-2", + ), + ), + ), + ), + ), + trajectory_evidence=( + _evidence(EvidenceKind.TRAJECTORY, "observation-1", "observation-1"), + ), + environment_evidence=( + _evidence(EvidenceKind.ENVIRONMENT, "ticket-123", "state-1"), + ), + confidence=Confidence(0.9), + unresolved_questions=(), + invalid_reason=None, + ) + + +def test_search_focuses_natural_query_and_adapt_compares_signals(tmp_path: Path) -> None: + revision = _revision(tmp_path) + collection = _collection( + tmp_path, + revision, + ("x" * 1500 + " update failed after retry", "Ticket remains open"), + ) + nomination = _nomination(FailureSourceKind.DOWNSTREAM_FAILURE) + prior_signal = replace( + nomination.sources[0], + id=FailureSourceId("signal-2"), + trace_id=TraceId("trace-2"), + kind=FailureSourceKind.HUMAN_FEEDBACK, + ) + tools = MiningTools( + TraceMiningCase( + nomination.task, + _context(collection.traces[0].observation_ids), + nomination.sources, + ), + collection, + RecordedEnvironmentVerifier(CompletionStatus.UNKNOWN, None), + (*nomination.sources, prior_signal), + ) + + search = tools.search_trajectory( + TrajectorySearchRequest("completion failed retries", ObservationContentField.ANY, 5) + ) + adapted = tools.adapt( + AdaptationRequest( + (FailureSourceKind.DOWNSTREAM_FAILURE, FailureSourceKind.HUMAN_FEEDBACK), + 5, + ) + ) + + assert search.status is ToolStatus.OK + assert "update failed" in search.hits[0].excerpt + assert tuple(signal.id for signal in adapted.signals) == ( + FailureSourceId("signal-1"), + FailureSourceId("signal-2"), + ) From f320336d72f52a9e8460cebf5b7d187d50cf693c Mon Sep 17 00:00:00 2001 From: divo12 Date: Tue, 25 Aug 2026 14:57:31 +0530 Subject: [PATCH 12/18] expose failure mining to Codex over MCP --- pyproject.toml | 1 + skills/ofw-mine-failures/SKILL.md | 108 +++++++++ src/ofw/__init__.py | 2 + src/ofw/mcp.py | 351 ++++++++++++++++++++++++++++++ src/ofw/mine.py | 4 +- tests/test_mcp.py | 72 ++++++ tests/test_mine.py | 16 +- 7 files changed, 544 insertions(+), 10 deletions(-) create mode 100644 skills/ofw-mine-failures/SKILL.md create mode 100644 src/ofw/mcp.py create mode 100644 tests/test_mcp.py diff --git a/pyproject.toml b/pyproject.toml index 9d69239..c849d1b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ dependencies = [ "e2b>=2.38,<3", "httpx>=0.27,<1", "langfuse>=4.7,<5", + "mcp>=2,<3", "pydantic>=2.10,<3", ] diff --git a/skills/ofw-mine-failures/SKILL.md b/skills/ofw-mine-failures/SKILL.md new file mode 100644 index 0000000..78f89bc --- /dev/null +++ b/skills/ofw-mine-failures/SKILL.md @@ -0,0 +1,108 @@ +--- +name: ofw-mine-failures +description: Mines observable failures from executed Hermes trajectories through a connected OpenFlyWheel failure-mining MCP server. Use when asked to judge, triage, or mine failures from OFW/Langfuse traces. Do not use for root-cause diagnosis, clustering, eval generation, rubric refinement, or modifying Hermes. +--- + +# Mine Hermes failures with OpenFlyWheel + +Investigate one OFW mining case and return an evidence-grounded failure-mining +result. Hermes is the executed agent. You are the Codex operator using OFW. +Stay at observable behavior; diagnosis and improvement are separate work. + +## Principles + +1. **The oracle decides completion.** A Hermes claim is evidence of what it + claimed, not proof that the task succeeded. Verify every required outcome + against its declared environment source. +2. **Recovery matters.** A failed action is not a task failure when Hermes later + recovers and every required outcome is completed. +3. **Use only issued evidence.** Cite observation and environment evidence + returned by OFW tools. Never invent identifiers, digests, state, or tool + results. +4. **Read the complete trajectory.** Search to find relevant regions, then page + through `read_trajectory` until `next_cursor` is null. Keep only relevant + evidence in the result. +5. **Calibration cannot override state.** `adapt` returns human and production + signals from nominated trajectories. Use them to challenge an interpretation, + never to replace source-of-truth verification. +6. **No diagnosis.** Do not identify a root cause, responsible component, bad + prompt, broken tool, or proposed fix. Do not cluster failures, create evals, + or change a rubric. + +## Workflow + +1. Call `get_mining_case`. Read the task intent, constraints, required outcomes, + available Hermes tools, environment sources, observation IDs, and nominated + signals. +2. Search the current trajectory with `search_trajectory`: + - search task-specific entities and required outcomes; + - search completion claims such as `done`, `success`, or `completed`; + - search errors, failed actions, retries, cancellations, and verification; + - follow new questions raised by each useful hit with another focused search. +3. Page through the ordered trajectory with `read_trajectory`, beginning with a + null cursor and continuing with each returned `next_cursor` until it is null. + Track whether errors were retried, recovered, abandoned, or contradicted by a + later action. +4. Use `search_prior_trajectories` only when a similar run, prior disagreement, + or repeated signal would clarify the current observable behavior. Prior runs + do not prove the current outcome. +5. For every required outcome, call `verify_environment` with the exact + `source_id` and `check_id` returned by `get_mining_case`. +6. Call `adapt` for the relevant nominated signal kinds. Compare those signals + with the trajectory and verification result. Record disagreement as an + unresolved question; do not silently choose the signal you prefer. +7. Apply the verdict rules and return the result in the output shape below. + +## Verdict rules + +- `confirmed_failure`: at least one required outcome is `not_completed`, and a + concrete `FailureBehavior` is grounded in trajectory plus environment + evidence. +- `no_failure`: every required outcome is `completed`; `failure_behavior` must + be null, including when an intermediate action failed but recovery succeeded. +- `ambiguous`: completion cannot be established because required environment + state is unavailable or evidence materially conflicts; `failure_behavior` + must be null and `unresolved_questions` must explain the uncertainty. +- Never return `confirmed_failure` from a tool error alone. + +Use only these observable behavior categories: + +- `outcome_mismatch` +- `false_completion` +- `required_action_omitted` +- `forbidden_state_change` +- `unrecovered_action_failure` +- `no_progress_loop` +- `abandoned_before_completion` + +## Output + +Return one `FailureMiningResult`-shaped object containing: + +```text +task +context +verdict +source_ids +completion_checks +failure_behavior | null +trajectory_evidence +environment_evidence +confidence +unresolved_questions +invalid_reason | null +``` + +For a confirmed failure, each behavior observation must identify its behavior +kind, phase, first and optional last observation IDs, recovery status, and +trajectory evidence. Keep the summary factual and free of causal claims. + +## Decision examples + +- A command fails, Hermes retries successfully, and the oracle confirms the + required state: `no_failure`. +- Hermes says the work is complete, but the oracle shows the required state was + not reached: `confirmed_failure` with `false_completion` or + `outcome_mismatch`, depending on the observed behavior. +- Hermes appears to stop early, but the environment cannot be queried: + `ambiguous`, not an inferred failure. diff --git a/src/ofw/__init__.py b/src/ofw/__init__.py index 28d6548..679e740 100644 --- a/src/ofw/__init__.py +++ b/src/ofw/__init__.py @@ -27,6 +27,7 @@ WorkspaceFile, ) from ofw.harness import EditableFile, Harness, Subagent, Tool, editable +from ofw.mcp import FailureMiningMcpServer from ofw.mine import ( AdaptationRequest, AdaptationResult, @@ -184,6 +185,7 @@ def read_observation_content( "EvidenceRecordId", "EvidenceReference", "FailureMiningResult", + "FailureMiningMcpServer", "FailureMiningRun", "FailureBehavior", "FailureBehaviorKind", diff --git a/src/ofw/mcp.py b/src/ofw/mcp.py new file mode 100644 index 0000000..6b60462 --- /dev/null +++ b/src/ofw/mcp.py @@ -0,0 +1,351 @@ +"""MCP transport exposing a live OFW failure-mining case to Codex.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime + +from mcp.server import MCPServer +from pydantic import BaseModel, ConfigDict + +from ofw.mine import ( + AdaptationRequest, + CompletionStatus, + ConstraintKind, + EnvironmentCheckId, + EnvironmentCheckRequest, + EnvironmentSourceId, + EnvironmentSourceKind, + EvidenceKind, + EvidenceReference, + FailureSource, + FailureSourceKind, + MiningTools, + ToolAccess, + ToolAction, + ToolStatus, + TrajectoryPageRequest, + TrajectorySearchRequest, + TrajectorySearchResult, +) +from ofw.observability.langfuse.domain import ObservationContentField, ObservationId + + +class McpModel(BaseModel): + model_config = ConfigDict(frozen=True) + + +class McpEvidence(McpModel): + kind: EvidenceKind + record_id: str + digest: str + + +class McpRequiredOutcome(McpModel): + check_id: str + source_id: str + description: str + + +class McpConstraint(McpModel): + kind: ConstraintKind + description: str + + +class McpToolCapability(McpModel): + name: str + access: ToolAccess + + +class McpEnvironmentSource(McpModel): + id: str + kind: EnvironmentSourceKind + summary: str + + +class McpFailureSignal(McpModel): + id: str + kind: FailureSourceKind + trace_id: str + observed_at: datetime + summary: str + evidence: tuple[McpEvidence, ...] + + +class McpMiningCase(McpModel): + task_id: str + intent: str + required_outcomes: tuple[McpRequiredOutcome, ...] + constraints: tuple[McpConstraint, ...] + revision_id: str + trace_id: str + trace_digest: str + observation_ids: tuple[str, ...] + session_id: str | None + environment_name: str | None + release: str | None + available_tools: tuple[McpToolCapability, ...] + environment_sources: tuple[McpEnvironmentSource, ...] + initial_state_evidence: tuple[McpEvidence, ...] + failure_signals: tuple[McpFailureSignal, ...] + + +class McpSearchHit(McpModel): + observation_id: str + trace_id: str | None + field: ObservationContentField + excerpt: str + evidence: McpEvidence + + +class McpSearchResult(McpModel): + status: ToolStatus + summary: str + hits: tuple[McpSearchHit, ...] + next_actions: tuple[ToolAction, ...] + + +class McpTrajectoryObservation(McpModel): + id: str + parent_id: str | None + name: str | None + type: str + start_time: datetime + status_message: str | None + input: str | None + output: str | None + evidence: McpEvidence + + +class McpTrajectoryPage(McpModel): + status: ToolStatus + summary: str + observations: tuple[McpTrajectoryObservation, ...] + next_cursor: str | None + next_actions: tuple[ToolAction, ...] + + +class McpEnvironmentVerification(McpModel): + tool_status: ToolStatus + summary: str + completion_status: CompletionStatus | None + observed_state: str | None + evidence: tuple[McpEvidence, ...] + next_actions: tuple[ToolAction, ...] + + +class McpAdaptationResult(McpModel): + status: ToolStatus + summary: str + signals: tuple[McpFailureSignal, ...] + next_actions: tuple[ToolAction, ...] + + +@dataclass(slots=True) +class FailureMiningMcpServer: + """A local, read-only MCP server backed by one live mining case.""" + + tools: MiningTools + server: MCPServer[None] = field(init=False) + + def __post_init__(self) -> None: + server: MCPServer[None] = MCPServer( + name="openflywheel-failure-mining", + instructions=( + "Inspect and verify one executed Hermes trajectory. Do not diagnose causes, " + "propose fixes, cluster failures, generate evals, or mutate rubrics." + ), + ) + server.tool(structured_output=True)(self.get_mining_case) + server.tool(structured_output=True)(self.search_trajectory) + server.tool(structured_output=True)(self.search_prior_trajectories) + server.tool(structured_output=True)(self.read_trajectory) + server.tool(structured_output=True)(self.verify_environment) + server.tool(structured_output=True)(self.adapt) + self.server = server + + def get_mining_case(self) -> McpMiningCase: + """Return the task, context, signals, environment sources, and required outcomes.""" + case = self.tools.case + return McpMiningCase( + task_id=case.task.id.value, + intent=case.task.intent, + required_outcomes=tuple( + McpRequiredOutcome( + check_id=item.check_id.value, + source_id=item.source_id.value, + description=item.description, + ) + for item in case.task.required_outcomes + ), + constraints=tuple( + McpConstraint(kind=item.kind, description=item.description) + for item in case.task.constraints + ), + revision_id=case.context.revision_id.value, + trace_id=case.context.trace_id.value, + trace_digest=case.context.trace_digest.value, + observation_ids=tuple(item.value for item in case.context.observation_ids), + session_id=case.context.session_id, + environment_name=case.context.environment_name, + release=case.context.release, + available_tools=tuple( + McpToolCapability(name=item.name.value, access=item.access) + for item in case.context.available_tools + ), + environment_sources=tuple( + McpEnvironmentSource(id=item.id.value, kind=item.kind, summary=item.summary) + for item in case.context.environment_sources + ), + initial_state_evidence=_evidence(case.context.initial_state_evidence), + failure_signals=tuple(_signal(item) for item in case.sources), + ) + + def search_trajectory( + self, + text: str, + field: ObservationContentField = ObservationContentField.ANY, + limit: int = 10, + ) -> McpSearchResult: + """Search the current full trajectory for focused evidence.""" + return _search_result( + self.tools.search_trajectory(TrajectorySearchRequest(text, field, limit)) + ) + + def search_prior_trajectories( + self, + text: str, + field: ObservationContentField = ObservationContentField.ANY, + limit: int = 10, + ) -> McpSearchResult: + """Search other trajectories in the collection for comparable evidence.""" + return _search_result( + self.tools.search_prior_trajectories( + TrajectorySearchRequest(text, field, limit) + ) + ) + + def read_trajectory( + self, + cursor: str | None = None, + limit: int = 50, + ) -> McpTrajectoryPage: + """Read an ordered page; continue until next_cursor is null.""" + result = self.tools.read_trajectory( + TrajectoryPageRequest( + None if cursor is None else ObservationId(cursor), + limit, + ) + ) + return McpTrajectoryPage( + status=result.status, + summary=result.summary, + observations=tuple( + McpTrajectoryObservation( + id=item.record.id.value, + parent_id=( + None + if item.record.parent_observation_id is None + else item.record.parent_observation_id.value + ), + name=item.record.name, + type=item.record.type.value, + start_time=item.record.start_time, + status_message=item.record.status_message, + input=None if item.input_content is None else item.input_content.text, + output=None if item.output_content is None else item.output_content.text, + evidence=McpEvidence( + kind=EvidenceKind.TRAJECTORY, + record_id=item.record.id.value, + digest=item.record.digest.value, + ), + ) + for item in result.observations + ), + next_cursor=None if result.next_cursor is None else result.next_cursor.value, + next_actions=result.next_actions, + ) + + def verify_environment( + self, + source_id: str, + check_id: str, + ) -> McpEnvironmentVerification: + """Verify a declared required outcome against its source-of-truth environment.""" + result = self.tools.verify_environment( + EnvironmentCheckRequest( + EnvironmentSourceId(source_id), + EnvironmentCheckId(check_id), + ) + ) + verification = result.verification + return McpEnvironmentVerification( + tool_status=result.status, + summary=result.summary, + completion_status=None if verification is None else verification.status, + observed_state=None if verification is None else verification.observed_state, + evidence=_evidence(result.artifacts), + next_actions=result.next_actions, + ) + + def adapt( + self, + kinds: tuple[FailureSourceKind, ...], + limit: int = 20, + ) -> McpAdaptationResult: + """Read human and production calibration signals without changing a rubric.""" + result = self.tools.adapt(AdaptationRequest(kinds, limit)) + return McpAdaptationResult( + status=result.status, + summary=result.summary, + signals=tuple(_signal(item) for item in result.signals), + next_actions=result.next_actions, + ) + + def run_stdio(self) -> None: + """Serve this live case to a local Codex process over standard I/O.""" + self.server.run() + + +def _evidence(items: tuple[EvidenceReference, ...]) -> tuple[McpEvidence, ...]: + return tuple( + McpEvidence( + kind=item.kind, + record_id=item.record_id.value, + digest=item.digest.value, + ) + for item in items + ) + + +def _signal(item: FailureSource) -> McpFailureSignal: + return McpFailureSignal( + id=item.id.value, + kind=item.kind, + trace_id=item.trace_id.value, + observed_at=item.observed_at, + summary=item.summary, + evidence=_evidence(item.evidence), + ) + + +def _search_result(result: TrajectorySearchResult) -> McpSearchResult: + return McpSearchResult( + status=result.status, + summary=result.summary, + hits=tuple( + McpSearchHit( + observation_id=hit.observation_id.value, + trace_id=None if hit.trace_id is None else hit.trace_id.value, + field=hit.field, + excerpt=hit.excerpt, + evidence=McpEvidence( + kind=EvidenceKind.TRAJECTORY, + record_id=hit.observation_id.value, + digest=hit.reference.digest.value, + ), + ) + for hit in result.hits + ), + next_actions=result.next_actions, + ) diff --git a/src/ofw/mine.py b/src/ofw/mine.py index e58c10c..ef25f33 100644 --- a/src/ofw/mine.py +++ b/src/ofw/mine.py @@ -560,7 +560,7 @@ def verify( ) -> EnvironmentVerification: ... -class HermesJudge(Protocol): +class FailureJudge(Protocol): def investigate( self, case: TraceMiningCase, @@ -776,7 +776,7 @@ class Mine: revision: HarnessRevision collection: CollectionResult nominations: tuple[MiningNomination, ...] - judge: HermesJudge + judge: FailureJudge environment: EnvironmentVerifier def __post_init__(self) -> None: diff --git a/tests/test_mcp.py b/tests/test_mcp.py new file mode 100644 index 0000000..3a02b0b --- /dev/null +++ b/tests/test_mcp.py @@ -0,0 +1,72 @@ +"""Codex-facing MCP transport for failure mining.""" + +from __future__ import annotations + +import asyncio +from pathlib import Path + +from mcp import Client + +from ofw.mcp import FailureMiningMcpServer +from ofw.mine import ( + CompletionStatus, + FailureSourceKind, + MiningTools, + TraceMiningCase, +) +from test_mine import ( + RecordedEnvironmentVerifier, + _collection, + _context, + _nomination, + _revision, +) + + +def test_codex_can_discover_and_call_failure_mining_tools(tmp_path: Path) -> None: + revision = _revision(tmp_path) + collection = _collection( + tmp_path, + revision, + ("Close ticket", "update failed", "Ticket remains open"), + ) + nomination = _nomination(FailureSourceKind.DOWNSTREAM_FAILURE) + tools = MiningTools( + TraceMiningCase( + nomination.task, + _context(collection.traces[0].observation_ids), + nomination.sources, + ), + collection, + RecordedEnvironmentVerifier( + CompletionStatus.NOT_COMPLETED, + "Ticket remains open", + ), + nomination.sources, + ) + server = FailureMiningMcpServer(tools) + + async def exercise() -> None: + async with Client(server.server) as client: + discovered = await client.list_tools() + assert {tool.name for tool in discovered.tools} == { + "adapt", + "get_mining_case", + "read_trajectory", + "search_prior_trajectories", + "search_trajectory", + "verify_environment", + } + result = await client.call_tool("get_mining_case") + assert result.is_error is False + + asyncio.run(exercise()) + assert server.get_mining_case().task_id == "close-ticket" + assert server.search_trajectory("failed", limit=5).status.value == "ok" + assert server.search_prior_trajectories("failed", limit=5).status.value == "not_found" + assert len(server.read_trajectory(limit=2).observations) == 2 + assert ( + server.verify_environment("itsm-production", "ticket-closed").completion_status + is CompletionStatus.NOT_COMPLETED + ) + assert server.adapt((FailureSourceKind.DOWNSTREAM_FAILURE,), 5).status.value == "ok" diff --git a/tests/test_mine.py b/tests/test_mine.py index c98e1f5..a87d4c0 100644 --- a/tests/test_mine.py +++ b/tests/test_mine.py @@ -313,7 +313,7 @@ def verify( @dataclass(frozen=True, slots=True) -class FakeHermesJudge: +class FakeFailureJudge: search_text: str def investigate( @@ -446,8 +446,8 @@ def investigate( @dataclass(frozen=True, slots=True) -class ForgingHermesJudge: - delegate: FakeHermesJudge +class ForgingFailureJudge: + delegate: FakeFailureJudge def investigate( self, @@ -465,7 +465,7 @@ def investigate( @dataclass(frozen=True, slots=True) -class PartialTraceHermesJudge: +class PartialTraceFailureJudge: def investigate( self, case: TraceMiningCase, @@ -564,7 +564,7 @@ def test_mine_uses_complete_trajectory_and_verified_state( revision=revision, collection=_collection(tmp_path, revision, outputs), nominations=(_nomination(source_kind),), - judge=FakeHermesJudge(search), + judge=FakeFailureJudge(search), environment=RecordedEnvironmentVerifier(state, observed), ).run() @@ -605,7 +605,7 @@ def test_wrong_revision_or_corrupt_trace_is_invalid( revision=revision, collection=collection, nominations=(_nomination(FailureSourceKind.AGENT_ERROR),), - judge=FakeHermesJudge("successfully"), + judge=FakeFailureJudge("successfully"), environment=RecordedEnvironmentVerifier(CompletionStatus.COMPLETED, "closed"), ).run().results[0] @@ -670,7 +670,7 @@ def test_judge_cannot_invent_evidence_that_no_tool_returned(tmp_path: Path) -> N ("Close ticket", "update failed", "Ticket closed successfully"), ), nominations=(_nomination(FailureSourceKind.DOWNSTREAM_FAILURE),), - judge=ForgingHermesJudge(FakeHermesJudge("failed")), + judge=ForgingFailureJudge(FakeFailureJudge("failed")), environment=RecordedEnvironmentVerifier( CompletionStatus.NOT_COMPLETED, "Ticket remains open", @@ -691,7 +691,7 @@ def test_judge_must_read_the_full_trace_before_returning_verdict(tmp_path: Path) ("Close ticket", "update failed", "Ticket closed successfully"), ), nominations=(_nomination(FailureSourceKind.DOWNSTREAM_FAILURE),), - judge=PartialTraceHermesJudge(), + judge=PartialTraceFailureJudge(), environment=RecordedEnvironmentVerifier( CompletionStatus.NOT_COMPLETED, "Ticket remains open", From 2cf4ea9925ef478b4b9f2d4bb0e48a9e1f35b595 Mon Sep 17 00:00:00 2001 From: divo12 Date: Tue, 25 Aug 2026 15:03:40 +0530 Subject: [PATCH 13/18] package OpenFlyWheel Codex plugin --- .../openflywheel/.codex-plugin/plugin.json | 22 +++++ .../skills/integrate-ofw/SKILL.md | 82 +++++++++++++++++++ .../integrate-ofw/references/python-api.md | 68 +++++++++++++++ .../skills}/ofw-mine-failures/SKILL.md | 0 4 files changed, 172 insertions(+) create mode 100644 plugins/openflywheel/.codex-plugin/plugin.json create mode 100644 plugins/openflywheel/skills/integrate-ofw/SKILL.md create mode 100644 plugins/openflywheel/skills/integrate-ofw/references/python-api.md rename {skills => plugins/openflywheel/skills}/ofw-mine-failures/SKILL.md (100%) diff --git a/plugins/openflywheel/.codex-plugin/plugin.json b/plugins/openflywheel/.codex-plugin/plugin.json new file mode 100644 index 0000000..fc76566 --- /dev/null +++ b/plugins/openflywheel/.codex-plugin/plugin.json @@ -0,0 +1,22 @@ +{ + "name": "openflywheel", + "version": "0.1.0", + "description": "Integrate Hermes with OpenFlyWheel and mine evidence-backed failures with Codex.", + "author": { + "name": "OpenFlyWheel" + }, + "repository": "https://github.com/divo12/OpenFlyWheel", + "skills": "./skills/", + "interface": { + "displayName": "OpenFlyWheel", + "shortDescription": "Integrate Hermes and mine its failures", + "longDescription": "Codex workflows for integrating a Hermes codebase with OpenFlyWheel and mining observable failures from full Langfuse trajectories.", + "developerName": "OpenFlyWheel", + "category": "Developer Tools", + "capabilities": ["Read", "Write"], + "defaultPrompt": [ + "Integrate this Hermes codebase with OpenFlyWheel.", + "Mine failures from the connected OpenFlyWheel trajectory." + ] + } +} diff --git a/plugins/openflywheel/skills/integrate-ofw/SKILL.md b/plugins/openflywheel/skills/integrate-ofw/SKILL.md new file mode 100644 index 0000000..fc1b4f5 --- /dev/null +++ b/plugins/openflywheel/skills/integrate-ofw/SKILL.md @@ -0,0 +1,82 @@ +--- +name: integrate-ofw +description: Integrates a Hermes codebase with OpenFlyWheel by declaring its versioned harness components, connecting its existing Langfuse project, attributing real runs to an immutable revision, and verifying full trace collection. Use for onboarding or adding OFW to Hermes. Do not use for failure mining, diagnosis, harness optimization, or replacing the application's observability backend. +--- + +# Integrate Hermes with OpenFlyWheel + +Add OFW to the real Hermes execution path without changing Hermes behavior. +Read [references/python-api.md](references/python-api.md) before writing OFW +code; it contains the exact supported API. + +## Principles + +1. **Instrument the real path.** Find the command, service, or worker that + actually starts Hermes. Do not create a parallel demonstration agent. +2. **Discover components for the user.** Locate the active system prompt, tool + implementations, skills, subagent definitions, and middleware. The user + should not have to translate their repository into OFW schemas manually. +3. **Do not change behavior.** This workflow may add an OFW declaration and + revision attribution. It must not rewrite prompts, tools, skills, or Hermes + control flow. +4. **Keep secrets in the environment.** Never place Langfuse keys in code, + manifests, plugin files, logs, or chat. OFW records only environment-variable + names. +5. **Preserve Langfuse.** Connect the application's existing Langfuse project; + do not replace its SDK, exporter, or trace hierarchy. +6. **Verify with a real run.** Integration is complete only when one real Hermes + trajectory is attributed to the generated revision and collected with full + input/output content into local SQLite. + +## Workflow + +1. Inspect the repository and its dependency manager. Confirm that + `openflywheel` is available from the user's chosen package source. If it is + not resolvable, stop and ask for the intended install source; do not invent a + Git URL or published version. +2. Identify the git root and the active Hermes assets: + - one or more prompt files; + - named tool implementation files; + - skill files; + - subagent definitions, if present; + - middleware or lifecycle files, if present. +3. Add one small `ofw_harness.py` declaration at the git root. Follow the + reference exactly. Register every discovered active asset once. +4. Default only the primary prompt to `ofw.editable`. Keep tools, skills, + subagents, and middleware frozen unless the user explicitly authorizes OFW + to optimize them later. +5. Connect `LangfuseProject.from_env` using the deployment's real environment + name and existing Langfuse environment variables. Never read or copy their + values. +6. Process the harness to produce an immutable revision and manifest. +7. Add revision attribution around the real Hermes run using + `propagate_attributes` and metadata key `ofw.harness.revision`. Do not add a + second root trace. +8. Run one real Hermes request, flush the existing Langfuse client as required + by its runtime, then collect a UTC window containing that run with + `ofw.collect`. +9. Verify that the collection belongs to the revision, contains the expected + trace and ordered observations, has no completeness gaps, and that captured + input/output content can be read from the SQLite snapshot. + +## Stop conditions + +- Stop before editing when the active Hermes entry point or prompt cannot be + identified confidently. +- Stop before a live run when credentials are unavailable; report the exact + environment-variable names needed without exposing values. +- Do not report success from a generated manifest alone. +- Do not weaken full-trace capture, redact content, or silently accept missing + revision attribution. + +## Output + +Report: + +- the OFW declaration and execution path changed; +- the discovered component-to-file mapping and which files are editable; +- the immutable revision ID and manifest path; +- the real Hermes command/request used for verification; +- the collected trace ID, observation count, capability/gap status, and SQLite + path; +- any missing component, credential, attribution, or content evidence. diff --git a/plugins/openflywheel/skills/integrate-ofw/references/python-api.md b/plugins/openflywheel/skills/integrate-ofw/references/python-api.md new file mode 100644 index 0000000..84f5acb --- /dev/null +++ b/plugins/openflywheel/skills/integrate-ofw/references/python-api.md @@ -0,0 +1,68 @@ +# Supported Python integration + +Use only this API surface. Adapt paths and names to the inspected Hermes +repository; do not invent components that are not active. + +```python +from datetime import UTC, datetime, timedelta +from pathlib import Path + +from ofw import Harness, LangfuseProject, Tool, TraceWindow, ofw, propagate_attributes + +ROOT = Path(__file__).resolve().parent + +project = LangfuseProject.from_env(environment="production") +harness = Harness("hermes", root=ROOT) +harness.connect_prompt(ofw.editable(Path("path/to/system-prompt.md"))) +harness.connect_tools( + Tool(name="search", source=Path("path/to/search-tool.py")), +) +harness.connect_skills(Path("path/to/skill/SKILL.md")) +harness.connect_middleware(Path("path/to/middleware.py")) +harness.connect_observability(project) +revision = harness.process() +``` + +Register only component kinds that exist. For named subagents use +`Subagent(name=..., source=...)` with `connect_subagents`. Tool and subagent +names must be lowercase identifiers accepted by OFW. Every registered path is +relative to the git root and may belong to only one component. + +Run the real Hermes entry point inside revision attribution: + +```python +with propagate_attributes( + metadata={"ofw.harness.revision": str(revision.id)}, +): + run_real_hermes_request() +``` + +The surrounding application remains responsible for its existing Langfuse +client lifecycle and flush behavior. + +Collect the verified run after Langfuse ingestion: + +```python +end = datetime.now(UTC) +collection = ofw.collect( + revision, + window=TraceWindow(end - timedelta(minutes=30), end), +) +``` + +The default snapshot path is `/.ofw/collection.sqlite`. A ready +integration has an exactly attributed trace, ordered observations, captured +input/output content, and no trace gaps. Use OFW's public +`search_observation_content`, `read_trace_observations`, and +`read_observation_content` helpers to prove content is queryable. + +Required environment variables normally remain: + +```text +LANGFUSE_PUBLIC_KEY +LANGFUSE_SECRET_KEY +LANGFUSE_BASE_URL +``` + +`LangfuseProject.from_env` accepts alternate environment-variable names when +the deployment already uses them. Pass the names, never their secret values. diff --git a/skills/ofw-mine-failures/SKILL.md b/plugins/openflywheel/skills/ofw-mine-failures/SKILL.md similarity index 100% rename from skills/ofw-mine-failures/SKILL.md rename to plugins/openflywheel/skills/ofw-mine-failures/SKILL.md From 6e48d41155f85a69ae36e4be85bb4f851773c262 Mon Sep 17 00:00:00 2001 From: divo12 Date: Wed, 26 Aug 2026 07:12:11 +0530 Subject: [PATCH 14/18] generalize OFW to any Langfuse harness --- .../2026-08-25-ofw-full-sdk-blueprint.md | 951 ++++++++++++++++++ .../openflywheel/.codex-plugin/plugin.json | 8 +- .../skills/integrate-ofw/SKILL.md | 22 +- .../integrate-ofw/references/python-api.md | 8 +- .../skills/ofw-mine-failures/SKILL.md | 18 +- src/ofw/mcp.py | 2 +- 6 files changed, 980 insertions(+), 29 deletions(-) create mode 100644 docs/plans/2026-08-25-ofw-full-sdk-blueprint.md diff --git a/docs/plans/2026-08-25-ofw-full-sdk-blueprint.md b/docs/plans/2026-08-25-ofw-full-sdk-blueprint.md new file mode 100644 index 0000000..f40e5f8 --- /dev/null +++ b/docs/plans/2026-08-25-ofw-full-sdk-blueprint.md @@ -0,0 +1,951 @@ +# OpenFlyWheel full SDK blueprint + +Status: proposed +Date: 2026-08-25 +Base: merge PR #5 into `fresh`, then execute PR6–PR16 in dependency order +Executed system: any agent harness connected to Langfuse +Reference integration and test harness: Hermes +OFW operator/reasoning agent: Codex +Outsourced subsystem: Langfuse trace instrumentation, ingestion, storage, and trace UI + +## 1. Objective + +Build OpenFlyWheel as the complete local-first SDK for turning production agent-harness +experience into evidence-backed evaluation and gated harness improvement, while +using Langfuse instead of building an OFW trace collector. + +OFW owns: + +- immutable harness definitions and revisions; +- production-signal semantics; +- task, context, behavior, and environment-verification contracts; +- Codex-facing query, judge, diagnosis, eval, and optimization workflows; +- failure records, diagnoses, clusters, and living eval suites; +- reproducible workspaces and experiment execution; +- candidate generation, gates, promotion, rollback, and pins; +- durable lineage, audit history, resumability, CLI/MCP surfaces, and Codex skills. + +Langfuse owns: + +- client instrumentation and framework adapters; +- trace/span ingestion; +- raw trace persistence and trace UI; +- transport-level buffering, batching, and delivery. + +OFW imports immutable, attributed Langfuse windows into local SQLite. It does +not proxy model calls, export OTLP, or become another observability backend. + +## 1.1 Research basis + +This blueprint derives the product shape from NeoSigma's public +[skills repository](https://github.com/neosigmaai/skills), especially +`integrate-sdk` and `import-verifiers`, plus its posts on +[production evals](https://neosigma.ai/blog/the-most-important-eval-isnt-on-a-leaderboard), +[self-improving systems](https://neosigma.ai/blog/self-improving-agentic-systems), +[agent workspaces](https://neosigma.ai/blog/agent-workspaces), and +[model–harness co-design](https://neosigma.ai/blog/investigating-the-optimal-harness-for-a-model). +Auto Harness is intentionally excluded; it is a minimal user-operated example, +not evidence for NeoSigma's product SDK architecture. + +## 2. Product boundary + +```text +Any production agent harness + └─ Langfuse SDK/exporter + └─ complete traces, tool calls, scores, feedback + └─ OFW Langfuse connector → immutable local snapshot + +Codex + OpenFlyWheel plugin + ├─ Search / Read / Verify / Adapt + ├─ Mine observable failures + ├─ Diagnose confirmed failures + ├─ Build and maintain eval cases + ├─ Propose bounded harness changes + └─ Interpret experiment evidence + +OFW deterministic services + ├─ contracts and lineage + ├─ state store + ├─ environment oracles and verification + ├─ workspace reset and execution + ├─ regression/frontier gates + ├─ keep/revert/pin + └─ budgets, stop conditions, and audit log +``` + +The agent never decides whether its own change ships. Codex proposes and +investigates; typed OFW services reproduce, verify, gate, and record. + +## 3. Target SDK experience + +```python +project = ofw.Project( + harness=ofw.Harness(...), + traces=ofw.LangfuseProject.from_env(environment="production"), + state_path=Path(".ofw/ofw.sqlite"), +) + +revision = project.process() +collection = project.collect(window=window) +mining_run = project.mine(collection, signals=signal_sources) +diagnosis_run = project.diagnose(mining_run.confirmed_failures) +suite_revision = project.build_evals(diagnosis_run) +candidate = project.fit(suite_revision, budget=budget) +decision = project.gate(candidate) +project.pin(decision) # succeeds only for an admitted candidate +``` + +The exact facade may change during implementation. The domain objects and +authority boundaries below may not. + +## 4. Invariants + +1. The connected agent harness is the executed system; Codex is the only reasoning + agent using OFW. Hermes is the reference integration, never a domain assumption. +2. Langfuse content is read-only. OFW never edits or deletes source traces. +3. Every admitted trace references one immutable harness revision. +4. An executed agent's completion claim is never an oracle. +5. Stateful success requires read-only environment evidence or a durable + recorded equivalent. +6. Intermediate action failure is not task failure after verified recovery. +7. Mining observes behavior; diagnosis explains cause; eval building reproduces + the failure; Fit proposes changes. These remain separate stages and types. +8. Agent outputs are proposals until deterministic validation admits them. +9. Eval holdouts and test traces remain invisible to Codex candidate generation. +10. Every state transition is append-only, content-addressed, and attributable + to input revisions, evidence, agent session, and code commit. +11. No `Any`, untyped metadata dictionaries, dynamic `getattr`/`setattr`, or + stringly categorical state in public Python contracts. +12. Failed or interrupted work can resume without replaying completed external + operations. + +## 5. Target package layout + +```text +src/ofw/ + contracts.py existing revision primitives + harness.py existing component registry + runtime.py existing canary/runtime boundary + project.py final public facade + state/ + store.py OFW-owned SQLite state + migrations/ + observability/langfuse/ existing outsourced-collector connector + signals.py production/human signal normalization + verification.py environment/oracle/test contracts + mining/ task/context/behavior + Codex judge sessions + diagnosis/ cause and improvement-hypothesis evidence + evals/ cases, suites, reproduction, lifecycle + clusters.py recurring failure-mode registry + workspace/ resettable E2B execution + experiments.py champion/candidate executions + gate.py suite and promotion policies + fit.py bounded Codex harness optimizer + loop.py resumable flywheel orchestration + mcp/ Codex tool servers over the same services +plugins/openflywheel/ focused Codex skills +``` + +Do not perform a directory-only refactor. Move code only when a PR introduces +the owning service and preserves public imports. + +## 6. Dependency graph + +```text +PR5 evidence-backed mining (current) + ├─ PR6 durable state + production signals ─┐ + └─ PR7 verification SDK + importer ────────┤ + ▼ + PR8 executable Codex judge runtime + ▼ + PR9 failure diagnosis + ▼ + PR10 eval case reproduction + ┌───────┴────────┐ + ▼ ▼ + PR11 cluster/eval registry PR12 workspaces/experiments + └───────┬────────┘ + ▼ + PR13 gates + suite transitions + ▼ + PR14 bounded Fit optimizer + ▼ + PR15 resumable flywheel loop + ▼ + PR16 SDK/plugin hardening +``` + +Parallel lanes: + +- PR6 and PR7 may run in parallel after PR5 merges; they own disjoint modules. +- PR11 and PR12 may run in parallel after PR10; one owns intelligence state, + the other owns execution. +- All other steps are serial because they consume contracts from the preceding + step. + +## 7. PR6 — Durable OFW state and production signals + +Branch: `codex/ofw-state-signals` +Depends on: PR5 +Risk: medium; first OFW-owned persistent schema + +### Context brief + +Langfuse is the source of raw traces and scores, but OFW needs durable state for +its own derived objects. The existing collection SQLite is a source snapshot, +not the flywheel database. Create a separate `.ofw/ofw.sqlite` with explicit +schema versions and append-only lineage. + +Normalize feedback and outcomes without building an OFW event collector. +Initial adapters read Langfuse scores and accept typed caller-provided records. + +### Contracts + +- `OfwStateVersion` +- `SignalId`, `SignalKind`, `SignalSubject` +- `ProductionSignal` +- `SignalSource` protocol +- `LangfuseScoreSignalSource` +- `StateRecordDigest` + +Signal kinds initially cover human feedback, user correction, trusted score, +downstream failure, incident, rollback, reopened work, environment mismatch, +and agent error. Do not add generic string event names to the domain layer. + +### Work + +1. Add OFW state migrations and a version-checking SQLite store. +2. Persist source provenance, trace/revision identity, timestamps, evidence + references, and canonical digests. +3. Import Langfuse scores without mutating the collection snapshot. +4. Make ingestion idempotent by signal identity and content digest. +5. Add query methods by revision, trace, kind, and time window. +6. Move `FailureSource` toward the shared signal contract without breaking the + PR5 public API. + +### Verification + +```bash +uv run pytest tests/test_state.py tests/test_signals.py -q +uv run pytest -q +uv run ruff check src tests +uv run mypy src tests +``` + +Test duplicate ingestion, conflicting identity, wrong revision, corrupt digest, +restart, migration mismatch, and preservation of source evidence. + +### Exit criteria + +The same Langfuse window can be imported repeatedly with one durable signal +record per source fact, and every record resolves to a collected trace and +revision. + +### Rollback + +Revert the code while retaining the new SQLite file. Never downgrade or delete +derived records automatically; later code may ignore unsupported schema +versions. + +## 8. PR7 — Verification SDK and verifier-import skill + +Branch: `codex/verification-contracts` +Depends on: PR5 +Parallel with: PR6 +Risk: high; this defines what “task completed” means + +### Context brief + +PR5 has `EnvironmentSource`, `RequiredOutcome`, and an `EnvironmentVerifier` +protocol. Consolidate them into the language-agnostic contract previously +chosen by the user: + +```text +environment +oracle +verification tests +``` + +The environment owns reproducible state. The oracle defines the success fact. +Verification tests observe that fact. Test implementation language is an edge +adapter detail, never a domain category. + +### Contracts + +- `EnvironmentContract` +- `OracleContract` +- `VerificationTest` +- `VerificationPlan` +- `VerificationAttempt` +- `VerificationEvidence` +- `VerificationStatus` +- `VerificationAdapter` protocol + +Initial adapters may wrap recorded state and deterministic commands already +supported by `runtime.py`. External APIs, databases, GitHub, and ticket systems +remain caller-provided adapters until a real integration is requested. + +### Work + +1. Add I/O-free verification domain objects and canonical digests. +2. Adapt PR5 environment verification to the new plan without compatibility + shims that duplicate truth. +3. Require read-only operation for mining/judging verification. +4. Record observed version/time and freshness for mutable sources. +5. Add `import-ofw-verifiers` to the Codex plugin. It scans existing tests and + produces typed plans so users do not write schemas manually. +6. Explicitly skip mechanical checks that do not express agent outcomes and + stateful checks for which no safe oracle adapter exists. + +### Verification + +```bash +uv run pytest tests/test_verification.py tests/test_runtime.py -q +python3 ~/.codex/skills/.system/plugin-creator/scripts/validate_plugin.py plugins/openflywheel +uv run pytest -q +uv run ruff check src tests +uv run mypy src tests +``` + +Test unsupported oracle, wrong environment version, stale state, mutating +adapter rejection, language-independent command tests, and unavailable state. + +### Exit criteria + +A user repository can be inspected by Codex and represented as an environment, +oracle, and verification-test plan without hand-authoring OFW objects; a known +state yields a reproducible typed verdict. + +### Rollback + +Keep PR5 verification operational until this PR's migration tests pass. Remove +old contracts only in the same PR after every caller moves. + +## 9. PR8 — Executable Codex judge runtime + +Branch: `codex/judge-runtime` +Depends on: PR6, PR7 +Risk: high; first automated Codex-to-OFW control loop + +### Context brief + +PR5 exposes one live mining case through MCP, but OFW cannot yet run a complete +Codex judge session or accept its result through a tool. Build the production +runtime that turns the existing `ofw-mine-failures` skill into an executable, +auditable workflow. + +### Contracts + +- `JudgeSessionId`, `JudgeSessionState` +- `JudgeBudget`, `JudgeAttempt` +- `FailureJudge` protocol +- `CodexFailureJudge` +- `MiningSubmission` + +### Work + +1. Persist a mining session before launching Codex. +2. Serve multiple nominated cases through MCP with stable case IDs. +3. Add `submit_failure_result`; validate the typed PR5 result and tool-issued + evidence before admission. +4. Add a Codex CLI/SDK adapter with explicit timeout, token/tool-call budget, + working directory, plugin/skill selection, and captured session ID. +5. Resume sessions from OFW state after process interruption. +6. Fail closed when Codex returns text without a valid submission. +7. Keep mining read-only: no repository-edit or agent-execution tools. + +### Verification + +```bash +uv run pytest tests/test_judge_runtime.py tests/test_mcp.py tests/test_mine.py -q +uv run pytest -q +uv run ruff check src tests +uv run mypy src tests +``` + +Run an integration smoke over the six existing heterogeneous traces. Require +admitted results for two known successes and four known failures; record +precision errors instead of weakening validators. + +### Exit criteria + +`project.mine(...)` can launch Codex, expose bounded evidence tools, accept one +typed result per case, persist the session, and resume safely. + +### Rollback + +Disable the Codex adapter and retain manual/in-process `FailureJudge` support. +Persisted sessions remain inspectable but cannot auto-resume on unsupported +runtime versions. + +## 10. PR9 — Failure diagnosis + +Branch: `codex/failure-diagnosis` +Depends on: PR8 +Risk: high; must separate evidence from speculation + +### Context brief + +Mining answers whether and where the executed agent failed. Diagnosis is the next stage and +may inspect the connected harness repository. It must not mutate files or generate +evals. + +### Contracts + +- `DiagnosisId` +- `FailureMechanism` +- `FailureLocation` +- `CausalEvidence` +- `CounterfactualCheck` +- `ImprovementHypothesis` +- `FailureDiagnosis` +- `DiagnosisVerdict` including abstention + +### Work + +1. Add read-only tools for mined evidence, harness revision assets, repository + search/read, and verifier evidence. +2. Add `ofw-diagnose-failures` Codex skill with explicit no-edit boundary. +3. Require causal claims to cite both failure evidence and relevant harness + evidence. +4. Distinguish confirmed mechanism, plausible hypothesis, and unknown. +5. Allow multiple hypotheses; never force one root cause. +6. Persist diagnoses with judge session and model/harness revision provenance. + +### Verification + +```bash +uv run pytest tests/test_diagnosis.py tests/test_diagnosis_mcp.py -q +uv run pytest -q +uv run ruff check src tests +uv run mypy src tests +``` + +Test recovered errors, missing source code, multiple plausible mechanisms, +stale harness revision, contradictory evidence, and mandatory abstention. + +### Exit criteria + +Every confirmed failure has zero or more versioned diagnoses; no diagnosis can +exist without a mined failure and cited evidence. + +### Rollback + +Diagnosis is additive. Disable the workflow without changing mining results. + +## 11. PR10 — Eval case construction and reproduction + +Branch: `codex/eval-cases` +Depends on: PR7, PR9 +Risk: high; generated evals must reproduce real failures + +### Context brief + +Convert diagnosed production failures into reproducible cases. The core case +contract is: + +```text +task +context seed +environment +oracle +verification tests +source failure and diagnosis +``` + +An agent proposes a case; a reproduction run admits it. + +### Contracts + +- `EvalCaseId`, `EvalCaseRevision` +- `ContextSeed` +- `EvalCaseCandidate` +- `EvalCase` +- `ReproductionAttempt` +- `ReproductionVerdict` + +### Work + +1. Add content-addressed eval candidates and immutable admitted revisions. +2. Add `ofw-build-evals` skill and `submit_eval_candidate` tool. +3. Reconstruct only information available before the executed agent begins the task. +4. Bind the case to PR7 environment/oracle/tests. +5. Run the frozen failing revision and require the expected failure behavior or + failed oracle to reproduce. +6. Reject leakage from final answers, hidden test outputs, and held-out traces. +7. Support deterministic and human-approved admission; an LLM proposal cannot + admit itself. + +### Verification + +```bash +uv run pytest tests/test_evals.py tests/test_reproduction.py -q +uv run pytest -q +uv run ruff check src tests +uv run mypy src tests +``` + +Test non-reproduction, flaky reproduction, leaked outcome, wrong environment, +missing oracle, source lineage, canonical ID, and duplicate candidates. + +### Exit criteria + +At least one failure from each supported corpus type can become a reproducible, +immutable eval case without embedding the hidden answer in agent-visible input. + +### Rollback + +Candidate records remain; mark unsupported candidates rejected. Never delete +admitted eval history. + +## 12. PR11 — Failure clusters and living eval registry + +Branch: `codex/failure-registry` +Depends on: PR6, PR9, PR10 +Parallel with: PR12 +Risk: medium + +### Context brief + +Maintain compact coverage over recurring failure mechanisms rather than one +eval per incident. Start with typed signatures and Codex proposals; do not add a +vector database until retrieval quality measurements justify it. + +### Contracts + +- `FailureClusterId`, `FailureClusterRevision` +- `FailureSignature` +- `ClusterMembershipProposal` +- `ClusterStatus` +- `EvalLifecycle`: candidate, frontier, regression, retired +- `EvalSuiteRevision` +- `VerifierRevision` +- `CalibrationCase` +- `VerifierCalibration` + +### Work + +1. Derive a stable signature from behavior, diagnosed mechanism, phase, + recovery, and required-outcome IDs. +2. Add exact/structured candidate retrieval before semantic similarity. +3. Let Codex propose cluster merge/split/membership with cited examples. +4. Validate proposals against immutable failure and diagnosis records. +5. Track occurrence, severity, resolution attempts, reproduction rate, and eval + coverage. +6. Add append-only suite transitions and reasons. +7. Compare trace verifiers against human labels, deterministic outcomes, + environment checks, judge disagreements, and different-outcome runs. +8. Let Codex propose one focused verifier/rubric revision at a time. Admit it + only on a sealed calibration set; it may not weaken or replace an environment + oracle. + +### Verification + +```bash +uv run pytest tests/test_clusters.py tests/test_eval_registry.py -q +uv run pytest -q +uv run ruff check src tests +uv run mypy src tests +``` + +Test accidental merge, duplicate failure, split lineage, cluster reopening, +frontier-to-regression transition, retention of critical rare cases, verifier +false-positive/false-negative changes, and rejected rubric goalpost movement. + +### Exit criteria + +The six-session corpus produces a reviewable set of clusters with explicit +membership evidence and at least one admitted eval for every covered cluster. + +### Rollback + +Revert cluster algorithms while preserving proposals and previous registry +revisions. Active suite selection points to the last admitted revision. + +## 13. PR12 — Reproducible workspaces and experiment execution + +Branch: `codex/workspace-experiments` +Depends on: PR7, PR10, existing PR4 runtime +Parallel with: PR11 +Risk: high; executes untrusted agent actions + +### Context brief + +PR4 can run a canary in E2B. Generalize that narrow path into resettable eval +workspaces without building a NeoSigma-style fleet control plane. E2B remains +the first provider; the protocol must permit another provider later. + +### Contracts + +- `WorkspaceId`, `WorkspaceSnapshotId` +- `WorkspaceSpecification` +- `ExperimentRunId` +- `ExperimentAttempt` +- `RunArtifact` + +### Work + +1. Provision E2B from a versioned environment specification. Do not introduce + a provider protocol until a second workspace backend is actually required. +2. Materialize the exact harness revision and eval case. +3. Run setup, health checks, the connected agent harness, and verification with independent limits. +4. Reset filesystem and state between attempts; prove isolation. +5. Keep secrets outside persisted artifacts and agent-visible logs. +6. Collect the resulting trace through Langfuse; do not implement an OFW trace + exporter. +7. Persist commands, exit state, verifier evidence, trace ID, cost, and timing. + +### Verification + +```bash +uv run pytest tests/test_workspace.py tests/test_experiments.py -q +uv run pytest -q +uv run ruff check src tests +uv run mypy src tests +``` + +Use a small E2B client fake in CI and one opt-in live integration test. Test timeout, +workspace loss, dirty reset, service-not-ready, trace delay, verifier failure, +and secret absence. + +### Exit criteria + +Champion and candidate revisions can run the same eval from identical initial +state with independently verified outcomes and Langfuse trace IDs. + +### Rollback + +Keep PR4 canary API working. Disable multi-attempt experiments if the provider +cannot guarantee reset; never reuse uncertain state. + +## 14. PR13 — Gates and eval-suite transitions + +Branch: `codex/gates-suites` +Depends on: PR11, PR12 +Risk: critical; only this layer may admit improvement + +### Context brief + +Implement the deterministic acceptance boundary before building an optimizer. +Codex may interpret failures but cannot waive gates or see held-out evidence. + +### Contracts + +- `ChampionRevision` +- `CandidateRevision` +- `GatePolicy` +- `GateAttempt` +- `GateReason` +- `PromotionDecision` +- `Pin` + +### Work + +1. Run the regression suite first and stop on critical-case regression. +2. Run a sealed frontier/validation slice regardless of Codex hypotheses. +3. Compare outcome, critical failures, latency, and cost under explicit policy. +4. Require no unapproved harness files changed. +5. Promote newly resolved frontier cases only after repeated verified success. +6. Record keep/revert and pin decisions append-only. +7. Never delete a prior champion or suite revision. + +### Verification + +```bash +uv run pytest tests/test_gate.py tests/test_promotion.py -q +uv run pytest -q +uv run ruff check src tests +uv run mypy src tests +``` + +Test better average with critical regression, flaky improvement, missing result, +cost dominance, unauthorized edit, holdout leakage, promotion, and rollback. + +### Exit criteria + +A manually supplied candidate can be admitted or rejected without any Codex +judgment in the acceptance path, and the decision is reproducible from stored +evidence. + +### Rollback + +Pin the last champion before rollout. Reverting code leaves that pin active; +newer unsupported decisions become read-only history. + +## 15. PR14 — Bounded Fit optimizer + +Branch: `codex/fit-optimizer` +Depends on: PR9, PR12, PR13 +Risk: critical; grants Codex repository-write authority + +### Context brief + +Now allow Codex to propose one focused connected-harness change at a time. The +editable component surface comes from immutable OFW revision metadata. Codex +does not edit OFW infrastructure, verifiers, gates, holdouts, or budgets. + +### Contracts + +- `FitCampaignId`, `FitBudget` +- `ChangeHypothesis` +- `HarnessChangeProposal` +- `CandidateBuild` +- `FitAttempt` +- `FitOutcome` + +### Work + +1. Expose only revision assets marked editable. +2. Add `ofw-evolve-harness` skill: inspect cluster/diagnosis/eval evidence, + propose one change, state expected fixes and at-risk regressions. +3. Validate and apply patches in an isolated git worktree. +4. Process a new immutable harness revision. +5. Execute PR13 gates and keep/revert automatically from the typed decision. +6. Store rejected attempts and their evidence so the next attempt does not + repeat a failed hypothesis blindly. +7. Enforce iteration, token, wall-clock, experiment, and spend limits. + +### Verification + +```bash +uv run pytest tests/test_fit.py tests/test_candidate.py -q +uv run pytest -q +uv run ruff check src tests +uv run mypy src tests +``` + +Test out-of-surface edit, verifier edit, multiple simultaneous hypotheses, +invalid patch, candidate build failure, gate rejection, accepted change, +budget exhaustion, and resume. + +### Exit criteria + +Starting from a seeded failing reference harness, Codex can produce a candidate, +OFW can gate it, and only an admitted revision becomes the new champion. + +### Rollback + +Candidates live in isolated worktrees until admission. Rejected worktrees may +be archived, then deleted; champion branches and pins are never reset +destructively. + +## 16. PR15 — Resumable flywheel orchestration + +Branch: `codex/flywheel-loop` +Depends on: PR14 +Risk: high; composes every stage + +### Context brief + +Compose existing services; do not duplicate their logic in a new “god loop.” +Every transition reads and writes durable state before external work. + +### Contracts + +- `FlywheelRunId`, `FlywheelStage` +- `FlywheelPolicy` +- `StageAttempt` +- `StopReason` +- `FlywheelReport` + +### Work + +1. Implement the stage machine: + collect → signal → mine → diagnose → build/reproduce evals → cluster → fit → + gate → pin. +2. Make each stage idempotent and resumable. +3. Add stage-specific budgets and global stop conditions. +4. Support human approval gates without requiring them by default for safe, + local experiments. +5. Emit compact progress reports with artifact IDs, not copied trace content. +6. Add CLI and Python facade entry points over the same service. + +### Verification + +```bash +uv run pytest tests/test_loop.py tests/test_e2e_release.py -q +uv run pytest -q +uv run ruff check src tests +uv run mypy src tests +``` + +Test crash/restart at every boundary, duplicate external response, unavailable +Langfuse, Codex timeout, no failures, no diagnosis, no reproducible eval, +budget exhaustion, rejected candidate, and admitted candidate. + +### Exit criteria + +An end-to-end fixture resumes after deliberate process termination and reaches +the same final pin without duplicate cases, experiments, or decisions. + +### Rollback + +Disable automatic progression and retain manual stage APIs. Durable state shows +the exact last completed stage and safe resume point. + +## 17. PR16 — End-to-end SDK facade and Codex plugin release + +Branch: `codex/sdk-hardening` +Depends on: PR15 +Risk: medium + +### Context brief + +Assemble the already-public stage APIs into the final end-to-end experience. +Every preceding PR must harden its own public imports and must not defer basic +SDK quality to this step. Do not design the facade in advance of service evidence. + +### Work + +1. Add the `Project` facade over existing services without hiding typed results. +2. Freeze public import paths and add compatibility tests. +3. Complete the OpenFlyWheel Codex plugin with focused skills: + `integrate-ofw`, `ofw-mine-failures`, `import-ofw-verifiers`, + `ofw-diagnose-failures`, `ofw-build-evals`, and `ofw-evolve-harness`. +4. Package the supported MCP servers and declare tool dependencies. +5. Add documented generic integration guidance plus a Hermes reference fixture + using Langfuse and E2B. +6. Add security/privacy documentation, threat boundaries, and secret scanning. +7. Publish migration notes from PR5 APIs and explicit non-goals. + +### Verification + +```bash +python3 ~/.codex/skills/.system/plugin-creator/scripts/validate_plugin.py plugins/openflywheel +uv run pytest -q +uv run ruff check src tests +uv run mypy src tests +uv build +``` + +Run a clean-environment install, plugin discovery test, typed example, and one +opt-in live Langfuse/E2B smoke. + +### Exit criteria + +A fresh Codex session can install the plugin, integrate an existing Langfuse-connected harness +repository without manual OFW schema authoring, run one bounded flywheel cycle, +and inspect every decision artifact. + +### Rollback + +Keep lower-level service APIs supported for one release. Revert facade/plugin +packaging without changing stored domain records. + +## 18. Cross-PR quality gates + +Every PR must satisfy: + +- tests written before non-trivial implementation; +- full test suite, Ruff, strict mypy, and `git diff --check` clean; +- no secrets, raw credentials, or production trace content committed; +- public contracts use typed classes/enums, never `Any` or metadata bags; +- one migration test for every persisted schema change; +- public imports and typed examples updated in the same PR that adds a service; +- one invalid/adversarial fixture for every admission boundary; +- compatibility maintained or removed explicitly in the same PR; +- PR body states authority gained by Codex and the deterministic control that + bounds it. + +## 19. Success metrics + +System quality: + +- mining precision/recall against reviewed production failures; +- environment-verification availability and stale-evidence rate; +- diagnosis agreement/abstention against reviewed cases; +- eval reproduction rate and flake rate; +- percentage of recurring failure clusters covered by admitted evals; +- regression-suite escape rate; +- candidate acceptance rate and improvement on sealed validation; +- cost and elapsed time per admitted harness improvement. + +SDK quality: + +- integration time for a new agent-harness repository; +- fraction of components discovered without user schema authoring; +- restart success at every stage boundary; +- zero duplicate derived records after retry; +- stable public API and plugin activation accuracy. + +## 20. Explicit non-goals + +- no OFW trace SDK, OTLP collector, trace database, or trace UI; +- no model gateway or proxy; +- no general-purpose production scheduler or warm sandbox fleet in this plan; +- no model weight training or fine-tuning; +- no generic agent framework replacing the connected harness; +- no live production mutation by mining or diagnosis tools; +- no semantic vector database before structured retrieval is measured and found + insufficient; +- no single `ofw.improve()` tool that collapses proposal and admission. + +## 21. Anti-pattern catalog + +- **Trace collector creep:** adding spans/exporters because a Langfuse field is + inconvenient. Extend the connector or require attribution instead. +- **Judge as oracle:** accepting Codex or executed-agent prose instead of environment + evidence. +- **Stage collapse:** mining records root cause, diagnosis generates evals, or + Fit edits before a gate exists. +- **Self-admission:** the agent that proposes an artifact also activates it. +- **Holdout leakage:** exposing test traces, expected outputs, or verifier + internals to candidate generation. +- **Averages hide harm:** accepting better mean reward with critical regression. +- **Mutable history:** updating a failure, cluster, suite, or pin in place. +- **Provider types in domain:** E2B, Langfuse, GitHub, or pytest classes leaking + into provider-agnostic contracts. +- **Premature orchestration:** building PR15 before individual stages are + executable and restart-safe. +- **Skill as implementation:** placing business rules only in Codex prose rather + than typed OFW validators. + +## 22. Plan mutation protocol + +When implementation evidence invalidates a step: + +1. Record the new fact in this plan under a dated `Plan amendments` section. +2. Do not silently broaden the active PR. +3. Split a PR when it gains a second independently releasable authority boundary + or exceeds one migration plus one service. +4. Insert a prerequisite PR when a required contract or deterministic validator + is missing. +5. Reorder only when dependency edges and owned files remain valid. +6. Skip a PR only when its exit criteria are already proven by committed tests; + link the evidence. +7. Abandon a direction after three evidence-backed failures of the same + assumption, preserving attempts and the selected alternative. + +## 23. First move + +Merge PR5. Then start PR6 and PR7 in parallel. Do not start diagnosis, eval +generation, clustering, or Fit until the durable signal and verification +contracts are merged. + +## 24. Adversarial review decisions + +Review completed 2026-08-25. + +Accepted: + +- Keep E2B as the only workspace backend; remove the speculative provider + abstraction until a second implementation exists. +- Harden public SDK imports incrementally rather than postponing them to the + final PR. +- Keep the final flywheel orchestrator thin and dependent on independently + executable, restart-safe stages. +- Add explicit verifier/rubric calibration to the living eval registry so OFW + covers the evaluation-maintenance capability, not only case clustering. + +Not accepted: + +- Merging workspaces, gates, and Fit into one PR. Gate construction must land + and be testable before Codex receives repository-write authority; combining + them would let proposal and admission arrive in the same change. +- Collapsing the remaining roadmap to eight PRs. The current eleven steps stay + within Blueprint's normal range and each high-risk authority boundary remains + independently reviewable and reversible. + +## 25. Plan amendments + +None yet. diff --git a/plugins/openflywheel/.codex-plugin/plugin.json b/plugins/openflywheel/.codex-plugin/plugin.json index fc76566..912b0c8 100644 --- a/plugins/openflywheel/.codex-plugin/plugin.json +++ b/plugins/openflywheel/.codex-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "openflywheel", "version": "0.1.0", - "description": "Integrate Hermes with OpenFlyWheel and mine evidence-backed failures with Codex.", + "description": "Integrate any Langfuse-connected agent harness with OpenFlyWheel and mine evidence-backed failures with Codex.", "author": { "name": "OpenFlyWheel" }, @@ -9,13 +9,13 @@ "skills": "./skills/", "interface": { "displayName": "OpenFlyWheel", - "shortDescription": "Integrate Hermes and mine its failures", - "longDescription": "Codex workflows for integrating a Hermes codebase with OpenFlyWheel and mining observable failures from full Langfuse trajectories.", + "shortDescription": "Integrate agent harnesses and mine failures", + "longDescription": "Codex workflows for integrating any Langfuse-connected agent harness with OpenFlyWheel and mining observable failures from full trajectories.", "developerName": "OpenFlyWheel", "category": "Developer Tools", "capabilities": ["Read", "Write"], "defaultPrompt": [ - "Integrate this Hermes codebase with OpenFlyWheel.", + "Integrate this agent harness with OpenFlyWheel.", "Mine failures from the connected OpenFlyWheel trajectory." ] } diff --git a/plugins/openflywheel/skills/integrate-ofw/SKILL.md b/plugins/openflywheel/skills/integrate-ofw/SKILL.md index fc1b4f5..08a5c8b 100644 --- a/plugins/openflywheel/skills/integrate-ofw/SKILL.md +++ b/plugins/openflywheel/skills/integrate-ofw/SKILL.md @@ -1,30 +1,30 @@ --- name: integrate-ofw -description: Integrates a Hermes codebase with OpenFlyWheel by declaring its versioned harness components, connecting its existing Langfuse project, attributing real runs to an immutable revision, and verifying full trace collection. Use for onboarding or adding OFW to Hermes. Do not use for failure mining, diagnosis, harness optimization, or replacing the application's observability backend. +description: Integrates any agent harness already connected to Langfuse with OpenFlyWheel by declaring versioned harness components, attributing real runs to an immutable revision, and verifying full trace collection. Use for onboarding an agent system to OFW. Do not use for failure mining, diagnosis, harness optimization, or replacing Langfuse. --- -# Integrate Hermes with OpenFlyWheel +# Integrate an agent harness with OpenFlyWheel -Add OFW to the real Hermes execution path without changing Hermes behavior. +Add OFW to the real agent execution path without changing harness behavior. Read [references/python-api.md](references/python-api.md) before writing OFW code; it contains the exact supported API. ## Principles 1. **Instrument the real path.** Find the command, service, or worker that - actually starts Hermes. Do not create a parallel demonstration agent. + actually starts the agent harness. Do not create a parallel demonstration agent. 2. **Discover components for the user.** Locate the active system prompt, tool implementations, skills, subagent definitions, and middleware. The user should not have to translate their repository into OFW schemas manually. 3. **Do not change behavior.** This workflow may add an OFW declaration and - revision attribution. It must not rewrite prompts, tools, skills, or Hermes + revision attribution. It must not rewrite prompts, tools, skills, or harness control flow. 4. **Keep secrets in the environment.** Never place Langfuse keys in code, manifests, plugin files, logs, or chat. OFW records only environment-variable names. 5. **Preserve Langfuse.** Connect the application's existing Langfuse project; do not replace its SDK, exporter, or trace hierarchy. -6. **Verify with a real run.** Integration is complete only when one real Hermes +6. **Verify with a real run.** Integration is complete only when one real agent trajectory is attributed to the generated revision and collected with full input/output content into local SQLite. @@ -34,7 +34,7 @@ code; it contains the exact supported API. `openflywheel` is available from the user's chosen package source. If it is not resolvable, stop and ask for the intended install source; do not invent a Git URL or published version. -2. Identify the git root and the active Hermes assets: +2. Identify the git root and the active harness assets: - one or more prompt files; - named tool implementation files; - skill files; @@ -49,10 +49,10 @@ code; it contains the exact supported API. name and existing Langfuse environment variables. Never read or copy their values. 6. Process the harness to produce an immutable revision and manifest. -7. Add revision attribution around the real Hermes run using +7. Add revision attribution around the real agent run using `propagate_attributes` and metadata key `ofw.harness.revision`. Do not add a second root trace. -8. Run one real Hermes request, flush the existing Langfuse client as required +8. Run one real agent request, flush the existing Langfuse client as required by its runtime, then collect a UTC window containing that run with `ofw.collect`. 9. Verify that the collection belongs to the revision, contains the expected @@ -61,7 +61,7 @@ code; it contains the exact supported API. ## Stop conditions -- Stop before editing when the active Hermes entry point or prompt cannot be +- Stop before editing when the active agent entry point or prompt cannot be identified confidently. - Stop before a live run when credentials are unavailable; report the exact environment-variable names needed without exposing values. @@ -76,7 +76,7 @@ Report: - the OFW declaration and execution path changed; - the discovered component-to-file mapping and which files are editable; - the immutable revision ID and manifest path; -- the real Hermes command/request used for verification; +- the real agent command/request used for verification; - the collected trace ID, observation count, capability/gap status, and SQLite path; - any missing component, credential, attribution, or content evidence. diff --git a/plugins/openflywheel/skills/integrate-ofw/references/python-api.md b/plugins/openflywheel/skills/integrate-ofw/references/python-api.md index 84f5acb..e3ca4da 100644 --- a/plugins/openflywheel/skills/integrate-ofw/references/python-api.md +++ b/plugins/openflywheel/skills/integrate-ofw/references/python-api.md @@ -1,6 +1,6 @@ # Supported Python integration -Use only this API surface. Adapt paths and names to the inspected Hermes +Use only this API surface. Adapt paths and names to the inspected agent-harness repository; do not invent components that are not active. ```python @@ -12,7 +12,7 @@ from ofw import Harness, LangfuseProject, Tool, TraceWindow, ofw, propagate_attr ROOT = Path(__file__).resolve().parent project = LangfuseProject.from_env(environment="production") -harness = Harness("hermes", root=ROOT) +harness = Harness("agent-harness", root=ROOT) harness.connect_prompt(ofw.editable(Path("path/to/system-prompt.md"))) harness.connect_tools( Tool(name="search", source=Path("path/to/search-tool.py")), @@ -28,13 +28,13 @@ Register only component kinds that exist. For named subagents use names must be lowercase identifiers accepted by OFW. Every registered path is relative to the git root and may belong to only one component. -Run the real Hermes entry point inside revision attribution: +Run the real agent entry point inside revision attribution: ```python with propagate_attributes( metadata={"ofw.harness.revision": str(revision.id)}, ): - run_real_hermes_request() + run_real_agent_request() ``` The surrounding application remains responsible for its existing Langfuse diff --git a/plugins/openflywheel/skills/ofw-mine-failures/SKILL.md b/plugins/openflywheel/skills/ofw-mine-failures/SKILL.md index 78f89bc..315cde4 100644 --- a/plugins/openflywheel/skills/ofw-mine-failures/SKILL.md +++ b/plugins/openflywheel/skills/ofw-mine-failures/SKILL.md @@ -1,20 +1,20 @@ --- name: ofw-mine-failures -description: Mines observable failures from executed Hermes trajectories through a connected OpenFlyWheel failure-mining MCP server. Use when asked to judge, triage, or mine failures from OFW/Langfuse traces. Do not use for root-cause diagnosis, clustering, eval generation, rubric refinement, or modifying Hermes. +description: Mines observable failures from any executed agent harness through a connected OpenFlyWheel failure-mining MCP server backed by full Langfuse trajectories. Use when asked to judge, triage, or mine failures from OFW traces. Do not use for root-cause diagnosis, clustering, eval generation, rubric refinement, or modifying the evaluated harness. --- -# Mine Hermes failures with OpenFlyWheel +# Mine agent-harness failures with OpenFlyWheel Investigate one OFW mining case and return an evidence-grounded failure-mining -result. Hermes is the executed agent. You are the Codex operator using OFW. +result. The connected harness is the executed system. You are the Codex operator using OFW. Stay at observable behavior; diagnosis and improvement are separate work. ## Principles -1. **The oracle decides completion.** A Hermes claim is evidence of what it +1. **The oracle decides completion.** An agent claim is evidence of what it claimed, not proof that the task succeeded. Verify every required outcome against its declared environment source. -2. **Recovery matters.** A failed action is not a task failure when Hermes later +2. **Recovery matters.** A failed action is not a task failure when the agent later recovers and every required outcome is completed. 3. **Use only issued evidence.** Cite observation and environment evidence returned by OFW tools. Never invent identifiers, digests, state, or tool @@ -32,7 +32,7 @@ Stay at observable behavior; diagnosis and improvement are separate work. ## Workflow 1. Call `get_mining_case`. Read the task intent, constraints, required outcomes, - available Hermes tools, environment sources, observation IDs, and nominated + available agent tools, environment sources, observation IDs, and nominated signals. 2. Search the current trajectory with `search_trajectory`: - search task-specific entities and required outcomes; @@ -99,10 +99,10 @@ trajectory evidence. Keep the summary factual and free of causal claims. ## Decision examples -- A command fails, Hermes retries successfully, and the oracle confirms the +- A command fails, the agent retries successfully, and the oracle confirms the required state: `no_failure`. -- Hermes says the work is complete, but the oracle shows the required state was +- The agent says the work is complete, but the oracle shows the required state was not reached: `confirmed_failure` with `false_completion` or `outcome_mismatch`, depending on the observed behavior. -- Hermes appears to stop early, but the environment cannot be queried: +- The agent appears to stop early, but the environment cannot be queried: `ambiguous`, not an inferred failure. diff --git a/src/ofw/mcp.py b/src/ofw/mcp.py index 6b60462..5b6e438 100644 --- a/src/ofw/mcp.py +++ b/src/ofw/mcp.py @@ -152,7 +152,7 @@ def __post_init__(self) -> None: server: MCPServer[None] = MCPServer( name="openflywheel-failure-mining", instructions=( - "Inspect and verify one executed Hermes trajectory. Do not diagnose causes, " + "Inspect and verify one executed agent-harness trajectory. Do not diagnose causes, " "propose fixes, cluster failures, generate evals, or mutate rubrics." ), ) From 6e33e46744d089b036e2a495fd442350cd06b05f Mon Sep 17 00:00:00 2001 From: divo12 Date: Wed, 26 Aug 2026 07:13:35 +0530 Subject: [PATCH 15/18] keep SDK blueprint local --- .../2026-08-25-ofw-full-sdk-blueprint.md | 951 ------------------ 1 file changed, 951 deletions(-) delete mode 100644 docs/plans/2026-08-25-ofw-full-sdk-blueprint.md diff --git a/docs/plans/2026-08-25-ofw-full-sdk-blueprint.md b/docs/plans/2026-08-25-ofw-full-sdk-blueprint.md deleted file mode 100644 index f40e5f8..0000000 --- a/docs/plans/2026-08-25-ofw-full-sdk-blueprint.md +++ /dev/null @@ -1,951 +0,0 @@ -# OpenFlyWheel full SDK blueprint - -Status: proposed -Date: 2026-08-25 -Base: merge PR #5 into `fresh`, then execute PR6–PR16 in dependency order -Executed system: any agent harness connected to Langfuse -Reference integration and test harness: Hermes -OFW operator/reasoning agent: Codex -Outsourced subsystem: Langfuse trace instrumentation, ingestion, storage, and trace UI - -## 1. Objective - -Build OpenFlyWheel as the complete local-first SDK for turning production agent-harness -experience into evidence-backed evaluation and gated harness improvement, while -using Langfuse instead of building an OFW trace collector. - -OFW owns: - -- immutable harness definitions and revisions; -- production-signal semantics; -- task, context, behavior, and environment-verification contracts; -- Codex-facing query, judge, diagnosis, eval, and optimization workflows; -- failure records, diagnoses, clusters, and living eval suites; -- reproducible workspaces and experiment execution; -- candidate generation, gates, promotion, rollback, and pins; -- durable lineage, audit history, resumability, CLI/MCP surfaces, and Codex skills. - -Langfuse owns: - -- client instrumentation and framework adapters; -- trace/span ingestion; -- raw trace persistence and trace UI; -- transport-level buffering, batching, and delivery. - -OFW imports immutable, attributed Langfuse windows into local SQLite. It does -not proxy model calls, export OTLP, or become another observability backend. - -## 1.1 Research basis - -This blueprint derives the product shape from NeoSigma's public -[skills repository](https://github.com/neosigmaai/skills), especially -`integrate-sdk` and `import-verifiers`, plus its posts on -[production evals](https://neosigma.ai/blog/the-most-important-eval-isnt-on-a-leaderboard), -[self-improving systems](https://neosigma.ai/blog/self-improving-agentic-systems), -[agent workspaces](https://neosigma.ai/blog/agent-workspaces), and -[model–harness co-design](https://neosigma.ai/blog/investigating-the-optimal-harness-for-a-model). -Auto Harness is intentionally excluded; it is a minimal user-operated example, -not evidence for NeoSigma's product SDK architecture. - -## 2. Product boundary - -```text -Any production agent harness - └─ Langfuse SDK/exporter - └─ complete traces, tool calls, scores, feedback - └─ OFW Langfuse connector → immutable local snapshot - -Codex + OpenFlyWheel plugin - ├─ Search / Read / Verify / Adapt - ├─ Mine observable failures - ├─ Diagnose confirmed failures - ├─ Build and maintain eval cases - ├─ Propose bounded harness changes - └─ Interpret experiment evidence - -OFW deterministic services - ├─ contracts and lineage - ├─ state store - ├─ environment oracles and verification - ├─ workspace reset and execution - ├─ regression/frontier gates - ├─ keep/revert/pin - └─ budgets, stop conditions, and audit log -``` - -The agent never decides whether its own change ships. Codex proposes and -investigates; typed OFW services reproduce, verify, gate, and record. - -## 3. Target SDK experience - -```python -project = ofw.Project( - harness=ofw.Harness(...), - traces=ofw.LangfuseProject.from_env(environment="production"), - state_path=Path(".ofw/ofw.sqlite"), -) - -revision = project.process() -collection = project.collect(window=window) -mining_run = project.mine(collection, signals=signal_sources) -diagnosis_run = project.diagnose(mining_run.confirmed_failures) -suite_revision = project.build_evals(diagnosis_run) -candidate = project.fit(suite_revision, budget=budget) -decision = project.gate(candidate) -project.pin(decision) # succeeds only for an admitted candidate -``` - -The exact facade may change during implementation. The domain objects and -authority boundaries below may not. - -## 4. Invariants - -1. The connected agent harness is the executed system; Codex is the only reasoning - agent using OFW. Hermes is the reference integration, never a domain assumption. -2. Langfuse content is read-only. OFW never edits or deletes source traces. -3. Every admitted trace references one immutable harness revision. -4. An executed agent's completion claim is never an oracle. -5. Stateful success requires read-only environment evidence or a durable - recorded equivalent. -6. Intermediate action failure is not task failure after verified recovery. -7. Mining observes behavior; diagnosis explains cause; eval building reproduces - the failure; Fit proposes changes. These remain separate stages and types. -8. Agent outputs are proposals until deterministic validation admits them. -9. Eval holdouts and test traces remain invisible to Codex candidate generation. -10. Every state transition is append-only, content-addressed, and attributable - to input revisions, evidence, agent session, and code commit. -11. No `Any`, untyped metadata dictionaries, dynamic `getattr`/`setattr`, or - stringly categorical state in public Python contracts. -12. Failed or interrupted work can resume without replaying completed external - operations. - -## 5. Target package layout - -```text -src/ofw/ - contracts.py existing revision primitives - harness.py existing component registry - runtime.py existing canary/runtime boundary - project.py final public facade - state/ - store.py OFW-owned SQLite state - migrations/ - observability/langfuse/ existing outsourced-collector connector - signals.py production/human signal normalization - verification.py environment/oracle/test contracts - mining/ task/context/behavior + Codex judge sessions - diagnosis/ cause and improvement-hypothesis evidence - evals/ cases, suites, reproduction, lifecycle - clusters.py recurring failure-mode registry - workspace/ resettable E2B execution - experiments.py champion/candidate executions - gate.py suite and promotion policies - fit.py bounded Codex harness optimizer - loop.py resumable flywheel orchestration - mcp/ Codex tool servers over the same services -plugins/openflywheel/ focused Codex skills -``` - -Do not perform a directory-only refactor. Move code only when a PR introduces -the owning service and preserves public imports. - -## 6. Dependency graph - -```text -PR5 evidence-backed mining (current) - ├─ PR6 durable state + production signals ─┐ - └─ PR7 verification SDK + importer ────────┤ - ▼ - PR8 executable Codex judge runtime - ▼ - PR9 failure diagnosis - ▼ - PR10 eval case reproduction - ┌───────┴────────┐ - ▼ ▼ - PR11 cluster/eval registry PR12 workspaces/experiments - └───────┬────────┘ - ▼ - PR13 gates + suite transitions - ▼ - PR14 bounded Fit optimizer - ▼ - PR15 resumable flywheel loop - ▼ - PR16 SDK/plugin hardening -``` - -Parallel lanes: - -- PR6 and PR7 may run in parallel after PR5 merges; they own disjoint modules. -- PR11 and PR12 may run in parallel after PR10; one owns intelligence state, - the other owns execution. -- All other steps are serial because they consume contracts from the preceding - step. - -## 7. PR6 — Durable OFW state and production signals - -Branch: `codex/ofw-state-signals` -Depends on: PR5 -Risk: medium; first OFW-owned persistent schema - -### Context brief - -Langfuse is the source of raw traces and scores, but OFW needs durable state for -its own derived objects. The existing collection SQLite is a source snapshot, -not the flywheel database. Create a separate `.ofw/ofw.sqlite` with explicit -schema versions and append-only lineage. - -Normalize feedback and outcomes without building an OFW event collector. -Initial adapters read Langfuse scores and accept typed caller-provided records. - -### Contracts - -- `OfwStateVersion` -- `SignalId`, `SignalKind`, `SignalSubject` -- `ProductionSignal` -- `SignalSource` protocol -- `LangfuseScoreSignalSource` -- `StateRecordDigest` - -Signal kinds initially cover human feedback, user correction, trusted score, -downstream failure, incident, rollback, reopened work, environment mismatch, -and agent error. Do not add generic string event names to the domain layer. - -### Work - -1. Add OFW state migrations and a version-checking SQLite store. -2. Persist source provenance, trace/revision identity, timestamps, evidence - references, and canonical digests. -3. Import Langfuse scores without mutating the collection snapshot. -4. Make ingestion idempotent by signal identity and content digest. -5. Add query methods by revision, trace, kind, and time window. -6. Move `FailureSource` toward the shared signal contract without breaking the - PR5 public API. - -### Verification - -```bash -uv run pytest tests/test_state.py tests/test_signals.py -q -uv run pytest -q -uv run ruff check src tests -uv run mypy src tests -``` - -Test duplicate ingestion, conflicting identity, wrong revision, corrupt digest, -restart, migration mismatch, and preservation of source evidence. - -### Exit criteria - -The same Langfuse window can be imported repeatedly with one durable signal -record per source fact, and every record resolves to a collected trace and -revision. - -### Rollback - -Revert the code while retaining the new SQLite file. Never downgrade or delete -derived records automatically; later code may ignore unsupported schema -versions. - -## 8. PR7 — Verification SDK and verifier-import skill - -Branch: `codex/verification-contracts` -Depends on: PR5 -Parallel with: PR6 -Risk: high; this defines what “task completed” means - -### Context brief - -PR5 has `EnvironmentSource`, `RequiredOutcome`, and an `EnvironmentVerifier` -protocol. Consolidate them into the language-agnostic contract previously -chosen by the user: - -```text -environment -oracle -verification tests -``` - -The environment owns reproducible state. The oracle defines the success fact. -Verification tests observe that fact. Test implementation language is an edge -adapter detail, never a domain category. - -### Contracts - -- `EnvironmentContract` -- `OracleContract` -- `VerificationTest` -- `VerificationPlan` -- `VerificationAttempt` -- `VerificationEvidence` -- `VerificationStatus` -- `VerificationAdapter` protocol - -Initial adapters may wrap recorded state and deterministic commands already -supported by `runtime.py`. External APIs, databases, GitHub, and ticket systems -remain caller-provided adapters until a real integration is requested. - -### Work - -1. Add I/O-free verification domain objects and canonical digests. -2. Adapt PR5 environment verification to the new plan without compatibility - shims that duplicate truth. -3. Require read-only operation for mining/judging verification. -4. Record observed version/time and freshness for mutable sources. -5. Add `import-ofw-verifiers` to the Codex plugin. It scans existing tests and - produces typed plans so users do not write schemas manually. -6. Explicitly skip mechanical checks that do not express agent outcomes and - stateful checks for which no safe oracle adapter exists. - -### Verification - -```bash -uv run pytest tests/test_verification.py tests/test_runtime.py -q -python3 ~/.codex/skills/.system/plugin-creator/scripts/validate_plugin.py plugins/openflywheel -uv run pytest -q -uv run ruff check src tests -uv run mypy src tests -``` - -Test unsupported oracle, wrong environment version, stale state, mutating -adapter rejection, language-independent command tests, and unavailable state. - -### Exit criteria - -A user repository can be inspected by Codex and represented as an environment, -oracle, and verification-test plan without hand-authoring OFW objects; a known -state yields a reproducible typed verdict. - -### Rollback - -Keep PR5 verification operational until this PR's migration tests pass. Remove -old contracts only in the same PR after every caller moves. - -## 9. PR8 — Executable Codex judge runtime - -Branch: `codex/judge-runtime` -Depends on: PR6, PR7 -Risk: high; first automated Codex-to-OFW control loop - -### Context brief - -PR5 exposes one live mining case through MCP, but OFW cannot yet run a complete -Codex judge session or accept its result through a tool. Build the production -runtime that turns the existing `ofw-mine-failures` skill into an executable, -auditable workflow. - -### Contracts - -- `JudgeSessionId`, `JudgeSessionState` -- `JudgeBudget`, `JudgeAttempt` -- `FailureJudge` protocol -- `CodexFailureJudge` -- `MiningSubmission` - -### Work - -1. Persist a mining session before launching Codex. -2. Serve multiple nominated cases through MCP with stable case IDs. -3. Add `submit_failure_result`; validate the typed PR5 result and tool-issued - evidence before admission. -4. Add a Codex CLI/SDK adapter with explicit timeout, token/tool-call budget, - working directory, plugin/skill selection, and captured session ID. -5. Resume sessions from OFW state after process interruption. -6. Fail closed when Codex returns text without a valid submission. -7. Keep mining read-only: no repository-edit or agent-execution tools. - -### Verification - -```bash -uv run pytest tests/test_judge_runtime.py tests/test_mcp.py tests/test_mine.py -q -uv run pytest -q -uv run ruff check src tests -uv run mypy src tests -``` - -Run an integration smoke over the six existing heterogeneous traces. Require -admitted results for two known successes and four known failures; record -precision errors instead of weakening validators. - -### Exit criteria - -`project.mine(...)` can launch Codex, expose bounded evidence tools, accept one -typed result per case, persist the session, and resume safely. - -### Rollback - -Disable the Codex adapter and retain manual/in-process `FailureJudge` support. -Persisted sessions remain inspectable but cannot auto-resume on unsupported -runtime versions. - -## 10. PR9 — Failure diagnosis - -Branch: `codex/failure-diagnosis` -Depends on: PR8 -Risk: high; must separate evidence from speculation - -### Context brief - -Mining answers whether and where the executed agent failed. Diagnosis is the next stage and -may inspect the connected harness repository. It must not mutate files or generate -evals. - -### Contracts - -- `DiagnosisId` -- `FailureMechanism` -- `FailureLocation` -- `CausalEvidence` -- `CounterfactualCheck` -- `ImprovementHypothesis` -- `FailureDiagnosis` -- `DiagnosisVerdict` including abstention - -### Work - -1. Add read-only tools for mined evidence, harness revision assets, repository - search/read, and verifier evidence. -2. Add `ofw-diagnose-failures` Codex skill with explicit no-edit boundary. -3. Require causal claims to cite both failure evidence and relevant harness - evidence. -4. Distinguish confirmed mechanism, plausible hypothesis, and unknown. -5. Allow multiple hypotheses; never force one root cause. -6. Persist diagnoses with judge session and model/harness revision provenance. - -### Verification - -```bash -uv run pytest tests/test_diagnosis.py tests/test_diagnosis_mcp.py -q -uv run pytest -q -uv run ruff check src tests -uv run mypy src tests -``` - -Test recovered errors, missing source code, multiple plausible mechanisms, -stale harness revision, contradictory evidence, and mandatory abstention. - -### Exit criteria - -Every confirmed failure has zero or more versioned diagnoses; no diagnosis can -exist without a mined failure and cited evidence. - -### Rollback - -Diagnosis is additive. Disable the workflow without changing mining results. - -## 11. PR10 — Eval case construction and reproduction - -Branch: `codex/eval-cases` -Depends on: PR7, PR9 -Risk: high; generated evals must reproduce real failures - -### Context brief - -Convert diagnosed production failures into reproducible cases. The core case -contract is: - -```text -task -context seed -environment -oracle -verification tests -source failure and diagnosis -``` - -An agent proposes a case; a reproduction run admits it. - -### Contracts - -- `EvalCaseId`, `EvalCaseRevision` -- `ContextSeed` -- `EvalCaseCandidate` -- `EvalCase` -- `ReproductionAttempt` -- `ReproductionVerdict` - -### Work - -1. Add content-addressed eval candidates and immutable admitted revisions. -2. Add `ofw-build-evals` skill and `submit_eval_candidate` tool. -3. Reconstruct only information available before the executed agent begins the task. -4. Bind the case to PR7 environment/oracle/tests. -5. Run the frozen failing revision and require the expected failure behavior or - failed oracle to reproduce. -6. Reject leakage from final answers, hidden test outputs, and held-out traces. -7. Support deterministic and human-approved admission; an LLM proposal cannot - admit itself. - -### Verification - -```bash -uv run pytest tests/test_evals.py tests/test_reproduction.py -q -uv run pytest -q -uv run ruff check src tests -uv run mypy src tests -``` - -Test non-reproduction, flaky reproduction, leaked outcome, wrong environment, -missing oracle, source lineage, canonical ID, and duplicate candidates. - -### Exit criteria - -At least one failure from each supported corpus type can become a reproducible, -immutable eval case without embedding the hidden answer in agent-visible input. - -### Rollback - -Candidate records remain; mark unsupported candidates rejected. Never delete -admitted eval history. - -## 12. PR11 — Failure clusters and living eval registry - -Branch: `codex/failure-registry` -Depends on: PR6, PR9, PR10 -Parallel with: PR12 -Risk: medium - -### Context brief - -Maintain compact coverage over recurring failure mechanisms rather than one -eval per incident. Start with typed signatures and Codex proposals; do not add a -vector database until retrieval quality measurements justify it. - -### Contracts - -- `FailureClusterId`, `FailureClusterRevision` -- `FailureSignature` -- `ClusterMembershipProposal` -- `ClusterStatus` -- `EvalLifecycle`: candidate, frontier, regression, retired -- `EvalSuiteRevision` -- `VerifierRevision` -- `CalibrationCase` -- `VerifierCalibration` - -### Work - -1. Derive a stable signature from behavior, diagnosed mechanism, phase, - recovery, and required-outcome IDs. -2. Add exact/structured candidate retrieval before semantic similarity. -3. Let Codex propose cluster merge/split/membership with cited examples. -4. Validate proposals against immutable failure and diagnosis records. -5. Track occurrence, severity, resolution attempts, reproduction rate, and eval - coverage. -6. Add append-only suite transitions and reasons. -7. Compare trace verifiers against human labels, deterministic outcomes, - environment checks, judge disagreements, and different-outcome runs. -8. Let Codex propose one focused verifier/rubric revision at a time. Admit it - only on a sealed calibration set; it may not weaken or replace an environment - oracle. - -### Verification - -```bash -uv run pytest tests/test_clusters.py tests/test_eval_registry.py -q -uv run pytest -q -uv run ruff check src tests -uv run mypy src tests -``` - -Test accidental merge, duplicate failure, split lineage, cluster reopening, -frontier-to-regression transition, retention of critical rare cases, verifier -false-positive/false-negative changes, and rejected rubric goalpost movement. - -### Exit criteria - -The six-session corpus produces a reviewable set of clusters with explicit -membership evidence and at least one admitted eval for every covered cluster. - -### Rollback - -Revert cluster algorithms while preserving proposals and previous registry -revisions. Active suite selection points to the last admitted revision. - -## 13. PR12 — Reproducible workspaces and experiment execution - -Branch: `codex/workspace-experiments` -Depends on: PR7, PR10, existing PR4 runtime -Parallel with: PR11 -Risk: high; executes untrusted agent actions - -### Context brief - -PR4 can run a canary in E2B. Generalize that narrow path into resettable eval -workspaces without building a NeoSigma-style fleet control plane. E2B remains -the first provider; the protocol must permit another provider later. - -### Contracts - -- `WorkspaceId`, `WorkspaceSnapshotId` -- `WorkspaceSpecification` -- `ExperimentRunId` -- `ExperimentAttempt` -- `RunArtifact` - -### Work - -1. Provision E2B from a versioned environment specification. Do not introduce - a provider protocol until a second workspace backend is actually required. -2. Materialize the exact harness revision and eval case. -3. Run setup, health checks, the connected agent harness, and verification with independent limits. -4. Reset filesystem and state between attempts; prove isolation. -5. Keep secrets outside persisted artifacts and agent-visible logs. -6. Collect the resulting trace through Langfuse; do not implement an OFW trace - exporter. -7. Persist commands, exit state, verifier evidence, trace ID, cost, and timing. - -### Verification - -```bash -uv run pytest tests/test_workspace.py tests/test_experiments.py -q -uv run pytest -q -uv run ruff check src tests -uv run mypy src tests -``` - -Use a small E2B client fake in CI and one opt-in live integration test. Test timeout, -workspace loss, dirty reset, service-not-ready, trace delay, verifier failure, -and secret absence. - -### Exit criteria - -Champion and candidate revisions can run the same eval from identical initial -state with independently verified outcomes and Langfuse trace IDs. - -### Rollback - -Keep PR4 canary API working. Disable multi-attempt experiments if the provider -cannot guarantee reset; never reuse uncertain state. - -## 14. PR13 — Gates and eval-suite transitions - -Branch: `codex/gates-suites` -Depends on: PR11, PR12 -Risk: critical; only this layer may admit improvement - -### Context brief - -Implement the deterministic acceptance boundary before building an optimizer. -Codex may interpret failures but cannot waive gates or see held-out evidence. - -### Contracts - -- `ChampionRevision` -- `CandidateRevision` -- `GatePolicy` -- `GateAttempt` -- `GateReason` -- `PromotionDecision` -- `Pin` - -### Work - -1. Run the regression suite first and stop on critical-case regression. -2. Run a sealed frontier/validation slice regardless of Codex hypotheses. -3. Compare outcome, critical failures, latency, and cost under explicit policy. -4. Require no unapproved harness files changed. -5. Promote newly resolved frontier cases only after repeated verified success. -6. Record keep/revert and pin decisions append-only. -7. Never delete a prior champion or suite revision. - -### Verification - -```bash -uv run pytest tests/test_gate.py tests/test_promotion.py -q -uv run pytest -q -uv run ruff check src tests -uv run mypy src tests -``` - -Test better average with critical regression, flaky improvement, missing result, -cost dominance, unauthorized edit, holdout leakage, promotion, and rollback. - -### Exit criteria - -A manually supplied candidate can be admitted or rejected without any Codex -judgment in the acceptance path, and the decision is reproducible from stored -evidence. - -### Rollback - -Pin the last champion before rollout. Reverting code leaves that pin active; -newer unsupported decisions become read-only history. - -## 15. PR14 — Bounded Fit optimizer - -Branch: `codex/fit-optimizer` -Depends on: PR9, PR12, PR13 -Risk: critical; grants Codex repository-write authority - -### Context brief - -Now allow Codex to propose one focused connected-harness change at a time. The -editable component surface comes from immutable OFW revision metadata. Codex -does not edit OFW infrastructure, verifiers, gates, holdouts, or budgets. - -### Contracts - -- `FitCampaignId`, `FitBudget` -- `ChangeHypothesis` -- `HarnessChangeProposal` -- `CandidateBuild` -- `FitAttempt` -- `FitOutcome` - -### Work - -1. Expose only revision assets marked editable. -2. Add `ofw-evolve-harness` skill: inspect cluster/diagnosis/eval evidence, - propose one change, state expected fixes and at-risk regressions. -3. Validate and apply patches in an isolated git worktree. -4. Process a new immutable harness revision. -5. Execute PR13 gates and keep/revert automatically from the typed decision. -6. Store rejected attempts and their evidence so the next attempt does not - repeat a failed hypothesis blindly. -7. Enforce iteration, token, wall-clock, experiment, and spend limits. - -### Verification - -```bash -uv run pytest tests/test_fit.py tests/test_candidate.py -q -uv run pytest -q -uv run ruff check src tests -uv run mypy src tests -``` - -Test out-of-surface edit, verifier edit, multiple simultaneous hypotheses, -invalid patch, candidate build failure, gate rejection, accepted change, -budget exhaustion, and resume. - -### Exit criteria - -Starting from a seeded failing reference harness, Codex can produce a candidate, -OFW can gate it, and only an admitted revision becomes the new champion. - -### Rollback - -Candidates live in isolated worktrees until admission. Rejected worktrees may -be archived, then deleted; champion branches and pins are never reset -destructively. - -## 16. PR15 — Resumable flywheel orchestration - -Branch: `codex/flywheel-loop` -Depends on: PR14 -Risk: high; composes every stage - -### Context brief - -Compose existing services; do not duplicate their logic in a new “god loop.” -Every transition reads and writes durable state before external work. - -### Contracts - -- `FlywheelRunId`, `FlywheelStage` -- `FlywheelPolicy` -- `StageAttempt` -- `StopReason` -- `FlywheelReport` - -### Work - -1. Implement the stage machine: - collect → signal → mine → diagnose → build/reproduce evals → cluster → fit → - gate → pin. -2. Make each stage idempotent and resumable. -3. Add stage-specific budgets and global stop conditions. -4. Support human approval gates without requiring them by default for safe, - local experiments. -5. Emit compact progress reports with artifact IDs, not copied trace content. -6. Add CLI and Python facade entry points over the same service. - -### Verification - -```bash -uv run pytest tests/test_loop.py tests/test_e2e_release.py -q -uv run pytest -q -uv run ruff check src tests -uv run mypy src tests -``` - -Test crash/restart at every boundary, duplicate external response, unavailable -Langfuse, Codex timeout, no failures, no diagnosis, no reproducible eval, -budget exhaustion, rejected candidate, and admitted candidate. - -### Exit criteria - -An end-to-end fixture resumes after deliberate process termination and reaches -the same final pin without duplicate cases, experiments, or decisions. - -### Rollback - -Disable automatic progression and retain manual stage APIs. Durable state shows -the exact last completed stage and safe resume point. - -## 17. PR16 — End-to-end SDK facade and Codex plugin release - -Branch: `codex/sdk-hardening` -Depends on: PR15 -Risk: medium - -### Context brief - -Assemble the already-public stage APIs into the final end-to-end experience. -Every preceding PR must harden its own public imports and must not defer basic -SDK quality to this step. Do not design the facade in advance of service evidence. - -### Work - -1. Add the `Project` facade over existing services without hiding typed results. -2. Freeze public import paths and add compatibility tests. -3. Complete the OpenFlyWheel Codex plugin with focused skills: - `integrate-ofw`, `ofw-mine-failures`, `import-ofw-verifiers`, - `ofw-diagnose-failures`, `ofw-build-evals`, and `ofw-evolve-harness`. -4. Package the supported MCP servers and declare tool dependencies. -5. Add documented generic integration guidance plus a Hermes reference fixture - using Langfuse and E2B. -6. Add security/privacy documentation, threat boundaries, and secret scanning. -7. Publish migration notes from PR5 APIs and explicit non-goals. - -### Verification - -```bash -python3 ~/.codex/skills/.system/plugin-creator/scripts/validate_plugin.py plugins/openflywheel -uv run pytest -q -uv run ruff check src tests -uv run mypy src tests -uv build -``` - -Run a clean-environment install, plugin discovery test, typed example, and one -opt-in live Langfuse/E2B smoke. - -### Exit criteria - -A fresh Codex session can install the plugin, integrate an existing Langfuse-connected harness -repository without manual OFW schema authoring, run one bounded flywheel cycle, -and inspect every decision artifact. - -### Rollback - -Keep lower-level service APIs supported for one release. Revert facade/plugin -packaging without changing stored domain records. - -## 18. Cross-PR quality gates - -Every PR must satisfy: - -- tests written before non-trivial implementation; -- full test suite, Ruff, strict mypy, and `git diff --check` clean; -- no secrets, raw credentials, or production trace content committed; -- public contracts use typed classes/enums, never `Any` or metadata bags; -- one migration test for every persisted schema change; -- public imports and typed examples updated in the same PR that adds a service; -- one invalid/adversarial fixture for every admission boundary; -- compatibility maintained or removed explicitly in the same PR; -- PR body states authority gained by Codex and the deterministic control that - bounds it. - -## 19. Success metrics - -System quality: - -- mining precision/recall against reviewed production failures; -- environment-verification availability and stale-evidence rate; -- diagnosis agreement/abstention against reviewed cases; -- eval reproduction rate and flake rate; -- percentage of recurring failure clusters covered by admitted evals; -- regression-suite escape rate; -- candidate acceptance rate and improvement on sealed validation; -- cost and elapsed time per admitted harness improvement. - -SDK quality: - -- integration time for a new agent-harness repository; -- fraction of components discovered without user schema authoring; -- restart success at every stage boundary; -- zero duplicate derived records after retry; -- stable public API and plugin activation accuracy. - -## 20. Explicit non-goals - -- no OFW trace SDK, OTLP collector, trace database, or trace UI; -- no model gateway or proxy; -- no general-purpose production scheduler or warm sandbox fleet in this plan; -- no model weight training or fine-tuning; -- no generic agent framework replacing the connected harness; -- no live production mutation by mining or diagnosis tools; -- no semantic vector database before structured retrieval is measured and found - insufficient; -- no single `ofw.improve()` tool that collapses proposal and admission. - -## 21. Anti-pattern catalog - -- **Trace collector creep:** adding spans/exporters because a Langfuse field is - inconvenient. Extend the connector or require attribution instead. -- **Judge as oracle:** accepting Codex or executed-agent prose instead of environment - evidence. -- **Stage collapse:** mining records root cause, diagnosis generates evals, or - Fit edits before a gate exists. -- **Self-admission:** the agent that proposes an artifact also activates it. -- **Holdout leakage:** exposing test traces, expected outputs, or verifier - internals to candidate generation. -- **Averages hide harm:** accepting better mean reward with critical regression. -- **Mutable history:** updating a failure, cluster, suite, or pin in place. -- **Provider types in domain:** E2B, Langfuse, GitHub, or pytest classes leaking - into provider-agnostic contracts. -- **Premature orchestration:** building PR15 before individual stages are - executable and restart-safe. -- **Skill as implementation:** placing business rules only in Codex prose rather - than typed OFW validators. - -## 22. Plan mutation protocol - -When implementation evidence invalidates a step: - -1. Record the new fact in this plan under a dated `Plan amendments` section. -2. Do not silently broaden the active PR. -3. Split a PR when it gains a second independently releasable authority boundary - or exceeds one migration plus one service. -4. Insert a prerequisite PR when a required contract or deterministic validator - is missing. -5. Reorder only when dependency edges and owned files remain valid. -6. Skip a PR only when its exit criteria are already proven by committed tests; - link the evidence. -7. Abandon a direction after three evidence-backed failures of the same - assumption, preserving attempts and the selected alternative. - -## 23. First move - -Merge PR5. Then start PR6 and PR7 in parallel. Do not start diagnosis, eval -generation, clustering, or Fit until the durable signal and verification -contracts are merged. - -## 24. Adversarial review decisions - -Review completed 2026-08-25. - -Accepted: - -- Keep E2B as the only workspace backend; remove the speculative provider - abstraction until a second implementation exists. -- Harden public SDK imports incrementally rather than postponing them to the - final PR. -- Keep the final flywheel orchestrator thin and dependent on independently - executable, restart-safe stages. -- Add explicit verifier/rubric calibration to the living eval registry so OFW - covers the evaluation-maintenance capability, not only case clustering. - -Not accepted: - -- Merging workspaces, gates, and Fit into one PR. Gate construction must land - and be testable before Codex receives repository-write authority; combining - them would let proposal and admission arrive in the same change. -- Collapsing the remaining roadmap to eight PRs. The current eleven steps stay - within Blueprint's normal range and each high-risk authority boundary remains - independently reviewable and reversible. - -## 25. Plan amendments - -None yet. From 98d6cb7eea4e8a52c8cb3ce9ae9a1f41c08ab99f Mon Sep 17 00:00:00 2001 From: divo12 Date: Wed, 26 Aug 2026 08:49:05 +0530 Subject: [PATCH 16/18] replace harness builder with repository revisions --- .../openflywheel/.codex-plugin/plugin.json | 7 +- .../skills/integrate-ofw/SKILL.md | 82 --- .../integrate-ofw/references/python-api.md | 68 --- src/ofw/__init__.py | 21 +- src/ofw/contracts.py | 145 +---- src/ofw/harness.py | 498 ------------------ src/ofw/repository.py | 173 ++++++ src/ofw/runtime.py | 15 +- tests/test_harness.py | 422 --------------- tests/test_langfuse_collection.py | 9 +- tests/test_langfuse_contracts.py | 18 +- tests/test_mine.py | 3 - tests/test_repository.py | 102 ++++ tests/test_runtime.py | 113 +--- tests/test_typing.py | 2 +- 15 files changed, 316 insertions(+), 1362 deletions(-) delete mode 100644 plugins/openflywheel/skills/integrate-ofw/SKILL.md delete mode 100644 plugins/openflywheel/skills/integrate-ofw/references/python-api.md delete mode 100644 src/ofw/harness.py create mode 100644 src/ofw/repository.py delete mode 100644 tests/test_harness.py create mode 100644 tests/test_repository.py diff --git a/plugins/openflywheel/.codex-plugin/plugin.json b/plugins/openflywheel/.codex-plugin/plugin.json index 912b0c8..8c56f21 100644 --- a/plugins/openflywheel/.codex-plugin/plugin.json +++ b/plugins/openflywheel/.codex-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "openflywheel", "version": "0.1.0", - "description": "Integrate any Langfuse-connected agent harness with OpenFlyWheel and mine evidence-backed failures with Codex.", + "description": "Mine evidence-backed failures from Langfuse-connected agent harnesses with Codex.", "author": { "name": "OpenFlyWheel" }, @@ -9,13 +9,12 @@ "skills": "./skills/", "interface": { "displayName": "OpenFlyWheel", - "shortDescription": "Integrate agent harnesses and mine failures", - "longDescription": "Codex workflows for integrating any Langfuse-connected agent harness with OpenFlyWheel and mining observable failures from full trajectories.", + "shortDescription": "Mine failures from production traces", + "longDescription": "Codex workflows for mining observable failures from full Langfuse trajectories without explicit harness component mapping.", "developerName": "OpenFlyWheel", "category": "Developer Tools", "capabilities": ["Read", "Write"], "defaultPrompt": [ - "Integrate this agent harness with OpenFlyWheel.", "Mine failures from the connected OpenFlyWheel trajectory." ] } diff --git a/plugins/openflywheel/skills/integrate-ofw/SKILL.md b/plugins/openflywheel/skills/integrate-ofw/SKILL.md deleted file mode 100644 index 08a5c8b..0000000 --- a/plugins/openflywheel/skills/integrate-ofw/SKILL.md +++ /dev/null @@ -1,82 +0,0 @@ ---- -name: integrate-ofw -description: Integrates any agent harness already connected to Langfuse with OpenFlyWheel by declaring versioned harness components, attributing real runs to an immutable revision, and verifying full trace collection. Use for onboarding an agent system to OFW. Do not use for failure mining, diagnosis, harness optimization, or replacing Langfuse. ---- - -# Integrate an agent harness with OpenFlyWheel - -Add OFW to the real agent execution path without changing harness behavior. -Read [references/python-api.md](references/python-api.md) before writing OFW -code; it contains the exact supported API. - -## Principles - -1. **Instrument the real path.** Find the command, service, or worker that - actually starts the agent harness. Do not create a parallel demonstration agent. -2. **Discover components for the user.** Locate the active system prompt, tool - implementations, skills, subagent definitions, and middleware. The user - should not have to translate their repository into OFW schemas manually. -3. **Do not change behavior.** This workflow may add an OFW declaration and - revision attribution. It must not rewrite prompts, tools, skills, or harness - control flow. -4. **Keep secrets in the environment.** Never place Langfuse keys in code, - manifests, plugin files, logs, or chat. OFW records only environment-variable - names. -5. **Preserve Langfuse.** Connect the application's existing Langfuse project; - do not replace its SDK, exporter, or trace hierarchy. -6. **Verify with a real run.** Integration is complete only when one real agent - trajectory is attributed to the generated revision and collected with full - input/output content into local SQLite. - -## Workflow - -1. Inspect the repository and its dependency manager. Confirm that - `openflywheel` is available from the user's chosen package source. If it is - not resolvable, stop and ask for the intended install source; do not invent a - Git URL or published version. -2. Identify the git root and the active harness assets: - - one or more prompt files; - - named tool implementation files; - - skill files; - - subagent definitions, if present; - - middleware or lifecycle files, if present. -3. Add one small `ofw_harness.py` declaration at the git root. Follow the - reference exactly. Register every discovered active asset once. -4. Default only the primary prompt to `ofw.editable`. Keep tools, skills, - subagents, and middleware frozen unless the user explicitly authorizes OFW - to optimize them later. -5. Connect `LangfuseProject.from_env` using the deployment's real environment - name and existing Langfuse environment variables. Never read or copy their - values. -6. Process the harness to produce an immutable revision and manifest. -7. Add revision attribution around the real agent run using - `propagate_attributes` and metadata key `ofw.harness.revision`. Do not add a - second root trace. -8. Run one real agent request, flush the existing Langfuse client as required - by its runtime, then collect a UTC window containing that run with - `ofw.collect`. -9. Verify that the collection belongs to the revision, contains the expected - trace and ordered observations, has no completeness gaps, and that captured - input/output content can be read from the SQLite snapshot. - -## Stop conditions - -- Stop before editing when the active agent entry point or prompt cannot be - identified confidently. -- Stop before a live run when credentials are unavailable; report the exact - environment-variable names needed without exposing values. -- Do not report success from a generated manifest alone. -- Do not weaken full-trace capture, redact content, or silently accept missing - revision attribution. - -## Output - -Report: - -- the OFW declaration and execution path changed; -- the discovered component-to-file mapping and which files are editable; -- the immutable revision ID and manifest path; -- the real agent command/request used for verification; -- the collected trace ID, observation count, capability/gap status, and SQLite - path; -- any missing component, credential, attribution, or content evidence. diff --git a/plugins/openflywheel/skills/integrate-ofw/references/python-api.md b/plugins/openflywheel/skills/integrate-ofw/references/python-api.md deleted file mode 100644 index e3ca4da..0000000 --- a/plugins/openflywheel/skills/integrate-ofw/references/python-api.md +++ /dev/null @@ -1,68 +0,0 @@ -# Supported Python integration - -Use only this API surface. Adapt paths and names to the inspected agent-harness -repository; do not invent components that are not active. - -```python -from datetime import UTC, datetime, timedelta -from pathlib import Path - -from ofw import Harness, LangfuseProject, Tool, TraceWindow, ofw, propagate_attributes - -ROOT = Path(__file__).resolve().parent - -project = LangfuseProject.from_env(environment="production") -harness = Harness("agent-harness", root=ROOT) -harness.connect_prompt(ofw.editable(Path("path/to/system-prompt.md"))) -harness.connect_tools( - Tool(name="search", source=Path("path/to/search-tool.py")), -) -harness.connect_skills(Path("path/to/skill/SKILL.md")) -harness.connect_middleware(Path("path/to/middleware.py")) -harness.connect_observability(project) -revision = harness.process() -``` - -Register only component kinds that exist. For named subagents use -`Subagent(name=..., source=...)` with `connect_subagents`. Tool and subagent -names must be lowercase identifiers accepted by OFW. Every registered path is -relative to the git root and may belong to only one component. - -Run the real agent entry point inside revision attribution: - -```python -with propagate_attributes( - metadata={"ofw.harness.revision": str(revision.id)}, -): - run_real_agent_request() -``` - -The surrounding application remains responsible for its existing Langfuse -client lifecycle and flush behavior. - -Collect the verified run after Langfuse ingestion: - -```python -end = datetime.now(UTC) -collection = ofw.collect( - revision, - window=TraceWindow(end - timedelta(minutes=30), end), -) -``` - -The default snapshot path is `/.ofw/collection.sqlite`. A ready -integration has an exactly attributed trace, ordered observations, captured -input/output content, and no trace gaps. Use OFW's public -`search_observation_content`, `read_trace_observations`, and -`read_observation_content` helpers to prove content is queryable. - -Required environment variables normally remain: - -```text -LANGFUSE_PUBLIC_KEY -LANGFUSE_SECRET_KEY -LANGFUSE_BASE_URL -``` - -`LangfuseProject.from_env` accepts alternate environment-variable names when -the deployment already uses them. Pass the names, never their secret values. diff --git a/src/ofw/__init__.py b/src/ofw/__init__.py index 679e740..060a049 100644 --- a/src/ofw/__init__.py +++ b/src/ofw/__init__.py @@ -13,20 +13,14 @@ ) from ofw.contracts import ( - AssetAccess, - ComponentKind, GitCommit, - HarnessAsset, - HarnessComponent, HarnessErrorCode, HarnessRevision, HarnessRevisionId, HarnessValidationError, RepositorySnapshot, Sha256Digest, - WorkspaceFile, ) -from ofw.harness import EditableFile, Harness, Subagent, Tool, editable from ofw.mcp import FailureMiningMcpServer from ofw.mine import ( AdaptationRequest, @@ -91,6 +85,7 @@ read_trace_observations, search_observation_content, ) +from ofw.repository import process_repository from ofw.runtime import ( CanaryCase, CaseId, @@ -120,9 +115,6 @@ class _OfwNamespace: CanaryCase = CanaryCase CaseId = CaseId - def editable(self, path: Path) -> EditableFile: - return editable(path) - def collect( self, revision: HarnessRevision, @@ -157,13 +149,11 @@ def read_observation_content( ofw = _OfwNamespace() __all__ = [ - "AssetAccess", "AdaptationRequest", "AdaptationResult", "BehaviorObservation", "CanaryCase", "CaseId", - "ComponentKind", "CollectionError", "CollectionErrorCode", "CollectionResult", @@ -174,7 +164,6 @@ def read_observation_content( "Confidence", "ConstraintKind", "E2BSandbox", - "EditableFile", "EnvironmentCheckId", "EnvironmentCheckRequest", "EnvironmentSource", @@ -194,9 +183,6 @@ def read_observation_content( "FailureSourceId", "FailureSourceKind", "GitCommit", - "Harness", - "HarnessAsset", - "HarnessComponent", "HarnessErrorCode", "HarnessRevision", "HarnessRevisionId", @@ -227,10 +213,8 @@ def read_observation_content( "RunResult", "RunStatus", "Sha256Digest", - "Subagent", "TaskConstraint", "TaskId", - "Tool", "ToolAccess", "ToolCapability", "ToolName", @@ -238,13 +222,12 @@ def read_observation_content( "TraceMiningCase", "VerifierResult", "VerifierVerdict", - "WorkspaceFile", "collect", - "editable", "get_client", "is_default_export_span", "observe", "ofw", + "process_repository", "propagate_attributes", "read_observation_content", "read_trace_observations", diff --git a/src/ofw/contracts.py b/src/ofw/contracts.py index 28f9741..ccb0832 100644 --- a/src/ofw/contracts.py +++ b/src/ofw/contracts.py @@ -1,4 +1,4 @@ -"""Immutable, language-neutral harness component contracts.""" +"""Immutable, language-neutral repository revision contracts.""" from __future__ import annotations @@ -14,43 +14,14 @@ class HarnessSchemaVersion(IntEnum): V1 = 1 -class ComponentKind(StrEnum): - PROMPT = "prompt" - TOOL = "tool" - SKILL = "skill" - SUBAGENT = "subagent" - MIDDLEWARE = "middleware" - - -class AssetAccess(StrEnum): - FROZEN = "frozen" - FIT_EDITABLE = "fit_editable" - - class HarnessErrorCode(StrEnum): INVALID_NAME = "invalid_name" INVALID_SOURCE = "invalid_source" ROOT_NOT_FOUND = "root_not_found" ROOT_NOT_DIRECTORY = "root_not_directory" - PROMPT_REQUIRED = "prompt_required" - MISSING_ASSET = "missing_asset" - PATH_OUTSIDE_ROOT = "path_outside_root" - NOT_A_FILE = "not_a_file" - DUPLICATE_ASSET = "duplicate_asset" - CONFLICTING_ACCESS = "conflicting_access" - COMPONENT_OVERLAP = "component_overlap" - INVALID_TOOL_NAME = "invalid_tool_name" - DUPLICATE_TOOL = "duplicate_tool" - INVALID_SUBAGENT_NAME = "invalid_subagent_name" - DUPLICATE_SUBAGENT = "duplicate_subagent" GIT_REPOSITORY_REQUIRED = "git_repository_required" GIT_COMMAND_FAILED = "git_command_failed" MANIFEST_WRITE_FAILED = "manifest_write_failed" - SENSITIVE_ASSET = "sensitive_asset" - RUNTIME_INCOMPLETE = "runtime_incomplete" - CANARY_FAILED = "canary_failed" - DUPLICATE_VERIFIER = "duplicate_verifier" - RUNTIME_INVALID = "runtime_invalid" class HarnessValidationError(Exception): @@ -88,26 +59,6 @@ def __str__(self) -> str: return self.value -@dataclass(frozen=True, slots=True) -class WorkspaceFile: - relative_path: Path - - -@dataclass(frozen=True, slots=True) -class HarnessAsset: - name: str | None - access: AssetAccess - source: WorkspaceFile - digest: Sha256Digest - - -@dataclass(frozen=True, slots=True) -class HarnessComponent: - kind: ComponentKind - assets: tuple[HarnessAsset, ...] - digest: Sha256Digest - - @dataclass(frozen=True, slots=True) class RepositorySnapshot: commit: GitCommit @@ -115,31 +66,12 @@ class RepositorySnapshot: dirty_digest: Sha256Digest | None -@dataclass(frozen=True, slots=True) -class RuntimeConfiguration: - execution: Sha256Digest - lifecycle: Sha256Digest - verifiers: tuple[Sha256Digest, ...] - - def canonical_json(self) -> str: - verifiers = ",".join(_quote(str(verifier)) for verifier in self.verifiers) - return ( - "{" - f'"execution":{_quote(str(self.execution))},' - f'"lifecycle":{_quote(str(self.lifecycle))},' - f'"verifiers":[{verifiers}]' - "}" - ) - - @dataclass(frozen=True, slots=True) class HarnessRevisionContent: schema_version: HarnessSchemaVersion harness_name: str repository: RepositorySnapshot - components: tuple[HarnessComponent, ...] observability: LangfuseConnectionManifest | None - runtime: RuntimeConfiguration | None def canonical_json(self) -> str: return _render_content(self) @@ -152,47 +84,13 @@ class HarnessRevision: harness_name: str root: Path repository: RepositorySnapshot - components: tuple[HarnessComponent, ...] observability: LangfuseConnectionManifest | None - runtime: RuntimeConfiguration | None - canary_digest: Sha256Digest | None @property def manifest_path(self) -> Path: return self.root / ".ofw" / "revisions" / str(self.id) / "manifest.json" - @property - def canary_path(self) -> Path: - return self.manifest_path.with_name("canary.json") - - @property - def assets(self) -> tuple[HarnessAsset, ...]: - return tuple(asset for component in self.components for asset in component.assets) - - @property - def editable_files(self) -> tuple[Path, ...]: - return tuple( - asset.source.relative_path - for asset in self.assets - if asset.access is AssetAccess.FIT_EDITABLE - ) - - @property - def frozen_files(self) -> tuple[Path, ...]: - return tuple( - asset.source.relative_path - for asset in self.assets - if asset.access is AssetAccess.FROZEN - ) - - def component(self, kind: ComponentKind) -> HarnessComponent | None: - for component in self.components: - if component.kind is kind: - return component - return None - def to_json(self) -> str: - components = ",".join(_render_component(component) for component in self.components) return ( "{" f'"schema_version":{int(self.schema_version)},' @@ -200,24 +98,18 @@ def to_json(self) -> str: f'"harness_name":{_quote(self.harness_name)},' f'"root":{_quote(self.root.as_posix())},' f'"repository":{_render_repository(self.repository)},' - f'"observability":{_render_observability(self.observability)},' - f'"runtime":{_render_runtime(self.runtime)},' - f'"canary_digest":{_render_digest(self.canary_digest)},' - f'"components":[{components}]' + f'"observability":{_render_observability(self.observability)}' "}" ) def _render_content(content: HarnessRevisionContent) -> str: - components = ",".join(_render_component(component) for component in content.components) return ( "{" f'"schema_version":{int(content.schema_version)},' f'"harness_name":{_quote(content.harness_name)},' f'"repository":{_render_repository(content.repository)},' - f'"observability":{_render_observability(content.observability)},' - f'"runtime":{_render_runtime(content.runtime)},' - f'"components":[{components}]' + f'"observability":{_render_observability(content.observability)}' "}" ) @@ -240,36 +132,5 @@ def _render_observability(connection: LangfuseConnectionManifest | None) -> str: return "null" if connection is None else connection.to_json() -def _render_runtime(runtime: RuntimeConfiguration | None) -> str: - return "null" if runtime is None else runtime.canonical_json() - - -def _render_digest(digest: Sha256Digest | None) -> str: - return "null" if digest is None else _quote(str(digest)) - - -def _render_component(component: HarnessComponent) -> str: - assets = ",".join(_render_asset(asset) for asset in component.assets) - return ( - "{" - f'"kind":{_quote(component.kind.value)},' - f'"digest":{_quote(str(component.digest))},' - f'"assets":[{assets}]' - "}" - ) - - -def _render_asset(asset: HarnessAsset) -> str: - name = "null" if asset.name is None else _quote(asset.name) - return ( - "{" - f'"name":{name},' - f'"access":{_quote(asset.access.value)},' - f'"source":{{"relative_path":{_quote(asset.source.relative_path.as_posix())}}},' - f'"digest":{_quote(str(asset.digest))}' - "}" - ) - - def _quote(value: str) -> str: return json.dumps(value, ensure_ascii=False, separators=(",", ":")) diff --git a/src/ofw/harness.py b/src/ofw/harness.py deleted file mode 100644 index 831da84..0000000 --- a/src/ofw/harness.py +++ /dev/null @@ -1,498 +0,0 @@ -"""Compile a file-level harness workspace into an immutable revision.""" - -from __future__ import annotations - -import hashlib -import logging -import os -import re -import subprocess # nosec B404 -import tempfile -from dataclasses import dataclass, field -from pathlib import Path - -from ofw.contracts import ( - AssetAccess, - ComponentKind, - GitCommit, - HarnessAsset, - HarnessComponent, - HarnessErrorCode, - HarnessRevision, - HarnessRevisionContent, - HarnessRevisionId, - HarnessSchemaVersion, - HarnessValidationError, - RepositorySnapshot, - RuntimeConfiguration, - Sha256Digest, - WorkspaceFile, -) -from ofw.observability.langfuse.contracts import LangfuseProject -from ofw.runtime import ( - CanaryCase, - CanaryReport, - ExecutionEnvironment, - LifecycleAdapter, - VerifierAdapter, - run_canary, - runtime_configuration, -) - -logger = logging.getLogger(__name__) - -_NAME_PATTERN = re.compile(r"[a-z0-9]+(?:-[a-z0-9]+)*") -_NAMED_SOURCE_PATTERN = re.compile(r"[a-z][a-z0-9_-]*") - - -@dataclass(frozen=True, slots=True) -class EditableFile: - path: Path - - -@dataclass(frozen=True, slots=True) -class Tool: - name: str - source: Path | EditableFile - - def __post_init__(self) -> None: - _validate_named_source(self.name, self.source, HarnessErrorCode.INVALID_TOOL_NAME) - - -@dataclass(frozen=True, slots=True) -class Subagent: - name: str - source: Path | EditableFile - - def __post_init__(self) -> None: - _validate_named_source(self.name, self.source, HarnessErrorCode.INVALID_SUBAGENT_NAME) - - -@dataclass(frozen=True, slots=True) -class _FileRegistration: - component: ComponentKind - name: str | None - path: Path - access: AssetAccess - - -@dataclass(frozen=True, slots=True) -class _CompiledAsset: - component: ComponentKind - asset: HarnessAsset - - -def editable(path: Path) -> EditableFile: - """Grant Fit authority to edit one workspace file.""" - if not isinstance(path, Path): - raise HarnessValidationError(HarnessErrorCode.INVALID_SOURCE, repr(path)) - return EditableFile(path=path) - - -@dataclass(slots=True) -class Harness: - """Mutable component registry; ``process`` returns an immutable revision.""" - - name: str - root: Path - _files: list[_FileRegistration] = field(default_factory=list, init=False, repr=False) - _observability: LangfuseProject | None = field(default=None, init=False, repr=False) - _execution: ExecutionEnvironment | None = field(default=None, init=False, repr=False) - _lifecycle: LifecycleAdapter | None = field(default=None, init=False, repr=False) - _verifiers: list[VerifierAdapter] = field(default_factory=list, init=False, repr=False) - - def __post_init__(self) -> None: - if _NAME_PATTERN.fullmatch(self.name) is None: - raise HarnessValidationError(HarnessErrorCode.INVALID_NAME, self.name) - if not isinstance(self.root, Path): - raise HarnessValidationError(HarnessErrorCode.INVALID_SOURCE, repr(self.root)) - - def connect_prompt(self, *sources: Path | EditableFile) -> Harness: - self._register_files(ComponentKind.PROMPT, sources) - return self - - def connect_tools(self, *tools: Tool) -> Harness: - for tool in tools: - if any( - registration.component is ComponentKind.TOOL and registration.name == tool.name - for registration in self._files - ): - raise HarnessValidationError(HarnessErrorCode.DUPLICATE_TOOL, tool.name) - self._files.append(_registration(ComponentKind.TOOL, tool.source, tool.name)) - return self - - def connect_skills(self, *sources: Path | EditableFile) -> Harness: - self._register_files(ComponentKind.SKILL, sources) - return self - - def connect_subagents(self, *subagents: Subagent) -> Harness: - for subagent in subagents: - if any( - registration.component is ComponentKind.SUBAGENT - and registration.name == subagent.name - for registration in self._files - ): - raise HarnessValidationError( - HarnessErrorCode.DUPLICATE_SUBAGENT, - subagent.name, - ) - self._files.append( - _registration(ComponentKind.SUBAGENT, subagent.source, subagent.name) - ) - return self - - def connect_middleware(self, *sources: Path | EditableFile) -> Harness: - self._register_files(ComponentKind.MIDDLEWARE, sources) - return self - - def connect_observability(self, project: LangfuseProject) -> Harness: - self._observability = project - return self - - def connect_execute(self, environment: ExecutionEnvironment) -> Harness: - self._execution = environment - return self - - def connect_lifecycle(self, lifecycle: LifecycleAdapter) -> Harness: - self._lifecycle = lifecycle - return self - - def connect_verifiers(self, *verifiers: VerifierAdapter) -> Harness: - for verifier in verifiers: - if any(existing.name == verifier.name for existing in self._verifiers): - raise HarnessValidationError(HarnessErrorCode.DUPLICATE_VERIFIER, verifier.name) - self._verifiers.append(verifier) - return self - - def _register_files( - self, - component: ComponentKind, - sources: tuple[Path | EditableFile, ...], - ) -> None: - for source in sources: - self._files.append(_registration(component, source, None)) - - def process(self, *, canary: CanaryCase | None = None) -> HarnessRevision: - logger.debug("Compiling harness revision: %s", self.name) - root = _resolve_root(self.root) - if not _has_component(self._files, ComponentKind.PROMPT): - raise HarnessValidationError(HarnessErrorCode.PROMPT_REQUIRED, self.name) - - components = _compile_components(root, self._files) - repository = _snapshot_repository(root) - runtime = self._runtime(root) - content = HarnessRevisionContent( - schema_version=HarnessSchemaVersion.V1, - harness_name=self.name, - repository=repository, - components=components, - observability=(None if self._observability is None else self._observability.manifest()), - runtime=runtime, - ) - revision = _revision_from_content(content, root) - report: CanaryReport | None = None - if canary is not None: - if self._execution is None or self._lifecycle is None or not self._verifiers: - raise HarnessValidationError(HarnessErrorCode.RUNTIME_INCOMPLETE, self.name) - report = run_canary( - revision, - canary, - self._execution, - self._lifecycle, - tuple(self._verifiers), - ) - if not report.passed: - _write_canary(revision, report) - raise HarnessValidationError(HarnessErrorCode.CANARY_FAILED, canary.id.value) - revision = _revision_from_content(content, root, report.digest) - _write_manifest(revision) - if report is not None: - _write_canary(revision, report) - logger.debug("Compiled harness revision %s", revision.id) - return revision - - def _runtime(self, root: Path) -> RuntimeConfiguration | None: - connections = ( - self._execution is not None, - self._lifecycle is not None, - bool(self._verifiers), - ) - if not any(connections): - return None - if not all(connections) or self._execution is None or self._lifecycle is None: - raise HarnessValidationError(HarnessErrorCode.RUNTIME_INCOMPLETE, self.name) - try: - return runtime_configuration( - root, - self._execution, - self._lifecycle, - tuple(self._verifiers), - ) - except ValueError as error: - raise HarnessValidationError(HarnessErrorCode.RUNTIME_INVALID, self.name) from error - - -def _has_component(registrations: list[_FileRegistration], kind: ComponentKind) -> bool: - return any(registration.component is kind for registration in registrations) - - -def _revision_from_content( - content: HarnessRevisionContent, - root: Path, - canary_digest: Sha256Digest | None = None, -) -> HarnessRevision: - content_digest = _digest_text(content.canonical_json()) - return HarnessRevision( - schema_version=content.schema_version, - id=HarnessRevisionId(f"ofw_{content_digest.value[7:]}"), - harness_name=content.harness_name, - root=root, - repository=content.repository, - components=content.components, - observability=content.observability, - runtime=content.runtime, - canary_digest=canary_digest, - ) - - -def _resolve_root(root: Path) -> Path: - try: - resolved = root.expanduser().resolve(strict=True) - except FileNotFoundError as error: - raise HarnessValidationError(HarnessErrorCode.ROOT_NOT_FOUND, str(root)) from error - if not resolved.is_dir(): - raise HarnessValidationError(HarnessErrorCode.ROOT_NOT_DIRECTORY, str(resolved)) - return resolved - - -def _compile_components( - root: Path, - registrations: list[_FileRegistration], -) -> tuple[HarnessComponent, ...]: - compiled: list[_CompiledAsset] = [] - for registration in registrations: - resolved, relative = _resolve_file(root, registration.path) - compiled.append( - _CompiledAsset( - component=registration.component, - asset=HarnessAsset( - name=registration.name, - access=registration.access, - source=WorkspaceFile(relative_path=relative), - digest=_digest_file(resolved), - ), - ) - ) - compiled.sort(key=_compiled_asset_sort_key) - _validate_component_boundaries(compiled) - - components: list[HarnessComponent] = [] - for kind in ComponentKind: - assets = tuple(item.asset for item in compiled if item.component is kind) - if not assets: - continue - components.append( - HarnessComponent( - kind=kind, - assets=assets, - digest=_component_digest(kind, assets), - ) - ) - return tuple(components) - - -def _validate_component_boundaries(compiled: list[_CompiledAsset]) -> None: - for index, item in enumerate(compiled): - for existing in compiled[:index]: - if existing.asset.source.relative_path != item.asset.source.relative_path: - continue - if existing.component is not item.component: - raise HarnessValidationError( - HarnessErrorCode.COMPONENT_OVERLAP, - item.asset.source.relative_path.as_posix(), - ) - if ( - item.component in (ComponentKind.TOOL, ComponentKind.SUBAGENT) - and existing.asset.name != item.asset.name - ): - continue - code = ( - HarnessErrorCode.DUPLICATE_ASSET - if existing.asset.access is item.asset.access - else HarnessErrorCode.CONFLICTING_ACCESS - ) - raise HarnessValidationError(code, item.asset.source.relative_path.as_posix()) - - -def _component_digest( - kind: ComponentKind, - assets: tuple[HarnessAsset, ...], -) -> Sha256Digest: - fields = [kind.value] - for asset in assets: - fields.extend( - ( - asset.name or "", - asset.access.value, - asset.source.relative_path.as_posix(), - str(asset.digest), - ) - ) - return _digest_text("\0".join(fields)) - - -def _compiled_asset_sort_key(item: _CompiledAsset) -> tuple[str, str, str]: - return ( - item.component.value, - item.asset.source.relative_path.as_posix(), - item.asset.name or "", - ) - - -def _registration( - component: ComponentKind, - source: Path | EditableFile, - name: str | None, -) -> _FileRegistration: - if isinstance(source, EditableFile): - return _FileRegistration(component, name, source.path, AssetAccess.FIT_EDITABLE) - if isinstance(source, Path): - return _FileRegistration(component, name, source, AssetAccess.FROZEN) - raise HarnessValidationError(HarnessErrorCode.INVALID_SOURCE, repr(source)) - - -def _validate_named_source( - name: str, - source: Path | EditableFile, - error_code: HarnessErrorCode, -) -> None: - if _NAMED_SOURCE_PATTERN.fullmatch(name) is None: - raise HarnessValidationError(error_code, name) - if not isinstance(source, (Path, EditableFile)): - raise HarnessValidationError(HarnessErrorCode.INVALID_SOURCE, repr(source)) - - -def _resolve_file(root: Path, source: Path) -> tuple[Path, Path]: - if _is_sensitive_path(source): - raise HarnessValidationError(HarnessErrorCode.SENSITIVE_ASSET, str(source)) - candidate = source if source.is_absolute() else root / source - try: - resolved = candidate.resolve(strict=True) - except FileNotFoundError as error: - raise HarnessValidationError(HarnessErrorCode.MISSING_ASSET, str(source)) from error - try: - relative = resolved.relative_to(root) - except ValueError as error: - raise HarnessValidationError( - HarnessErrorCode.PATH_OUTSIDE_ROOT, - str(source), - ) from error - if not resolved.is_file(): - raise HarnessValidationError(HarnessErrorCode.NOT_A_FILE, str(source)) - return resolved, Path(relative.as_posix()) - - -def _is_sensitive_path(path: Path) -> bool: - return any( - part == ".env" or (part.startswith(".env.") and part != ".env.example") - for part in path.parts - ) - - -def _snapshot_repository(root: Path) -> RepositorySnapshot: - top_level = _run_git(root, "rev-parse", "--show-toplevel", repository_probe=True) - try: - git_root = Path(top_level.decode().strip()).resolve(strict=True) - except (UnicodeDecodeError, FileNotFoundError) as error: - raise HarnessValidationError( - HarnessErrorCode.GIT_REPOSITORY_REQUIRED, - str(root), - ) from error - if git_root != root: - raise HarnessValidationError(HarnessErrorCode.GIT_REPOSITORY_REQUIRED, str(root)) - - commit_bytes = _run_git(root, "rev-parse", "HEAD") - try: - commit = GitCommit(commit_bytes.decode().strip()) - except UnicodeDecodeError as error: - raise HarnessValidationError( - HarnessErrorCode.GIT_COMMAND_FAILED, "rev-parse HEAD" - ) from error - diff = _run_git(root, "diff", "--binary", "--no-ext-diff", "HEAD", "--") - return RepositorySnapshot( - commit=commit, - is_dirty=bool(diff), - dirty_digest=_digest_bytes(diff) if diff else None, - ) - - -def _run_git( - root: Path, - *arguments: str, - repository_probe: bool = False, -) -> bytes: - try: - # Every argument is selected internally; the validated root is one argv value. - result: subprocess.CompletedProcess[bytes] = subprocess.run( # nosec B603 - ("git", "-C", str(root), *arguments), - check=False, - capture_output=True, - ) - except OSError as error: - raise HarnessValidationError(HarnessErrorCode.GIT_COMMAND_FAILED, arguments[0]) from error - if result.returncode != 0: - code = ( - HarnessErrorCode.GIT_REPOSITORY_REQUIRED - if repository_probe - else HarnessErrorCode.GIT_COMMAND_FAILED - ) - raise HarnessValidationError(code, arguments[0]) - return result.stdout - - -def _digest_file(path: Path) -> Sha256Digest: - try: - return _digest_bytes(path.read_bytes()) - except OSError as error: - raise HarnessValidationError(HarnessErrorCode.MISSING_ASSET, str(path)) from error - - -def _digest_text(value: str) -> Sha256Digest: - return _digest_bytes(value.encode()) - - -def _digest_bytes(value: bytes) -> Sha256Digest: - return Sha256Digest(f"sha256:{hashlib.sha256(value).hexdigest()}") - - -def _write_manifest(revision: HarnessRevision) -> None: - _write_revision_file(revision.manifest_path, f"{revision.to_json()}\n") - - -def _write_canary(revision: HarnessRevision, report: CanaryReport) -> None: - _write_revision_file(revision.canary_path, f"{report.to_json()}\n") - - -def _write_revision_file(path: Path, payload: str) -> None: - try: - path.parent.mkdir(parents=True, exist_ok=True) - descriptor, temporary_name = tempfile.mkstemp( - dir=path.parent, - prefix=f".{path.stem}-", - suffix=".json", - text=True, - ) - temporary_path = Path(temporary_name) - try: - with os.fdopen(descriptor, "w", encoding="utf-8") as stream: - stream.write(payload) - stream.flush() - os.fsync(stream.fileno()) - temporary_path.replace(path) - finally: - temporary_path.unlink(missing_ok=True) - except OSError as error: - raise HarnessValidationError( - HarnessErrorCode.MANIFEST_WRITE_FAILED, - str(path), - ) from error diff --git a/src/ofw/repository.py b/src/ofw/repository.py new file mode 100644 index 0000000..de1cfc5 --- /dev/null +++ b/src/ofw/repository.py @@ -0,0 +1,173 @@ +"""Turn a complete git repository into an immutable harness revision.""" + +from __future__ import annotations + +import hashlib +import os +import re +import subprocess # nosec B404 +import tempfile +from pathlib import Path + +from ofw.contracts import ( + GitCommit, + HarnessErrorCode, + HarnessRevision, + HarnessRevisionContent, + HarnessRevisionId, + HarnessSchemaVersion, + HarnessValidationError, + RepositorySnapshot, + Sha256Digest, +) +from ofw.observability.langfuse.contracts import LangfuseProject + +_NAME_PATTERN = re.compile(r"[a-z0-9]+(?:-[a-z0-9]+)*") + + +def process_repository( + name: str, + root: Path, + *, + traces: LangfuseProject | None = None, +) -> HarnessRevision: + """Snapshot a whole agent-harness repository without component mapping.""" + if _NAME_PATTERN.fullmatch(name) is None: + raise HarnessValidationError(HarnessErrorCode.INVALID_NAME, name) + selected_root = _resolve_root(root) + content = HarnessRevisionContent( + schema_version=HarnessSchemaVersion.V1, + harness_name=name, + repository=_snapshot_repository(selected_root), + observability=None if traces is None else traces.manifest(), + ) + digest = _digest(content.canonical_json().encode()) + revision = HarnessRevision( + schema_version=content.schema_version, + id=HarnessRevisionId(f"ofw_{digest.value[7:]}"), + harness_name=content.harness_name, + root=selected_root, + repository=content.repository, + observability=content.observability, + ) + _write_manifest(revision) + return revision + + +def _resolve_root(root: Path) -> Path: + if not isinstance(root, Path): + raise HarnessValidationError(HarnessErrorCode.INVALID_SOURCE, repr(root)) + try: + resolved = root.expanduser().resolve(strict=True) + except FileNotFoundError as error: + raise HarnessValidationError(HarnessErrorCode.ROOT_NOT_FOUND, str(root)) from error + if not resolved.is_dir(): + raise HarnessValidationError(HarnessErrorCode.ROOT_NOT_DIRECTORY, str(resolved)) + return resolved + + +def _snapshot_repository(root: Path) -> RepositorySnapshot: + top_level = _run_git(root, "rev-parse", "--show-toplevel", repository_probe=True) + try: + git_root = Path(top_level.decode().strip()).resolve(strict=True) + except (UnicodeDecodeError, FileNotFoundError) as error: + raise HarnessValidationError( + HarnessErrorCode.GIT_REPOSITORY_REQUIRED, + str(root), + ) from error + if git_root != root: + raise HarnessValidationError(HarnessErrorCode.GIT_REPOSITORY_REQUIRED, str(root)) + commit_bytes = _run_git(root, "rev-parse", "HEAD") + try: + commit = GitCommit(commit_bytes.decode().strip()) + except UnicodeDecodeError as error: + raise HarnessValidationError( + HarnessErrorCode.GIT_COMMAND_FAILED, + "rev-parse HEAD", + ) from error + dirty = _dirty_payload(root) + return RepositorySnapshot( + commit=commit, + is_dirty=bool(dirty), + dirty_digest=None if not dirty else _digest(dirty), + ) + + +def _dirty_payload(root: Path) -> bytes: + payload = bytearray(_run_git(root, "diff", "--binary", "--no-ext-diff", "HEAD", "--")) + untracked = _run_git(root, "ls-files", "--others", "--exclude-standard", "-z") + for encoded_path in sorted(item for item in untracked.split(b"\0") if item): + relative = Path(os.fsdecode(encoded_path)) + if _ignored_internal_path(relative): + continue + path = root / relative + content = os.fsencode(os.readlink(path)) if path.is_symlink() else path.read_bytes() + payload.extend(b"\0untracked\0") + payload.extend(encoded_path) + payload.extend(b"\0") + payload.extend(content) + return bytes(payload) + + +def _ignored_internal_path(path: Path) -> bool: + return bool(path.parts) and ( + path.parts[0] == ".ofw" + or any(part == ".env" or part.startswith(".env.") for part in path.parts) + ) + + +def _run_git( + root: Path, + *arguments: str, + repository_probe: bool = False, +) -> bytes: + try: + result: subprocess.CompletedProcess[bytes] = subprocess.run( # nosec B603 + ("git", "-C", str(root), *arguments), + check=False, + capture_output=True, + ) + except OSError as error: + raise HarnessValidationError( + HarnessErrorCode.GIT_COMMAND_FAILED, + arguments[0], + ) from error + if result.returncode != 0: + code = ( + HarnessErrorCode.GIT_REPOSITORY_REQUIRED + if repository_probe + else HarnessErrorCode.GIT_COMMAND_FAILED + ) + raise HarnessValidationError(code, arguments[0]) + return result.stdout + + +def _digest(value: bytes) -> Sha256Digest: + return Sha256Digest(f"sha256:{hashlib.sha256(value).hexdigest()}") + + +def _write_manifest(revision: HarnessRevision) -> None: + path = revision.manifest_path + payload = f"{revision.to_json()}\n" + try: + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp( + dir=path.parent, + prefix=f".{path.stem}-", + suffix=".json", + text=True, + ) + temporary_path = Path(temporary_name) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + temporary_path.replace(path) + finally: + temporary_path.unlink(missing_ok=True) + except OSError as error: + raise HarnessValidationError( + HarnessErrorCode.MANIFEST_WRITE_FAILED, + str(path), + ) from error diff --git a/src/ofw/runtime.py b/src/ofw/runtime.py index 4c12b06..9d93825 100644 --- a/src/ofw/runtime.py +++ b/src/ofw/runtime.py @@ -21,7 +21,7 @@ from pydantic import TypeAdapter -from ofw.contracts import HarnessRevision, RuntimeConfiguration, Sha256Digest +from ofw.contracts import HarnessRevision, Sha256Digest _NAME_PATTERN = re.compile(r"[a-z][a-z0-9_-]*") _ENVIRONMENT_PATTERN = re.compile(r"[A-Z_][A-Z0-9_]*") @@ -393,19 +393,6 @@ def to_json(self) -> str: _CANARY_ADAPTER: TypeAdapter[CanaryReport] = TypeAdapter(CanaryReport) -def runtime_configuration( - root: Path, - execution: ExecutionEnvironment, - lifecycle: LifecycleAdapter, - verifiers: tuple[VerifierAdapter, ...], -) -> RuntimeConfiguration: - return RuntimeConfiguration( - execution.fingerprint(root), - lifecycle.fingerprint(root), - tuple(verifier.fingerprint(root) for verifier in verifiers), - ) - - def run_canary( revision: HarnessRevision, case: CanaryCase, diff --git a/tests/test_harness.py b/tests/test_harness.py deleted file mode 100644 index bf66af0..0000000 --- a/tests/test_harness.py +++ /dev/null @@ -1,422 +0,0 @@ -"""Component-observable harness revision behavior.""" - -from __future__ import annotations - -import subprocess -import sys -from dataclasses import FrozenInstanceError -from pathlib import Path - -import pytest - -from ofw import ( - AssetAccess, - ComponentKind, - EditableFile, - Harness, - HarnessAsset, - HarnessComponent, - HarnessErrorCode, - HarnessRevision, - HarnessValidationError, - Subagent, - Tool, - WorkspaceFile, - ofw, -) - - -def _run_git(root: Path, *arguments: str) -> None: - subprocess.run( - ("git", "-C", str(root), *arguments), - check=True, - capture_output=True, - text=True, - ) - - -def _repository(tmp_path: Path) -> Path: - root = tmp_path / "fixtureco-agent" - root.mkdir() - (root / "prompt.md").write_text("Be accurate.\n", encoding="utf-8") - _run_git(root, "init", "-q") - _run_git(root, "config", "user.email", "fixture@example.test") - _run_git(root, "config", "user.name", "FixtureCo") - _run_git(root, "add", ".") - _run_git(root, "commit", "-qm", "fixture baseline") - return root - - -def _configured_harness(root: Path) -> Harness: - harness = Harness("fixtureco-research-agent", root=root) - harness.connect_prompt(ofw.editable(Path("prompt.md"))) - return harness - - -def _required_component(revision: HarnessRevision, kind: ComponentKind) -> HarnessComponent: - component = revision.component(kind) - assert component is not None - return component - - -def test_process_creates_typed_immutable_revision_and_manifest(tmp_path: Path) -> None: - root = _repository(tmp_path) - harness = _configured_harness(root) - - assert not (root / ".ofw").exists() - revision = harness.process() - - assert isinstance(revision, HarnessRevision) - assert all(isinstance(component, HarnessComponent) for component in revision.components) - assert all(isinstance(asset, HarnessAsset) for asset in revision.assets) - assert all(isinstance(asset.source, WorkspaceFile) for asset in revision.assets) - assert revision.editable_files == (Path("prompt.md"),) - assert ( - revision.manifest_path == root / ".ofw" / "revisions" / str(revision.id) / "manifest.json" - ) - assert revision.manifest_path.read_text(encoding="utf-8") == f"{revision.to_json()}\n" - subprocess.run( - (sys.executable, "-m", "json.tool", str(revision.manifest_path)), - check=True, - capture_output=True, - text=True, - ) - with pytest.raises(FrozenInstanceError): - revision.harness_name = "changed" # type: ignore[misc] - - -def test_process_records_five_file_level_components_for_polyglot_agent(tmp_path: Path) -> None: - root = _repository(tmp_path) - files = ( - Path("tools/search.ts"), - Path("tools/worker.go"), - Path("skills/research/SKILL.md"), - Path("subagents/reviewer.yaml"), - Path("middleware/retry.ts"), - ) - for relative_path in files: - path = root / relative_path - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(f"fixture: {relative_path.as_posix()}\n", encoding="utf-8") - - harness = _configured_harness(root) - harness.connect_tools( - Tool(name="search", source=ofw.editable(Path("tools/search.ts"))), - Tool(name="worker", source=ofw.editable(Path("tools/worker.go"))), - ) - harness.connect_skills(ofw.editable(Path("skills/research/SKILL.md"))) - harness.connect_subagents( - Subagent( - name="reviewer", - source=ofw.editable(Path("subagents/reviewer.yaml")), - ) - ) - harness.connect_middleware(ofw.editable(Path("middleware/retry.ts"))) - - revision = harness.process() - - assert {component.kind for component in revision.components} == set(ComponentKind) - assert len(revision.components) == 5 - assert all(component.assets for component in revision.components) - assert all(str(component.digest).startswith("sha256:") for component in revision.components) - assert Path("tools/search.ts") in revision.editable_files - assert Path("tools/worker.go") in revision.editable_files - assert _required_component(revision, ComponentKind.SUBAGENT).assets[0].name == "reviewer" - - -def test_tool_object_preserves_name_and_source(tmp_path: Path) -> None: - root = _repository(tmp_path) - implementation = root / "search.ts" - implementation.write_text("export const search = () => [];\n", encoding="utf-8") - harness = _configured_harness(root) - harness.connect_tools(Tool(name="search", source=ofw.editable(Path("search.ts")))) - - revision = harness.process() - - tool_component = _required_component(revision, ComponentKind.TOOL) - assert tool_component.assets[0].name == "search" - assert tool_component.assets[0].source.relative_path == Path("search.ts") - - -def test_component_fingerprint_localizes_a_tool_change(tmp_path: Path) -> None: - root = _repository(tmp_path) - implementation = root / "search.ts" - implementation.write_text("export const search = () => 1;\n", encoding="utf-8") - first_harness = _configured_harness(root) - first_harness.connect_tools( - Tool(name="search", source=ofw.editable(Path("search.ts"))), - ) - first = first_harness.process() - - implementation.write_text("export const search = () => 2;\n", encoding="utf-8") - second_harness = _configured_harness(root) - second_harness.connect_tools( - Tool(name="search", source=ofw.editable(Path("search.ts"))), - ) - second = second_harness.process() - - assert ( - _required_component(first, ComponentKind.TOOL).digest - != _required_component(second, ComponentKind.TOOL).digest - ) - assert ( - _required_component(first, ComponentKind.PROMPT).digest - == _required_component(second, ComponentKind.PROMPT).digest - ) - - -def test_adding_middleware_does_not_change_prompt_component(tmp_path: Path) -> None: - root = _repository(tmp_path) - middleware = root / "middleware.ts" - middleware.write_text("export const beforeCall = () => {};\n", encoding="utf-8") - first = _configured_harness(root).process() - second_harness = _configured_harness(root) - second_harness.connect_middleware(ofw.editable(Path("middleware.ts"))) - second = second_harness.process() - - assert ( - _required_component(first, ComponentKind.PROMPT).digest - == _required_component(second, ComponentKind.PROMPT).digest - ) - assert second.component(ComponentKind.MIDDLEWARE) is not None - - -def test_file_cannot_be_owned_by_two_components(tmp_path: Path) -> None: - root = _repository(tmp_path) - harness = _configured_harness(root) - harness.connect_skills(ofw.editable(Path("prompt.md"))) - - with pytest.raises(HarnessValidationError) as raised: - harness.process() - - assert raised.value.code is HarnessErrorCode.COMPONENT_OVERLAP - - -@pytest.mark.parametrize("name", ("", "contains spaces", "UPPERCASE")) -def test_invalid_tool_name_fails(name: str) -> None: - with pytest.raises(HarnessValidationError) as raised: - Tool(name=name, source=Path("tool.py")) - assert raised.value.code is HarnessErrorCode.INVALID_TOOL_NAME - - -def test_duplicate_tool_name_fails(tmp_path: Path) -> None: - root = _repository(tmp_path) - first = root / "first.py" - second = root / "second.ts" - first.write_text("def run(): pass\n", encoding="utf-8") - second.write_text("export const run = () => {};\n", encoding="utf-8") - harness = _configured_harness(root) - - with pytest.raises(HarnessValidationError) as raised: - harness.connect_tools( - Tool(name="run", source=Path("first.py")), - Tool(name="run", source=Path("second.ts")), - ) - - assert raised.value.code is HarnessErrorCode.DUPLICATE_TOOL - - -def test_multiple_named_tools_may_share_one_source_file(tmp_path: Path) -> None: - root = _repository(tmp_path) - source = root / "tools.py" - source.write_text("def read(): pass\n\ndef write(): pass\n", encoding="utf-8") - harness = _configured_harness(root) - harness.connect_tools( - Tool(name="read", source=ofw.editable(Path("tools.py"))), - Tool(name="write", source=ofw.editable(Path("tools.py"))), - ) - - revision = harness.process() - - tool_component = _required_component(revision, ComponentKind.TOOL) - assert tuple(asset.name for asset in tool_component.assets) == ("read", "write") - - -@pytest.mark.parametrize("name", ("", "contains spaces", "UPPERCASE")) -def test_invalid_subagent_name_fails(name: str) -> None: - with pytest.raises(HarnessValidationError) as raised: - Subagent(name=name, source=Path("subagent.py")) - assert raised.value.code is HarnessErrorCode.INVALID_SUBAGENT_NAME - - -def test_duplicate_subagent_name_fails(tmp_path: Path) -> None: - root = _repository(tmp_path) - source = root / "subagents.py" - source.write_text("reviewer = 1\n", encoding="utf-8") - harness = _configured_harness(root) - - with pytest.raises(HarnessValidationError) as raised: - harness.connect_subagents( - Subagent(name="reviewer", source=Path("subagents.py")), - Subagent(name="reviewer", source=Path("subagents.py")), - ) - - assert raised.value.code is HarnessErrorCode.DUPLICATE_SUBAGENT - - -def test_multiple_named_subagents_may_share_one_source_file(tmp_path: Path) -> None: - root = _repository(tmp_path) - source = root / "subagents.py" - source.write_text("reviewer = 1\nresearcher = 2\n", encoding="utf-8") - harness = _configured_harness(root) - harness.connect_subagents( - Subagent(name="reviewer", source=ofw.editable(Path("subagents.py"))), - Subagent(name="researcher", source=ofw.editable(Path("subagents.py"))), - ) - - revision = harness.process() - - component = _required_component(revision, ComponentKind.SUBAGENT) - assert tuple(asset.name for asset in component.assets) == ("researcher", "reviewer") - - -def test_assets_are_frozen_unless_explicitly_editable(tmp_path: Path) -> None: - root = _repository(tmp_path) - skill = root / "SKILL.md" - skill.write_text("# Skill\n", encoding="utf-8") - harness = Harness("fixtureco-agent", root=root) - harness.connect_prompt(Path("prompt.md")) - harness.connect_skills(ofw.editable(Path("SKILL.md"))) - - revision = harness.process() - - assert ( - _required_component(revision, ComponentKind.PROMPT).assets[0].access is AssetAccess.FROZEN - ) - assert ( - _required_component(revision, ComponentKind.SKILL).assets[0].access - is AssetAccess.FIT_EDITABLE - ) - - -def test_environment_secret_file_is_never_fingerprinted(tmp_path: Path) -> None: - root = _repository(tmp_path) - (root / ".env").write_text("SECRET=do-not-read\n", encoding="utf-8") - harness = _configured_harness(root) - harness.connect_prompt(Path(".env")) - - with pytest.raises(HarnessValidationError) as raised: - harness.process() - - assert raised.value.code is HarnessErrorCode.SENSITIVE_ASSET - - -def test_same_inputs_produce_same_revision(tmp_path: Path) -> None: - root = _repository(tmp_path) - assert _configured_harness(root).process() == _configured_harness(root).process() - - -def test_file_change_produces_new_revision(tmp_path: Path) -> None: - root = _repository(tmp_path) - first = _configured_harness(root).process() - (root / "prompt.md").write_text("Be accurate and concise.\n", encoding="utf-8") - second = _configured_harness(root).process() - - assert second.id != first.id - assert second.repository.is_dirty - assert second.repository.dirty_digest is not None - - -def test_new_git_commit_produces_new_revision(tmp_path: Path) -> None: - root = _repository(tmp_path) - first = _configured_harness(root).process() - (root / "README.md").write_text("Fixture repository.\n", encoding="utf-8") - _run_git(root, "add", "README.md") - _run_git(root, "commit", "-qm", "document fixture") - second = _configured_harness(root).process() - - assert second.id != first.id - assert second.repository.commit != first.repository.commit - assert not second.repository.is_dirty - - -@pytest.mark.parametrize("name", ("", "contains spaces", "UPPERCASE")) -def test_invalid_harness_name_fails(name: str, tmp_path: Path) -> None: - with pytest.raises(HarnessValidationError) as raised: - Harness(name, root=tmp_path) - assert raised.value.code is HarnessErrorCode.INVALID_NAME - - -@pytest.mark.parametrize( - ("source", "code"), - ( - (Path("missing.md"), HarnessErrorCode.MISSING_ASSET), - (Path("folder"), HarnessErrorCode.NOT_A_FILE), - ), -) -def test_invalid_workspace_file_fails( - source: Path, - code: HarnessErrorCode, - tmp_path: Path, -) -> None: - root = _repository(tmp_path) - (root / "folder").mkdir() - harness = _configured_harness(root) - harness.connect_skills(source) - - with pytest.raises(HarnessValidationError) as raised: - harness.process() - assert raised.value.code is code - - -def test_path_and_symlink_escape_fail(tmp_path: Path) -> None: - root = _repository(tmp_path) - outside = tmp_path / "outside.md" - outside.write_text("outside", encoding="utf-8") - (root / "linked.md").symlink_to(outside) - - for source in (outside, Path("linked.md")): - harness = _configured_harness(root) - harness.connect_skills(source) - with pytest.raises(HarnessValidationError) as raised: - harness.process() - assert raised.value.code is HarnessErrorCode.PATH_OUTSIDE_ROOT - - -@pytest.mark.parametrize( - ("sources", "code"), - ( - ((Path("prompt.md"), Path("prompt.md")), HarnessErrorCode.DUPLICATE_ASSET), - ( - (Path("prompt.md"), ofw.editable(Path("prompt.md"))), - HarnessErrorCode.CONFLICTING_ACCESS, - ), - ), -) -def test_duplicate_component_asset_fails( - sources: tuple[Path | EditableFile, ...], - code: HarnessErrorCode, - tmp_path: Path, -) -> None: - root = _repository(tmp_path) - harness = Harness("fixtureco-agent", root=root) - first, second = sources - harness.connect_prompt(first, second) - - with pytest.raises(HarnessValidationError) as raised: - harness.process() - assert raised.value.code is code - - -def test_process_requires_prompt(tmp_path: Path) -> None: - root = _repository(tmp_path) - skill = root / "SKILL.md" - skill.write_text("# Skill\n", encoding="utf-8") - harness = Harness("fixtureco-agent", root=root) - harness.connect_skills(Path("SKILL.md")) - - with pytest.raises(HarnessValidationError) as raised: - harness.process() - assert raised.value.code is HarnessErrorCode.PROMPT_REQUIRED - - -def test_root_must_be_git_repository(tmp_path: Path) -> None: - root = tmp_path / "not-git" - root.mkdir() - (root / "prompt.md").write_text("hello", encoding="utf-8") - harness = Harness("fixtureco-agent", root=root) - harness.connect_prompt(Path("prompt.md")) - - with pytest.raises(HarnessValidationError) as raised: - harness.process() - assert raised.value.code is HarnessErrorCode.GIT_REPOSITORY_REQUIRED diff --git a/tests/test_langfuse_collection.py b/tests/test_langfuse_collection.py index f33a111..9aee130 100644 --- a/tests/test_langfuse_collection.py +++ b/tests/test_langfuse_collection.py @@ -16,15 +16,14 @@ from ofw import ( CollectionError, CollectionErrorCode, - Harness, HarnessRevision, LangfuseProject, ObservationContentField, ObservationContentMatch, ObservationContentQuery, - Tool, TraceWindow, ofw, + process_repository, ) from ofw.observability.langfuse.domain import ( AttributionLevel, @@ -258,11 +257,7 @@ def _revision( base_url=server.base_url, allow_private_network=True, ) - harness = Harness("fixture-agent", root=root) - harness.connect_prompt(ofw.editable(Path("prompt.md"))) - harness.connect_tools(Tool(name="run", source=ofw.editable(Path("tool.py")))) - harness.connect_observability(project) - return harness.process() + return process_repository("fixture-agent", root, traces=project) def _window() -> TraceWindow: diff --git a/tests/test_langfuse_contracts.py b/tests/test_langfuse_contracts.py index b4d8379..a95be2d 100644 --- a/tests/test_langfuse_contracts.py +++ b/tests/test_langfuse_contracts.py @@ -12,10 +12,10 @@ from ofw import ( CollectionError, CollectionErrorCode, - Harness, LangfuseProject, TraceWindow, ofw, + process_repository, ) @@ -105,25 +105,19 @@ def test_trace_window_requires_aware_utc_ordering() -> None: assert reversed_window.value.code is CollectionErrorCode.INVALID_WINDOW -def test_observability_connection_changes_harness_revision_without_persisting_secrets( +def test_observability_connection_changes_repository_revision_without_persisting_secrets( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: root = _harness_root(tmp_path) - baseline_harness = Harness("fixture-agent", root=root) - baseline_harness.connect_prompt(ofw.editable(Path("prompt.md"))) - baseline = baseline_harness.process() + baseline = process_repository("fixture-agent", root) monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-sensitive") monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-sensitive") project = LangfuseProject.from_env( environment="production", base_url="https://us.cloud.langfuse.com", ) - connected_harness = Harness("fixture-agent", root=root) - connected_harness.connect_prompt(ofw.editable(Path("prompt.md"))) - connected_harness.connect_observability(project) - - connected = connected_harness.process() + connected = process_repository("fixture-agent", root, traces=project) assert connected.id != baseline.id assert connected.observability is not None @@ -134,9 +128,7 @@ def test_observability_connection_changes_harness_revision_without_persisting_se def test_collect_requires_connected_observability(tmp_path: Path) -> None: root = _harness_root(tmp_path) - harness = Harness("fixture-agent", root=root) - harness.connect_prompt(ofw.editable(Path("prompt.md"))) - revision = harness.process() + revision = process_repository("fixture-agent", root) start = datetime(2026, 8, 22, tzinfo=UTC) with pytest.raises(CollectionError) as raised: diff --git a/tests/test_mine.py b/tests/test_mine.py index a87d4c0..aa40774 100644 --- a/tests/test_mine.py +++ b/tests/test_mine.py @@ -97,10 +97,7 @@ def _revision(tmp_path: Path, value: str = "revision-1") -> HarnessRevision: harness_name="support-agent", root=tmp_path, repository=RepositorySnapshot(GitCommit("abc123"), False, None), - components=(), observability=None, - runtime=None, - canary_digest=None, ) diff --git a/tests/test_repository.py b/tests/test_repository.py new file mode 100644 index 0000000..949afe2 --- /dev/null +++ b/tests/test_repository.py @@ -0,0 +1,102 @@ +"""Whole-repository harness revision behavior.""" + +from __future__ import annotations + +import subprocess +from dataclasses import FrozenInstanceError +from pathlib import Path + +import pytest + +from ofw import ( + HarnessErrorCode, + HarnessRevision, + HarnessValidationError, + LangfuseProject, + process_repository, +) + + +def _run_git(root: Path, *arguments: str) -> None: + subprocess.run( + ("git", "-C", str(root), *arguments), + check=True, + capture_output=True, + text=True, + ) + + +def _repository(tmp_path: Path) -> Path: + root = tmp_path / "fixture-agent" + root.mkdir() + (root / "agent.py").write_text("PROMPT = 'be accurate'\n", encoding="utf-8") + _run_git(root, "init", "-q") + _run_git(root, "config", "user.email", "fixture@example.test") + _run_git(root, "config", "user.name", "FixtureCo") + _run_git(root, "add", ".") + _run_git(root, "commit", "-qm", "fixture baseline") + return root + + +def test_process_repository_creates_an_immutable_revision(tmp_path: Path) -> None: + root = _repository(tmp_path) + + revision = process_repository("fixture-agent", root) + + assert isinstance(revision, HarnessRevision) + assert revision.root == root + assert revision.manifest_path.is_file() + assert revision.manifest_path.read_text(encoding="utf-8") == f"{revision.to_json()}\n" + with pytest.raises(FrozenInstanceError): + revision.harness_name = "changed" # type: ignore[misc] + + +def test_repository_commit_dirty_diff_and_untracked_files_change_revision( + tmp_path: Path, +) -> None: + root = _repository(tmp_path) + clean = process_repository("fixture-agent", root) + (root / "agent.py").write_text("PROMPT = 'be concise'\n", encoding="utf-8") + dirty = process_repository("fixture-agent", root) + (root / "new_skill.md").write_text("# New skill\n", encoding="utf-8") + untracked = process_repository("fixture-agent", root) + + assert clean.id != dirty.id != untracked.id + assert not clean.repository.is_dirty + assert dirty.repository.is_dirty + assert untracked.repository.is_dirty + + +def test_observability_connection_changes_revision_without_storing_secrets( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = _repository(tmp_path) + baseline = process_repository("fixture-agent", root) + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-sensitive") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-sensitive") + project = LangfuseProject.from_env(environment="production") + + connected = process_repository("fixture-agent", root, traces=project) + + assert connected.id != baseline.id + assert connected.observability == project.manifest() + assert "pk-sensitive" not in connected.to_json() + assert "sk-sensitive" not in connected.to_json() + + +@pytest.mark.parametrize("name", ("", "contains spaces", "UPPERCASE")) +def test_invalid_repository_name_fails(name: str, tmp_path: Path) -> None: + with pytest.raises(HarnessValidationError) as raised: + process_repository(name, tmp_path) + assert raised.value.code is HarnessErrorCode.INVALID_NAME + + +def test_root_must_be_a_git_repository(tmp_path: Path) -> None: + root = tmp_path / "not-git" + root.mkdir() + + with pytest.raises(HarnessValidationError) as raised: + process_repository("fixture-agent", root) + + assert raised.value.code is HarnessErrorCode.GIT_REPOSITORY_REQUIRED diff --git a/tests/test_runtime.py b/tests/test_runtime.py index ac4e6e6..5146f5b 100644 --- a/tests/test_runtime.py +++ b/tests/test_runtime.py @@ -20,10 +20,7 @@ CommandLoop, CommandVerifier, E2BSandbox, - Harness, - HarnessErrorCode, HarnessRevision, - HarnessValidationError, ModelFingerprint, ProcessCommand, ProcessLimits, @@ -31,6 +28,7 @@ RunResult, RunStatus, VerifierVerdict, + process_repository, ) @@ -192,9 +190,7 @@ def _repository(tmp_path: Path) -> Path: def _revision(root: Path) -> HarnessRevision: - harness = Harness("runtime-agent", root=root) - harness.connect_prompt(Path("prompt.md")) - return harness.process() + return process_repository("runtime-agent", root) def _verifier(mode: str, name: str = "verifier") -> CommandVerifier: @@ -208,77 +204,36 @@ def _command_loop() -> CommandLoop: ) -def _runtime_harness(root: Path) -> Harness: - harness = Harness("runtime-agent", root=root) - harness.connect_prompt(Path("prompt.md")) - harness.connect_execute(E2BSandbox(ProcessLimits(timedelta(seconds=2)))) - harness.connect_lifecycle(_command_loop()) - harness.connect_verifiers(_verifier("uppercase", "uppercase")) - return harness - - -def test_process_runs_e2b_command_canary_and_records_frozen_evidence(tmp_path: Path) -> None: - root = _repository(tmp_path) - harness = Harness("runtime-agent", root=root) - harness.connect_prompt(Path("prompt.md")) - harness.connect_execute(E2BSandbox(ProcessLimits(timedelta(seconds=2)))) - harness.connect_lifecycle(_command_loop()) - harness.connect_verifiers(_verifier("uppercase", "uppercase")) - - revision = harness.process(canary=CanaryCase(CaseId("smoke"), "ship")) - - assert revision.runtime is not None - assert revision.canary_digest is not None - assert revision.canary_path.is_file() - assert "pass" in revision.canary_path.read_text(encoding="utf-8") - - -def test_canary_evidence_does_not_change_runtime_revision_identity(tmp_path: Path) -> None: - root = _repository(tmp_path) - - without_canary = _runtime_harness(root).process() - with_canary = _runtime_harness(root).process(canary=CanaryCase(CaseId("identity"), "ship")) - - assert with_canary.id == without_canary.id - assert with_canary.canary_digest is not None - - -def test_partial_runtime_configuration_is_rejected(tmp_path: Path) -> None: +def test_run_canary_returns_frozen_evidence(tmp_path: Path) -> None: root = _repository(tmp_path) - harness = Harness("runtime-agent", root=root) - harness.connect_prompt(Path("prompt.md")) - harness.connect_execute(E2BSandbox(ProcessLimits(timedelta(seconds=1)))) - - with pytest.raises(HarnessValidationError) as raised: - harness.process() - - assert raised.value.code is HarnessErrorCode.RUNTIME_INCOMPLETE - - -def test_duplicate_verifier_name_is_rejected(tmp_path: Path) -> None: - harness = Harness("runtime-agent", root=_repository(tmp_path)) + revision = _revision(root) - with pytest.raises(HarnessValidationError) as raised: - harness.connect_verifiers( - _verifier("uppercase", "duplicate"), - _verifier("reject", "duplicate"), - ) + report = runtime_module.run_canary( + revision, + CanaryCase(CaseId("smoke"), "ship"), + E2BSandbox(ProcessLimits(timedelta(seconds=2))), + _command_loop(), + (_verifier("uppercase", "uppercase"),), + ) - assert raised.value.code is HarnessErrorCode.DUPLICATE_VERIFIER + assert report.passed + assert str(report.digest).startswith("sha256:") -def test_failed_canary_blocks_revision_creation(tmp_path: Path) -> None: +def test_failed_canary_is_reported_without_mutating_revision(tmp_path: Path) -> None: root = _repository(tmp_path) - harness = Harness("runtime-agent", root=root) - harness.connect_prompt(Path("prompt.md")) - harness.connect_execute(E2BSandbox(ProcessLimits(timedelta(seconds=2)))) - harness.connect_lifecycle(_command_loop()) - harness.connect_verifiers(_verifier("reject", "reject")) + revision = _revision(root) - with pytest.raises(HarnessValidationError) as raised: - harness.process(canary=CanaryCase(CaseId("rejected"), "ship")) + report = runtime_module.run_canary( + revision, + CanaryCase(CaseId("rejected"), "ship"), + E2BSandbox(ProcessLimits(timedelta(seconds=2))), + _command_loop(), + (_verifier("reject", "reject"),), + ) - assert raised.value.code is HarnessErrorCode.CANARY_FAILED + assert not report.passed + assert _revision(root).id == revision.id def test_e2b_reports_timeout_and_nonzero_exit(tmp_path: Path) -> None: @@ -487,26 +442,6 @@ def test_parallel_e2b_environments_are_distinct(tmp_path: Path) -> None: environment.destroy(second) -def test_runtime_fingerprint_changes_without_changing_assets(tmp_path: Path) -> None: - root = _repository(tmp_path) - first = Harness("runtime-agent", root=root) - first.connect_prompt(Path("prompt.md")) - first.connect_execute(E2BSandbox(ProcessLimits(timedelta(seconds=1)))) - first.connect_lifecycle(_command_loop()) - first.connect_verifiers(_verifier("uppercase", "uppercase")) - second = Harness("runtime-agent", root=root) - second.connect_prompt(Path("prompt.md")) - second.connect_execute(E2BSandbox(ProcessLimits(timedelta(seconds=2)))) - second.connect_lifecycle(_command_loop()) - second.connect_verifiers(_verifier("uppercase", "uppercase")) - - first_revision = first.process() - second_revision = second.process() - - assert first_revision.id != second_revision.id - assert first_revision.components == second_revision.components - - def test_command_verifier_is_terminated_at_environment_timeout(tmp_path: Path) -> None: root = _repository(tmp_path) revision = _revision(root) diff --git a/tests/test_typing.py b/tests/test_typing.py index 45c54da..015c964 100644 --- a/tests/test_typing.py +++ b/tests/test_typing.py @@ -11,6 +11,6 @@ def test_package_declares_inline_types_and_namespace_methods() -> None: assert package_file is not None assert Path(package_file).with_name("py.typed").is_file() assert callable(ofw.collect) - assert callable(ofw.editable) + assert callable(package.process_repository) assert callable(ofw.E2BSandbox) assert callable(ofw.ProcessLimits) From e09bfbd32d2a9855aa524ec8d2ee09406f2d321b Mon Sep 17 00:00:00 2001 From: divo12 Date: Wed, 26 Aug 2026 08:54:46 +0530 Subject: [PATCH 17/18] use git revisions for trace attribution --- src/ofw/repository.py | 12 ++++++++++-- tests/test_langfuse_contracts.py | 4 ++-- tests/test_repository.py | 3 ++- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/ofw/repository.py b/src/ofw/repository.py index de1cfc5..362da10 100644 --- a/src/ofw/repository.py +++ b/src/ofw/repository.py @@ -41,10 +41,10 @@ def process_repository( repository=_snapshot_repository(selected_root), observability=None if traces is None else traces.manifest(), ) - digest = _digest(content.canonical_json().encode()) + revision_id = _revision_id(content.repository) revision = HarnessRevision( schema_version=content.schema_version, - id=HarnessRevisionId(f"ofw_{digest.value[7:]}"), + id=revision_id, harness_name=content.harness_name, root=selected_root, repository=content.repository, @@ -54,6 +54,14 @@ def process_repository( return revision +def _revision_id(repository: RepositorySnapshot) -> HarnessRevisionId: + if repository.dirty_digest is None: + return HarnessRevisionId(str(repository.commit)) + return HarnessRevisionId( + f"{repository.commit.value}-dirty-{repository.dirty_digest.value[7:23]}" + ) + + def _resolve_root(root: Path) -> Path: if not isinstance(root, Path): raise HarnessValidationError(HarnessErrorCode.INVALID_SOURCE, repr(root)) diff --git a/tests/test_langfuse_contracts.py b/tests/test_langfuse_contracts.py index a95be2d..f1ec1a4 100644 --- a/tests/test_langfuse_contracts.py +++ b/tests/test_langfuse_contracts.py @@ -105,7 +105,7 @@ def test_trace_window_requires_aware_utc_ordering() -> None: assert reversed_window.value.code is CollectionErrorCode.INVALID_WINDOW -def test_observability_connection_changes_repository_revision_without_persisting_secrets( +def test_observability_connection_does_not_change_code_revision_or_persist_secrets( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -119,7 +119,7 @@ def test_observability_connection_changes_repository_revision_without_persisting ) connected = process_repository("fixture-agent", root, traces=project) - assert connected.id != baseline.id + assert connected.id == baseline.id assert connected.observability is not None assert connected.observability == project.manifest() assert "pk-sensitive" not in connected.to_json() diff --git a/tests/test_repository.py b/tests/test_repository.py index 949afe2..e97bff2 100644 --- a/tests/test_repository.py +++ b/tests/test_repository.py @@ -44,6 +44,7 @@ def test_process_repository_creates_an_immutable_revision(tmp_path: Path) -> Non revision = process_repository("fixture-agent", root) assert isinstance(revision, HarnessRevision) + assert str(revision.id) == str(revision.repository.commit) assert revision.root == root assert revision.manifest_path.is_file() assert revision.manifest_path.read_text(encoding="utf-8") == f"{revision.to_json()}\n" @@ -79,7 +80,7 @@ def test_observability_connection_changes_revision_without_storing_secrets( connected = process_repository("fixture-agent", root, traces=project) - assert connected.id != baseline.id + assert connected.id == baseline.id assert connected.observability == project.manifest() assert "pk-sensitive" not in connected.to_json() assert "sk-sensitive" not in connected.to_json() From 13a30fdbeeba508069a62453e360bc854f8f530b Mon Sep 17 00:00:00 2001 From: divo12 Date: Wed, 26 Aug 2026 09:04:15 +0530 Subject: [PATCH 18/18] remove premature MCP and plugin surfaces --- .../openflywheel/.codex-plugin/plugin.json | 21 -- .../skills/ofw-mine-failures/SKILL.md | 108 ------ pyproject.toml | 1 - src/ofw/__init__.py | 2 - src/ofw/mcp.py | 351 ------------------ tests/test_mcp.py | 72 ---- 6 files changed, 555 deletions(-) delete mode 100644 plugins/openflywheel/.codex-plugin/plugin.json delete mode 100644 plugins/openflywheel/skills/ofw-mine-failures/SKILL.md delete mode 100644 src/ofw/mcp.py delete mode 100644 tests/test_mcp.py diff --git a/plugins/openflywheel/.codex-plugin/plugin.json b/plugins/openflywheel/.codex-plugin/plugin.json deleted file mode 100644 index 8c56f21..0000000 --- a/plugins/openflywheel/.codex-plugin/plugin.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "name": "openflywheel", - "version": "0.1.0", - "description": "Mine evidence-backed failures from Langfuse-connected agent harnesses with Codex.", - "author": { - "name": "OpenFlyWheel" - }, - "repository": "https://github.com/divo12/OpenFlyWheel", - "skills": "./skills/", - "interface": { - "displayName": "OpenFlyWheel", - "shortDescription": "Mine failures from production traces", - "longDescription": "Codex workflows for mining observable failures from full Langfuse trajectories without explicit harness component mapping.", - "developerName": "OpenFlyWheel", - "category": "Developer Tools", - "capabilities": ["Read", "Write"], - "defaultPrompt": [ - "Mine failures from the connected OpenFlyWheel trajectory." - ] - } -} diff --git a/plugins/openflywheel/skills/ofw-mine-failures/SKILL.md b/plugins/openflywheel/skills/ofw-mine-failures/SKILL.md deleted file mode 100644 index 315cde4..0000000 --- a/plugins/openflywheel/skills/ofw-mine-failures/SKILL.md +++ /dev/null @@ -1,108 +0,0 @@ ---- -name: ofw-mine-failures -description: Mines observable failures from any executed agent harness through a connected OpenFlyWheel failure-mining MCP server backed by full Langfuse trajectories. Use when asked to judge, triage, or mine failures from OFW traces. Do not use for root-cause diagnosis, clustering, eval generation, rubric refinement, or modifying the evaluated harness. ---- - -# Mine agent-harness failures with OpenFlyWheel - -Investigate one OFW mining case and return an evidence-grounded failure-mining -result. The connected harness is the executed system. You are the Codex operator using OFW. -Stay at observable behavior; diagnosis and improvement are separate work. - -## Principles - -1. **The oracle decides completion.** An agent claim is evidence of what it - claimed, not proof that the task succeeded. Verify every required outcome - against its declared environment source. -2. **Recovery matters.** A failed action is not a task failure when the agent later - recovers and every required outcome is completed. -3. **Use only issued evidence.** Cite observation and environment evidence - returned by OFW tools. Never invent identifiers, digests, state, or tool - results. -4. **Read the complete trajectory.** Search to find relevant regions, then page - through `read_trajectory` until `next_cursor` is null. Keep only relevant - evidence in the result. -5. **Calibration cannot override state.** `adapt` returns human and production - signals from nominated trajectories. Use them to challenge an interpretation, - never to replace source-of-truth verification. -6. **No diagnosis.** Do not identify a root cause, responsible component, bad - prompt, broken tool, or proposed fix. Do not cluster failures, create evals, - or change a rubric. - -## Workflow - -1. Call `get_mining_case`. Read the task intent, constraints, required outcomes, - available agent tools, environment sources, observation IDs, and nominated - signals. -2. Search the current trajectory with `search_trajectory`: - - search task-specific entities and required outcomes; - - search completion claims such as `done`, `success`, or `completed`; - - search errors, failed actions, retries, cancellations, and verification; - - follow new questions raised by each useful hit with another focused search. -3. Page through the ordered trajectory with `read_trajectory`, beginning with a - null cursor and continuing with each returned `next_cursor` until it is null. - Track whether errors were retried, recovered, abandoned, or contradicted by a - later action. -4. Use `search_prior_trajectories` only when a similar run, prior disagreement, - or repeated signal would clarify the current observable behavior. Prior runs - do not prove the current outcome. -5. For every required outcome, call `verify_environment` with the exact - `source_id` and `check_id` returned by `get_mining_case`. -6. Call `adapt` for the relevant nominated signal kinds. Compare those signals - with the trajectory and verification result. Record disagreement as an - unresolved question; do not silently choose the signal you prefer. -7. Apply the verdict rules and return the result in the output shape below. - -## Verdict rules - -- `confirmed_failure`: at least one required outcome is `not_completed`, and a - concrete `FailureBehavior` is grounded in trajectory plus environment - evidence. -- `no_failure`: every required outcome is `completed`; `failure_behavior` must - be null, including when an intermediate action failed but recovery succeeded. -- `ambiguous`: completion cannot be established because required environment - state is unavailable or evidence materially conflicts; `failure_behavior` - must be null and `unresolved_questions` must explain the uncertainty. -- Never return `confirmed_failure` from a tool error alone. - -Use only these observable behavior categories: - -- `outcome_mismatch` -- `false_completion` -- `required_action_omitted` -- `forbidden_state_change` -- `unrecovered_action_failure` -- `no_progress_loop` -- `abandoned_before_completion` - -## Output - -Return one `FailureMiningResult`-shaped object containing: - -```text -task -context -verdict -source_ids -completion_checks -failure_behavior | null -trajectory_evidence -environment_evidence -confidence -unresolved_questions -invalid_reason | null -``` - -For a confirmed failure, each behavior observation must identify its behavior -kind, phase, first and optional last observation IDs, recovery status, and -trajectory evidence. Keep the summary factual and free of causal claims. - -## Decision examples - -- A command fails, the agent retries successfully, and the oracle confirms the - required state: `no_failure`. -- The agent says the work is complete, but the oracle shows the required state was - not reached: `confirmed_failure` with `false_completion` or - `outcome_mismatch`, depending on the observed behavior. -- The agent appears to stop early, but the environment cannot be queried: - `ambiguous`, not an inferred failure. diff --git a/pyproject.toml b/pyproject.toml index c849d1b..9d69239 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,6 @@ dependencies = [ "e2b>=2.38,<3", "httpx>=0.27,<1", "langfuse>=4.7,<5", - "mcp>=2,<3", "pydantic>=2.10,<3", ] diff --git a/src/ofw/__init__.py b/src/ofw/__init__.py index 060a049..fa338c7 100644 --- a/src/ofw/__init__.py +++ b/src/ofw/__init__.py @@ -21,7 +21,6 @@ RepositorySnapshot, Sha256Digest, ) -from ofw.mcp import FailureMiningMcpServer from ofw.mine import ( AdaptationRequest, AdaptationResult, @@ -174,7 +173,6 @@ def read_observation_content( "EvidenceRecordId", "EvidenceReference", "FailureMiningResult", - "FailureMiningMcpServer", "FailureMiningRun", "FailureBehavior", "FailureBehaviorKind", diff --git a/src/ofw/mcp.py b/src/ofw/mcp.py deleted file mode 100644 index 5b6e438..0000000 --- a/src/ofw/mcp.py +++ /dev/null @@ -1,351 +0,0 @@ -"""MCP transport exposing a live OFW failure-mining case to Codex.""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from datetime import datetime - -from mcp.server import MCPServer -from pydantic import BaseModel, ConfigDict - -from ofw.mine import ( - AdaptationRequest, - CompletionStatus, - ConstraintKind, - EnvironmentCheckId, - EnvironmentCheckRequest, - EnvironmentSourceId, - EnvironmentSourceKind, - EvidenceKind, - EvidenceReference, - FailureSource, - FailureSourceKind, - MiningTools, - ToolAccess, - ToolAction, - ToolStatus, - TrajectoryPageRequest, - TrajectorySearchRequest, - TrajectorySearchResult, -) -from ofw.observability.langfuse.domain import ObservationContentField, ObservationId - - -class McpModel(BaseModel): - model_config = ConfigDict(frozen=True) - - -class McpEvidence(McpModel): - kind: EvidenceKind - record_id: str - digest: str - - -class McpRequiredOutcome(McpModel): - check_id: str - source_id: str - description: str - - -class McpConstraint(McpModel): - kind: ConstraintKind - description: str - - -class McpToolCapability(McpModel): - name: str - access: ToolAccess - - -class McpEnvironmentSource(McpModel): - id: str - kind: EnvironmentSourceKind - summary: str - - -class McpFailureSignal(McpModel): - id: str - kind: FailureSourceKind - trace_id: str - observed_at: datetime - summary: str - evidence: tuple[McpEvidence, ...] - - -class McpMiningCase(McpModel): - task_id: str - intent: str - required_outcomes: tuple[McpRequiredOutcome, ...] - constraints: tuple[McpConstraint, ...] - revision_id: str - trace_id: str - trace_digest: str - observation_ids: tuple[str, ...] - session_id: str | None - environment_name: str | None - release: str | None - available_tools: tuple[McpToolCapability, ...] - environment_sources: tuple[McpEnvironmentSource, ...] - initial_state_evidence: tuple[McpEvidence, ...] - failure_signals: tuple[McpFailureSignal, ...] - - -class McpSearchHit(McpModel): - observation_id: str - trace_id: str | None - field: ObservationContentField - excerpt: str - evidence: McpEvidence - - -class McpSearchResult(McpModel): - status: ToolStatus - summary: str - hits: tuple[McpSearchHit, ...] - next_actions: tuple[ToolAction, ...] - - -class McpTrajectoryObservation(McpModel): - id: str - parent_id: str | None - name: str | None - type: str - start_time: datetime - status_message: str | None - input: str | None - output: str | None - evidence: McpEvidence - - -class McpTrajectoryPage(McpModel): - status: ToolStatus - summary: str - observations: tuple[McpTrajectoryObservation, ...] - next_cursor: str | None - next_actions: tuple[ToolAction, ...] - - -class McpEnvironmentVerification(McpModel): - tool_status: ToolStatus - summary: str - completion_status: CompletionStatus | None - observed_state: str | None - evidence: tuple[McpEvidence, ...] - next_actions: tuple[ToolAction, ...] - - -class McpAdaptationResult(McpModel): - status: ToolStatus - summary: str - signals: tuple[McpFailureSignal, ...] - next_actions: tuple[ToolAction, ...] - - -@dataclass(slots=True) -class FailureMiningMcpServer: - """A local, read-only MCP server backed by one live mining case.""" - - tools: MiningTools - server: MCPServer[None] = field(init=False) - - def __post_init__(self) -> None: - server: MCPServer[None] = MCPServer( - name="openflywheel-failure-mining", - instructions=( - "Inspect and verify one executed agent-harness trajectory. Do not diagnose causes, " - "propose fixes, cluster failures, generate evals, or mutate rubrics." - ), - ) - server.tool(structured_output=True)(self.get_mining_case) - server.tool(structured_output=True)(self.search_trajectory) - server.tool(structured_output=True)(self.search_prior_trajectories) - server.tool(structured_output=True)(self.read_trajectory) - server.tool(structured_output=True)(self.verify_environment) - server.tool(structured_output=True)(self.adapt) - self.server = server - - def get_mining_case(self) -> McpMiningCase: - """Return the task, context, signals, environment sources, and required outcomes.""" - case = self.tools.case - return McpMiningCase( - task_id=case.task.id.value, - intent=case.task.intent, - required_outcomes=tuple( - McpRequiredOutcome( - check_id=item.check_id.value, - source_id=item.source_id.value, - description=item.description, - ) - for item in case.task.required_outcomes - ), - constraints=tuple( - McpConstraint(kind=item.kind, description=item.description) - for item in case.task.constraints - ), - revision_id=case.context.revision_id.value, - trace_id=case.context.trace_id.value, - trace_digest=case.context.trace_digest.value, - observation_ids=tuple(item.value for item in case.context.observation_ids), - session_id=case.context.session_id, - environment_name=case.context.environment_name, - release=case.context.release, - available_tools=tuple( - McpToolCapability(name=item.name.value, access=item.access) - for item in case.context.available_tools - ), - environment_sources=tuple( - McpEnvironmentSource(id=item.id.value, kind=item.kind, summary=item.summary) - for item in case.context.environment_sources - ), - initial_state_evidence=_evidence(case.context.initial_state_evidence), - failure_signals=tuple(_signal(item) for item in case.sources), - ) - - def search_trajectory( - self, - text: str, - field: ObservationContentField = ObservationContentField.ANY, - limit: int = 10, - ) -> McpSearchResult: - """Search the current full trajectory for focused evidence.""" - return _search_result( - self.tools.search_trajectory(TrajectorySearchRequest(text, field, limit)) - ) - - def search_prior_trajectories( - self, - text: str, - field: ObservationContentField = ObservationContentField.ANY, - limit: int = 10, - ) -> McpSearchResult: - """Search other trajectories in the collection for comparable evidence.""" - return _search_result( - self.tools.search_prior_trajectories( - TrajectorySearchRequest(text, field, limit) - ) - ) - - def read_trajectory( - self, - cursor: str | None = None, - limit: int = 50, - ) -> McpTrajectoryPage: - """Read an ordered page; continue until next_cursor is null.""" - result = self.tools.read_trajectory( - TrajectoryPageRequest( - None if cursor is None else ObservationId(cursor), - limit, - ) - ) - return McpTrajectoryPage( - status=result.status, - summary=result.summary, - observations=tuple( - McpTrajectoryObservation( - id=item.record.id.value, - parent_id=( - None - if item.record.parent_observation_id is None - else item.record.parent_observation_id.value - ), - name=item.record.name, - type=item.record.type.value, - start_time=item.record.start_time, - status_message=item.record.status_message, - input=None if item.input_content is None else item.input_content.text, - output=None if item.output_content is None else item.output_content.text, - evidence=McpEvidence( - kind=EvidenceKind.TRAJECTORY, - record_id=item.record.id.value, - digest=item.record.digest.value, - ), - ) - for item in result.observations - ), - next_cursor=None if result.next_cursor is None else result.next_cursor.value, - next_actions=result.next_actions, - ) - - def verify_environment( - self, - source_id: str, - check_id: str, - ) -> McpEnvironmentVerification: - """Verify a declared required outcome against its source-of-truth environment.""" - result = self.tools.verify_environment( - EnvironmentCheckRequest( - EnvironmentSourceId(source_id), - EnvironmentCheckId(check_id), - ) - ) - verification = result.verification - return McpEnvironmentVerification( - tool_status=result.status, - summary=result.summary, - completion_status=None if verification is None else verification.status, - observed_state=None if verification is None else verification.observed_state, - evidence=_evidence(result.artifacts), - next_actions=result.next_actions, - ) - - def adapt( - self, - kinds: tuple[FailureSourceKind, ...], - limit: int = 20, - ) -> McpAdaptationResult: - """Read human and production calibration signals without changing a rubric.""" - result = self.tools.adapt(AdaptationRequest(kinds, limit)) - return McpAdaptationResult( - status=result.status, - summary=result.summary, - signals=tuple(_signal(item) for item in result.signals), - next_actions=result.next_actions, - ) - - def run_stdio(self) -> None: - """Serve this live case to a local Codex process over standard I/O.""" - self.server.run() - - -def _evidence(items: tuple[EvidenceReference, ...]) -> tuple[McpEvidence, ...]: - return tuple( - McpEvidence( - kind=item.kind, - record_id=item.record_id.value, - digest=item.digest.value, - ) - for item in items - ) - - -def _signal(item: FailureSource) -> McpFailureSignal: - return McpFailureSignal( - id=item.id.value, - kind=item.kind, - trace_id=item.trace_id.value, - observed_at=item.observed_at, - summary=item.summary, - evidence=_evidence(item.evidence), - ) - - -def _search_result(result: TrajectorySearchResult) -> McpSearchResult: - return McpSearchResult( - status=result.status, - summary=result.summary, - hits=tuple( - McpSearchHit( - observation_id=hit.observation_id.value, - trace_id=None if hit.trace_id is None else hit.trace_id.value, - field=hit.field, - excerpt=hit.excerpt, - evidence=McpEvidence( - kind=EvidenceKind.TRAJECTORY, - record_id=hit.observation_id.value, - digest=hit.reference.digest.value, - ), - ) - for hit in result.hits - ), - next_actions=result.next_actions, - ) diff --git a/tests/test_mcp.py b/tests/test_mcp.py deleted file mode 100644 index 3a02b0b..0000000 --- a/tests/test_mcp.py +++ /dev/null @@ -1,72 +0,0 @@ -"""Codex-facing MCP transport for failure mining.""" - -from __future__ import annotations - -import asyncio -from pathlib import Path - -from mcp import Client - -from ofw.mcp import FailureMiningMcpServer -from ofw.mine import ( - CompletionStatus, - FailureSourceKind, - MiningTools, - TraceMiningCase, -) -from test_mine import ( - RecordedEnvironmentVerifier, - _collection, - _context, - _nomination, - _revision, -) - - -def test_codex_can_discover_and_call_failure_mining_tools(tmp_path: Path) -> None: - revision = _revision(tmp_path) - collection = _collection( - tmp_path, - revision, - ("Close ticket", "update failed", "Ticket remains open"), - ) - nomination = _nomination(FailureSourceKind.DOWNSTREAM_FAILURE) - tools = MiningTools( - TraceMiningCase( - nomination.task, - _context(collection.traces[0].observation_ids), - nomination.sources, - ), - collection, - RecordedEnvironmentVerifier( - CompletionStatus.NOT_COMPLETED, - "Ticket remains open", - ), - nomination.sources, - ) - server = FailureMiningMcpServer(tools) - - async def exercise() -> None: - async with Client(server.server) as client: - discovered = await client.list_tools() - assert {tool.name for tool in discovered.tools} == { - "adapt", - "get_mining_case", - "read_trajectory", - "search_prior_trajectories", - "search_trajectory", - "verify_environment", - } - result = await client.call_tool("get_mining_case") - assert result.is_error is False - - asyncio.run(exercise()) - assert server.get_mining_case().task_id == "close-ticket" - assert server.search_trajectory("failed", limit=5).status.value == "ok" - assert server.search_prior_trajectories("failed", limit=5).status.value == "not_found" - assert len(server.read_trajectory(limit=2).observations) == 2 - assert ( - server.verify_environment("itsm-production", "ticket-closed").completion_status - is CompletionStatus.NOT_COMPLETED - ) - assert server.adapt((FailureSourceKind.DOWNSTREAM_FAILURE,), 5).status.value == "ok"