diff --git a/AGENTS.md b/AGENTS.md index 8c17665..ebf4842 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -72,6 +72,7 @@ uvx --from radon radon cc -s -a src tests plugins/openflywheel/scripts/mcp_serve python3 ~/.codex/skills/.system/skill-creator/scripts/quick_validate.py plugins/openflywheel/skills/trace-query-planner python3 ~/.codex/skills/.system/skill-creator/scripts/quick_validate.py plugins/openflywheel/skills/outcome-recorder python3 ~/.codex/skills/.system/skill-creator/scripts/quick_validate.py plugins/openflywheel/skills/failure-miner +python3 ~/.codex/skills/.system/skill-creator/scripts/quick_validate.py plugins/openflywheel/skills/failure-pattern-miner python3 ~/.codex/skills/.system/plugin-creator/scripts/validate_plugin.py plugins/openflywheel ``` diff --git a/plugins/openflywheel/.codex-plugin/plugin.json b/plugins/openflywheel/.codex-plugin/plugin.json index 7e93ea8..3a64262 100644 --- a/plugins/openflywheel/.codex-plugin/plugin.json +++ b/plugins/openflywheel/.codex-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "openflywheel", - "version": "0.5.0", - "description": "Initialize ITSM-bench harness workspaces, query Langfuse trajectories, and record outcomes and compact failure diagnoses.", + "version": "0.6.0", + "description": "Initialize ITSM-bench harness workspaces, query Langfuse trajectories, and mine compact failure diagnoses and exact patterns.", "author": { "name": "OpenFlyWheel" }, @@ -10,8 +10,8 @@ "skills": "./skills/", "interface": { "displayName": "OpenFlyWheel", - "shortDescription": "Prepare ITSM workspaces and mine failures", - "longDescription": "Initialize an ITSM-bench agent-harness optimization workspace, inspect bounded Langfuse trajectory evidence, record authoritative outcomes, and persist compact failure diagnoses in the prepared local workspace.", + "shortDescription": "Diagnose and mine ITSM failure patterns", + "longDescription": "Initialize an ITSM-bench agent-harness optimization workspace, inspect bounded Langfuse evidence, record authoritative outcomes and compact diagnoses, and mine exact recurring patterns without copying trace payloads.", "developerName": "OpenFlyWheel", "category": "Productivity", "capabilities": ["Read", "Write"], diff --git a/plugins/openflywheel/.mcp.json b/plugins/openflywheel/.mcp.json index 3114afa..2280c7c 100644 --- a/plugins/openflywheel/.mcp.json +++ b/plugins/openflywheel/.mcp.json @@ -4,7 +4,7 @@ "command": "uvx", "args": [ "--from", - "git+https://github.com/divo12/OpenFlyWheel.git@0d4af6e0499c54765e2f7784b48b53baf96d1ad5", + "git+https://github.com/divo12/OpenFlyWheel.git@ab0ef62cbe1e6cddf0bfd8ec61374d10120c61aa", "--with", "mcp>=1.13,<2", "openflywheel-mcp" diff --git a/plugins/openflywheel/program_templates/itsm.md b/plugins/openflywheel/program_templates/itsm.md index a732bf3..9174db3 100644 --- a/plugins/openflywheel/program_templates/itsm.md +++ b/plugins/openflywheel/program_templates/itsm.md @@ -44,6 +44,11 @@ under `.workspace/failures/` before forming a harness hypothesis. Record an inco diagnosis when the evidence is insufficient. Do not copy Langfuse trace payloads into the workspace; Langfuse remains their source of truth. +After recording the diagnoses for one bounded run, use `$failure-pattern-miner` and call +`mine_failure_patterns` with only the returned artifact IDs from that run. It groups by failure +type and exact normalized root cause; results are not semantic clusters. Keep inconclusive +diagnoses separate, and reread each supporting diagnosis before forming a shared hypothesis. + An intermediate tool error is evidence, not an outcome failure, when the agent recovered and the verifier passed. A technically clean trajectory is still a failure when the ITSM verifier shows that the required environment state was not achieved. diff --git a/plugins/openflywheel/skills/failure-pattern-miner/SKILL.md b/plugins/openflywheel/skills/failure-pattern-miner/SKILL.md new file mode 100644 index 0000000..60cd928 --- /dev/null +++ b/plugins/openflywheel/skills/failure-pattern-miner/SKILL.md @@ -0,0 +1,28 @@ +--- +name: failure-pattern-miner +description: Group explicit compact OpenFlywheel failure diagnoses by issue type and exact normalized root cause. Use after failure-miner records diagnoses for a bounded comparison scope; do not use to diagnose traces, infer outcomes, create semantic clusters, or edit a harness. +--- + +# Failure Pattern Miner + +Mine repeated patterns from diagnoses already recorded by `$failure-miner`. Use only +artifact IDs returned by `record_failure` from the same prepared workspace and the same +comparison scope, such as one baseline or candidate run. Pass one to fifty unique artifact +IDs to `mine_failure_patterns`; never scan the workspace or substitute trace IDs and paths. + +The tool groups supported diagnoses by failure type plus exact normalized root cause. Its +normalizer masks volatile absolute paths, recognizable opaque identifiers, and numbers before +fingerprinting the complete normalized cause; the returned cause text is only a bounded display +excerpt. Results are deterministic exact matches, not semantic clusters: similar wording may +remain separate, and matching wording does not prove one repair will fix every occurrence. +Inconclusive diagnoses remain separate and must not be forced into a pattern. + +Read patterns in their declared order: occurrence count descending, distinct task count +descending, latest occurrence descending, then fingerprint ascending. Preserve the returned +fingerprints, normalized causes, task IDs, trace IDs, artifact IDs, and time bounds. A +repeated pattern may prioritize later hypothesis work, but reread its compact diagnosis +artifacts before proposing a harness change. + +Do not query Langfuse, call `record_outcome` or `record_failure`, modify diagnosis artifacts, +merge results from unrelated experiment scopes, generate embeddings, infer a broader cause, +or edit the harness while following this skill. diff --git a/pyproject.toml b/pyproject.toml index 7937c3e..dd61077 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "openflywheel" -version = "0.5.0" +version = "0.6.0" description = "A governed self-improving agent harness" requires-python = ">=3.11" dependencies = [ diff --git a/src/ofw/__init__.py b/src/ofw/__init__.py index 7c7c385..d34c096 100644 --- a/src/ofw/__init__.py +++ b/src/ofw/__init__.py @@ -31,8 +31,15 @@ FailureDiagnosisError, FailureErrorCode, FailureEvidenceStatus, + FailurePatternMiningError, + FailurePatternMiningErrorCode, + FailurePatternMiningObservation, + FailurePatternMiningStatus, + FailurePatternOrdering, + FailurePatternSummary, FailureType, LangfuseOutcomeStore, + MineFailurePatternsInput, OutcomeErrorCode, OutcomeEvaluation, OutcomeEvaluationError, @@ -109,6 +116,12 @@ def editable(self, path: Path) -> EditableFile: "FailureDiagnosisError", "FailureErrorCode", "FailureEvidenceStatus", + "FailurePatternMiningError", + "FailurePatternMiningErrorCode", + "FailurePatternMiningObservation", + "FailurePatternMiningStatus", + "FailurePatternOrdering", + "FailurePatternSummary", "FailureType", "GitCommit", "Harness", @@ -124,6 +137,7 @@ def editable(self, path: Path) -> EditableFile: "LangfuseProject", "LangfuseSpan", "ModelFingerprint", + "MineFailurePatternsInput", "OutcomeErrorCode", "OutcomeEvaluation", "OutcomeEvaluationError", diff --git a/src/ofw/evaluation/__init__.py b/src/ofw/evaluation/__init__.py index ed4bb47..acf1962 100644 --- a/src/ofw/evaluation/__init__.py +++ b/src/ofw/evaluation/__init__.py @@ -7,6 +7,15 @@ FailureEvidenceStatus, FailureType, ) +from ofw.evaluation.failure_patterns import ( + FailurePatternMiningError, + FailurePatternMiningErrorCode, + FailurePatternMiningObservation, + FailurePatternMiningStatus, + FailurePatternOrdering, + FailurePatternSummary, + MineFailurePatternsInput, +) from ofw.evaluation.langfuse import ( LangfuseOutcomeStore, OutcomeScoreSubmission, @@ -26,8 +35,15 @@ "FailureDiagnosisError", "FailureErrorCode", "FailureEvidenceStatus", + "FailurePatternMiningError", + "FailurePatternMiningErrorCode", + "FailurePatternMiningObservation", + "FailurePatternMiningStatus", + "FailurePatternOrdering", + "FailurePatternSummary", "FailureType", "LangfuseOutcomeStore", + "MineFailurePatternsInput", "OutcomeErrorCode", "OutcomeEvaluation", "OutcomeEvaluationError", diff --git a/src/ofw/evaluation/failure_patterns.py b/src/ofw/evaluation/failure_patterns.py new file mode 100644 index 0000000..06c357a --- /dev/null +++ b/src/ofw/evaluation/failure_patterns.py @@ -0,0 +1,269 @@ +"""Bounded deterministic pattern mining over compact failure diagnoses.""" + +from __future__ import annotations + +import hashlib +import re +from dataclasses import dataclass +from datetime import datetime +from enum import StrEnum +from pathlib import Path +from typing import Annotated, Protocol + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from ofw.evaluation.failure import FailureDiagnosis, FailureEvidenceStatus, FailureType + +_ARTIFACT_ID_PATTERN = r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" +_PATTERN_ID_PATTERN = r"sha256:[0-9a-f]{64}" +_ABSOLUTE_PATH = re.compile(r"(?:/[\w.+-]+){2,}") +_OPAQUE_ID = re.compile( + r"\b(?:(?:request|trace|span|observation|score|session|run)(?:[_-]?id)?|rq)" + r"[_:/-][A-Za-z0-9_-]{12,}\b", + re.IGNORECASE, +) +_UUID = re.compile( + r"\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b", + re.IGNORECASE, +) +_HEX_DIGEST = re.compile(r"\b[0-9a-f]{16,}\b", re.IGNORECASE) +_ULID = re.compile(r"\b[0-7][0-9A-HJKMNP-TV-Z]{25}\b", re.IGNORECASE) +_DIGIT_RUN = re.compile(r"\d+") +_WHITESPACE = re.compile(r"\s+") +_NORMALIZED_CAUSE_LIMIT = 200 +_ARTIFACT_LIMIT = 50 + +ArtifactId = Annotated[str, Field(pattern=_ARTIFACT_ID_PATTERN)] +ArtifactIds = Annotated[ + tuple[ArtifactId, ...], + Field(min_length=1, max_length=_ARTIFACT_LIMIT), +] +PatternId = Annotated[str, Field(pattern=_PATTERN_ID_PATTERN)] +Identifier = Annotated[str, Field(min_length=1, max_length=256)] + + +class StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + +class FailurePatternMiningStatus(StrEnum): + SUCCESS = "success" + + +class FailurePatternOrdering(StrEnum): + OCCURRENCES_TASKS_LATEST_FINGERPRINT = "occurrences_desc_tasks_desc_latest_desc_fingerprint_asc" + + +class FailurePatternMiningErrorCode(StrEnum): + INVALID_WORKSPACE = "invalid_workspace" + ARTIFACT_NOT_FOUND = "artifact_not_found" + INVALID_ARTIFACT = "invalid_artifact" + READ_FAILED = "read_failed" + INVALID_READER_RESULT = "invalid_reader_result" + + +class FailurePatternMiningError(Exception): + """Typed failure while reading or reducing compact diagnosis artifacts.""" + + __slots__ = ("code", "subject") + + def __init__(self, code: FailurePatternMiningErrorCode, subject: str) -> None: + self.code = code + self.subject = subject + super().__init__(f"{code.value}: {subject}") + + +class MineFailurePatternsInput(StrictModel): + workspace_root: Path = Field(strict=False) + artifact_ids: ArtifactIds + + @field_validator("workspace_root") + @classmethod + def validate_workspace_root(cls, value: Path) -> Path: + if not value.is_absolute(): + raise ValueError("workspace_root must be absolute") + return value + + @field_validator("artifact_ids") + @classmethod + def validate_artifact_ids(cls, values: tuple[str, ...]) -> tuple[str, ...]: + if len(set(values)) != len(values): + raise ValueError("artifact_ids must be unique") + return values + + +class FailurePatternSummary(StrictModel): + pattern_id: PatternId + issue_type: FailureType + normalized_root_cause: str = Field(min_length=1, max_length=_NORMALIZED_CAUSE_LIMIT) + occurrence_count: int = Field(strict=True, ge=1, le=_ARTIFACT_LIMIT) + task_ids: tuple[Identifier, ...] = Field(min_length=1, max_length=_ARTIFACT_LIMIT) + trace_ids: tuple[Identifier, ...] = Field(min_length=1, max_length=_ARTIFACT_LIMIT) + artifact_ids: ArtifactIds + first_evaluated_at: datetime + last_evaluated_at: datetime + + +class FailurePatternMiningObservation(StrictModel): + status: FailurePatternMiningStatus + summary: str = Field(min_length=1, max_length=256) + next_actions: tuple[str, ...] = Field(max_length=2) + artifacts: tuple[PatternId, ...] = Field(max_length=_ARTIFACT_LIMIT) + ordering: FailurePatternOrdering + source_artifact_count: int = Field(strict=True, ge=1, le=_ARTIFACT_LIMIT) + patterns: tuple[FailurePatternSummary, ...] = Field(max_length=_ARTIFACT_LIMIT) + inconclusive_artifact_ids: tuple[ArtifactId, ...] = Field(max_length=_ARTIFACT_LIMIT) + + +class FailureArtifactReader(Protocol): + def read( + self, + root: Path, + artifact_ids: tuple[str, ...], + ) -> tuple[FailureDiagnosisRecord, ...]: ... + + +@dataclass(frozen=True, slots=True) +class FailureDiagnosisRecord: + artifact_id: str + diagnosis: FailureDiagnosis + + +@dataclass(slots=True) +class _PatternAccumulator: + pattern_id: str + issue_type: FailureType + normalized_root_cause: str + occurrences: list[FailureDiagnosisRecord] + + +@dataclass(frozen=True, slots=True) +class FailurePatternMiningService: + reader: FailureArtifactReader + + def mine(self, request: MineFailurePatternsInput) -> FailurePatternMiningObservation: + diagnoses = self.reader.read(request.workspace_root, request.artifact_ids) + _validate_reader_result(request.artifact_ids, diagnoses) + patterns, inconclusive = _mine(diagnoses) + summaries = tuple(sorted((_summary(pattern) for pattern in patterns), key=_sort_key)) + return FailurePatternMiningObservation( + status=FailurePatternMiningStatus.SUCCESS, + summary=( + f"Mined {len(summaries)} exact failure patterns from " + f"{len(diagnoses)} diagnosis artifacts." + ), + next_actions=( + "Use repeated supported patterns to prioritize a separate harness hypothesis.", + ), + artifacts=tuple(pattern.pattern_id for pattern in summaries), + ordering=FailurePatternOrdering.OCCURRENCES_TASKS_LATEST_FINGERPRINT, + source_artifact_count=len(diagnoses), + patterns=summaries, + inconclusive_artifact_ids=tuple(sorted(inconclusive)), + ) + + +def normalize_root_cause(value: str) -> str: + """Mask volatile path, identifier, and number runs before exact grouping.""" + normalized = _ABSOLUTE_PATH.sub("", value) + normalized = _OPAQUE_ID.sub("", normalized) + normalized = _UUID.sub("", normalized) + normalized = _HEX_DIGEST.sub("", normalized) + normalized = _ULID.sub("", normalized) + normalized = _DIGIT_RUN.sub("", normalized) + return _WHITESPACE.sub(" ", normalized).strip() + + +def failure_pattern_id(issue_type: FailureType, root_cause: str) -> str: + normalized = normalize_root_cause(root_cause) + identity = "\0".join(("ofw.failure-pattern/1", issue_type.value, normalized)) + return f"sha256:{hashlib.sha256(identity.encode('utf-8')).hexdigest()}" + + +def _validate_reader_result( + requested: tuple[str, ...], + diagnoses: tuple[FailureDiagnosisRecord, ...], +) -> None: + returned = tuple(diagnosis.artifact_id for diagnosis in diagnoses) + if returned != requested: + raise FailurePatternMiningError( + FailurePatternMiningErrorCode.INVALID_READER_RESULT, + "artifact_ids", + ) + + +def _mine( + occurrences: tuple[FailureDiagnosisRecord, ...], +) -> tuple[tuple[_PatternAccumulator, ...], tuple[str, ...]]: + patterns: list[_PatternAccumulator] = [] + inconclusive: list[str] = [] + for occurrence in occurrences: + diagnosis = occurrence.diagnosis + if diagnosis.evidence_status is FailureEvidenceStatus.INCONCLUSIVE: + inconclusive.append(occurrence.artifact_id) + continue + _add_pattern(patterns, occurrence) + return tuple(patterns), tuple(inconclusive) + + +def _add_pattern( + patterns: list[_PatternAccumulator], + occurrence: FailureDiagnosisRecord, +) -> None: + diagnosis = occurrence.diagnosis + issue_type = diagnosis.issue_type + root_cause = diagnosis.root_cause + if issue_type is None or root_cause is None: + raise FailurePatternMiningError( + FailurePatternMiningErrorCode.INVALID_ARTIFACT, + occurrence.artifact_id, + ) + pattern_id = failure_pattern_id(issue_type, root_cause) + for pattern in patterns: + if pattern.pattern_id == pattern_id: + pattern.occurrences.append(occurrence) + return + # ponytail: a linear scan is bounded at 50 artifacts; add an index only if that bound grows. + patterns.append( + _PatternAccumulator( + pattern_id=pattern_id, + issue_type=issue_type, + normalized_root_cause=normalize_root_cause(root_cause), + occurrences=[occurrence], + ) + ) + + +def _summary(pattern: _PatternAccumulator) -> FailurePatternSummary: + occurrences = sorted( + pattern.occurrences, + key=_evaluated_at, + ) + return FailurePatternSummary( + pattern_id=pattern.pattern_id, + issue_type=pattern.issue_type, + normalized_root_cause=pattern.normalized_root_cause[:_NORMALIZED_CAUSE_LIMIT], + occurrence_count=len(pattern.occurrences), + task_ids=tuple( + sorted({occurrence.diagnosis.outcome.task_id.value for occurrence in occurrences}) + ), + trace_ids=tuple( + sorted({occurrence.diagnosis.outcome.trace_id.value for occurrence in occurrences}) + ), + artifact_ids=tuple(sorted(occurrence.artifact_id for occurrence in occurrences)), + first_evaluated_at=occurrences[0].diagnosis.outcome.evaluated_at, + last_evaluated_at=occurrences[-1].diagnosis.outcome.evaluated_at, + ) + + +def _evaluated_at(occurrence: FailureDiagnosisRecord) -> datetime: + return occurrence.diagnosis.outcome.evaluated_at + + +def _sort_key(summary: FailurePatternSummary) -> tuple[int, int, float, str]: + return ( + -summary.occurrence_count, + -len(summary.task_ids), + -summary.last_evaluated_at.timestamp(), + summary.pattern_id, + ) diff --git a/src/ofw/evaluation/failure_workspace.py b/src/ofw/evaluation/failure_workspace.py index 36172ae..b2e0d3f 100644 --- a/src/ofw/evaluation/failure_workspace.py +++ b/src/ofw/evaluation/failure_workspace.py @@ -3,6 +3,8 @@ from __future__ import annotations import os +import re +import stat from collections.abc import Iterator from contextlib import contextmanager from dataclasses import dataclass @@ -12,15 +14,26 @@ from typing import Annotated, Literal, Never, Protocol from uuid import NAMESPACE_URL, uuid4, uuid5 -from pydantic import BaseModel, ConfigDict, Field, field_validator - -from ofw.evaluation.failure import FailureDiagnosis, FailureEvidenceStatus, FailureType -from ofw.evaluation.outcome import OutcomeEvaluation, TaskId, VerifierId +from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator + +from ofw.evaluation.failure import ( + FailureDiagnosis, + FailureDiagnosisError, + FailureEvidenceStatus, + FailureType, +) +from ofw.evaluation.failure_patterns import ( + FailureDiagnosisRecord, + FailurePatternMiningError, + FailurePatternMiningErrorCode, +) +from ofw.evaluation.outcome import OutcomeEvaluation, OutcomeEvaluationError, TaskId, VerifierId from ofw.observability.langfuse.domain import ObservationId, ScoreId, TraceId from ofw.runtime import EvidenceReference, VerifierVerdict _IDENTIFIER_PATTERN = r"[A-Za-z0-9][A-Za-z0-9._:@/-]*" -_ARTIFACT_ID_PATTERN = r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" +_ARTIFACT_ID_PATTERN = r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" +_ARTIFACT_ID = re.compile(_ARTIFACT_ID_PATTERN) _ARTIFACT_LIMIT_BYTES = 64 * 1024 _WORKSPACE_DIRECTORY = ".workspace" _FAILURE_DIRECTORY = "failures" @@ -28,7 +41,7 @@ _WORKSPACE_MARKERS = ("PROGRAM.md", "experiment_config.yaml") _DIRECTORY_FLAGS = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW _CREATE_FILE_FLAGS = os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW -_READ_FILE_FLAGS = os.O_RDONLY | os.O_NOFOLLOW +_READ_FILE_FLAGS = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK Identifier = Annotated[ str, @@ -180,6 +193,32 @@ def from_diagnosis( inconclusive_reason=diagnosis.inconclusive_reason, ) + def to_diagnosis(self) -> FailureDiagnosis: + outcome = OutcomeEvaluation( + trace_id=TraceId(self.trace_id), + task_id=TaskId(self.task_id), + verifier_id=VerifierId(self.verifier_id), + evaluated_at=self.evaluated_at, + verdict=VerifierVerdict.FAIL, + score=self.normalized_score, + evidence=tuple(EvidenceReference(value) for value in self.outcome_evidence), + ) + return FailureDiagnosis( + outcome=outcome, + outcome_score_id=ScoreId(self.outcome_score_id), + evidence_status=self.evidence_status, + issue_type=self.issue_type, + expected_outcome=self.expected_outcome, + actual_outcome=self.actual_outcome, + critical_observation_id=_observation_id(self.critical_observation_id), + evidence_observation_ids=tuple( + ObservationId(value) for value in self.evidence_observation_ids + ), + root_cause=self.root_cause, + counterfactual_action=self.counterfactual_action, + inconclusive_reason=self.inconclusive_reason, + ) + @dataclass(frozen=True, slots=True) class FailureArtifactReceipt: @@ -248,6 +287,23 @@ def store(self, root: Path, diagnosis: FailureDiagnosis) -> FailureArtifactRecei artifact_id, ) from None + def read( + self, + root: Path, + artifact_ids: tuple[str, ...], + ) -> tuple[FailureDiagnosisRecord, ...]: + try: + return self._read(root, artifact_ids) + except FailurePatternMiningError: + raise + except FailureWorkspaceFailure as error: + raise _pattern_read_error(error) from None + except (OSError, RuntimeError, UnicodeError): + raise FailurePatternMiningError( + FailurePatternMiningErrorCode.READ_FAILED, + "failure_artifacts", + ) from None + def _store( self, root: Path, @@ -273,6 +329,21 @@ def _store( _publish_or_validate(directory, path.name, content, artifact_id) return receipt + def _read( + self, + root: Path, + artifact_ids: tuple[str, ...], + ) -> tuple[FailureDiagnosisRecord, ...]: + prepared_root = _prepared_root(root) + workspace, failures = _workspace_paths(prepared_root) + directory_identity = _existing_workspace_directories( + prepared_root, + workspace, + failures, + ) + with _failure_directory_handle(prepared_root, directory_identity) as directory: + return tuple(_read_artifact(directory, artifact_id) for artifact_id in artifact_ids) + def _observation_id(value: str | None) -> ObservationId | None: return None if value is None else ObservationId(value) @@ -372,6 +443,23 @@ def _prepare_workspace_directories( ) +def _existing_workspace_directories( + root: Path, + workspace: Path, + failures: Path, +) -> _DirectoryChainIdentity: + if not workspace.is_dir() or not failures.is_dir(): + raise FailurePatternMiningError( + FailurePatternMiningErrorCode.ARTIFACT_NOT_FOUND, + _FAILURE_DIRECTORY, + ) + return _DirectoryChainIdentity( + root=_path_identity(root), + workspace=_path_identity(workspace), + failures=_path_identity(failures), + ) + + def _write_ignore_file(directory: int) -> None: try: _write_new_file(directory, ".gitignore", _IGNORE_CONTENT.encode("utf-8")) @@ -439,8 +527,63 @@ def _validate_existing( ) +def _read_artifact(directory: int, artifact_id: str) -> FailureDiagnosisRecord: + _require_artifact_id(artifact_id) + try: + content = _read_existing(directory, f"{artifact_id}.json", artifact_id) + except FileNotFoundError: + raise FailurePatternMiningError( + FailurePatternMiningErrorCode.ARTIFACT_NOT_FOUND, + artifact_id, + ) from None + except FailureWorkspaceFailure as error: + raise _pattern_read_error(error) from None + return _parse_artifact(content, artifact_id) + + +def _require_artifact_id(artifact_id: str) -> None: + if _ARTIFACT_ID.fullmatch(artifact_id) is None: + raise FailurePatternMiningError( + FailurePatternMiningErrorCode.INVALID_ARTIFACT, + "artifact_id", + ) + + +def _parse_artifact(content: bytes, artifact_id: str) -> FailureDiagnosisRecord: + try: + artifact = FailureArtifact.model_validate_json(content) + diagnosis = artifact.to_diagnosis() + except (FailureDiagnosisError, OutcomeEvaluationError, ValidationError): + raise FailurePatternMiningError( + FailurePatternMiningErrorCode.INVALID_ARTIFACT, + artifact_id, + ) from None + if artifact.artifact_id != artifact_id: + raise FailurePatternMiningError( + FailurePatternMiningErrorCode.INVALID_ARTIFACT, + artifact_id, + ) + return FailureDiagnosisRecord(artifact_id, diagnosis) + + +def _pattern_read_error(error: FailureWorkspaceFailure) -> FailurePatternMiningError: + if error.code is FailureWorkspaceErrorCode.INVALID_WORKSPACE: + code = FailurePatternMiningErrorCode.INVALID_WORKSPACE + elif error.code is FailureWorkspaceErrorCode.ARTIFACT_TOO_LARGE: + code = FailurePatternMiningErrorCode.INVALID_ARTIFACT + else: + code = FailurePatternMiningErrorCode.READ_FAILED + return FailurePatternMiningError(code, error.subject) + + def _read_existing(directory: int, name: str, artifact_id: str) -> bytes: descriptor = os.open(name, _READ_FILE_FLAGS, dir_fd=directory) + try: + if not stat.S_ISREG(os.fstat(descriptor).st_mode): + raise OSError("failure artifact is not a regular file") + except OSError: + os.close(descriptor) + raise with os.fdopen(descriptor, "rb") as stream: content = stream.read(_ARTIFACT_LIMIT_BYTES + 1) if len(content) > _ARTIFACT_LIMIT_BYTES: diff --git a/src/ofw/mcp.py b/src/ofw/mcp.py index 5728f83..572caa9 100644 --- a/src/ofw/mcp.py +++ b/src/ofw/mcp.py @@ -14,6 +14,11 @@ from mcp.types import ToolAnnotations from pydantic import BaseModel, Field +from ofw.evaluation.failure_patterns import ( + FailurePatternMiningObservation, + FailurePatternMiningService, + MineFailurePatternsInput, +) from ofw.evaluation.failure_workspace import ( FailureRecordObservation, FailureWorkspaceService, @@ -68,8 +73,8 @@ name="openflywheel", instructions=( "Prepare isolated ITSM harness workspaces, read bounded Langfuse trace evidence, and " - "record authoritative outcomes plus compact failure diagnoses. Never infer outcomes, " - "mutate traces, or copy trace payloads into local storage." + "record authoritative outcomes plus compact failure diagnoses and exact patterns. " + "Never infer outcomes, mutate traces, or copy trace payloads into local storage." ), log_level="DEBUG", ) @@ -130,6 +135,10 @@ def _failure_service() -> FailureWorkspaceService: return FailureWorkspaceService(FileFailureWorkspace()) +def _failure_pattern_service() -> FailurePatternMiningService: + return FailurePatternMiningService(FileFailureWorkspace()) + + def _program_template(name: str) -> str: content = files("ofw.preparation.templates").joinpath(name).read_bytes() if len(content) > _PROGRAM_TEMPLATE_LIMIT_BYTES: @@ -259,6 +268,14 @@ def record_failure(request: RecordFailureInput) -> FailureRecordObservation: return _failure_service().record(request) +@server.tool(annotations=read_only, structured_output=True) +def mine_failure_patterns( + request: MineFailurePatternsInput, +) -> FailurePatternMiningObservation: + """Group explicit compact diagnoses by exact normalized root cause.""" + return _failure_pattern_service().mine(request) + + def main() -> None: """Run the OpenFlywheel MCP server over stdio.""" server.run(transport="stdio") diff --git a/src/ofw/preparation/templates/itsm.md b/src/ofw/preparation/templates/itsm.md index a732bf3..9174db3 100644 --- a/src/ofw/preparation/templates/itsm.md +++ b/src/ofw/preparation/templates/itsm.md @@ -44,6 +44,11 @@ under `.workspace/failures/` before forming a harness hypothesis. Record an inco diagnosis when the evidence is insufficient. Do not copy Langfuse trace payloads into the workspace; Langfuse remains their source of truth. +After recording the diagnoses for one bounded run, use `$failure-pattern-miner` and call +`mine_failure_patterns` with only the returned artifact IDs from that run. It groups by failure +type and exact normalized root cause; results are not semantic clusters. Keep inconclusive +diagnoses separate, and reread each supporting diagnosis before forming a shared hypothesis. + An intermediate tool error is evidence, not an outcome failure, when the agent recovered and the verifier passed. A technically clean trajectory is still a failure when the ITSM verifier shows that the required environment state was not achieved. diff --git a/tests/test_failure_patterns.py b/tests/test_failure_patterns.py new file mode 100644 index 0000000..b318c21 --- /dev/null +++ b/tests/test_failure_patterns.py @@ -0,0 +1,428 @@ +"""Bounded deterministic failure-pattern mining tests.""" + +from __future__ import annotations + +import os +import subprocess +import sys +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from ofw.evaluation.failure import FailureEvidenceStatus, FailureType +from ofw.evaluation.failure_patterns import ( + FailureDiagnosisRecord, + FailurePatternMiningError, + FailurePatternMiningErrorCode, + FailurePatternMiningService, + FailurePatternMiningStatus, + FailurePatternOrdering, + MineFailurePatternsInput, + failure_pattern_id, + normalize_root_cause, +) +from ofw.evaluation.failure_workspace import ( + FailedOutcomeInput, + FailureWorkspaceService, + FileFailureWorkspace, + RecordFailureInput, +) + +_EVALUATED_AT = datetime(2026, 8, 28, 6, 0, tzinfo=UTC) + + +def _git(root: Path, *arguments: str) -> str: + return subprocess.run( + ("git", "-C", str(root), *arguments), + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + +def _prepared_workspace(tmp_path: Path) -> Path: + root = tmp_path / "harness" + root.mkdir() + (root / "PROGRAM.md").write_text("# Program\n", encoding="utf-8") + (root / "experiment_config.yaml").write_text( + "benchmark: itsm-bench\n", + encoding="utf-8", + ) + _git(root, "init", "-q") + _git(root, "config", "user.email", "test@example.com") + _git(root, "config", "user.name", "Test") + _git(root, "add", "PROGRAM.md", "experiment_config.yaml") + _git(root, "commit", "-qm", "prepare") + return root + + +def _request( + root: Path, + *, + trace_id: str, + task_id: str, + evaluated_at: datetime, + root_cause: str | None, + evidence_status: FailureEvidenceStatus = FailureEvidenceStatus.SUPPORTED, + issue_type: FailureType | None = FailureType.CONTROL_FLOW_FAILURE, +) -> RecordFailureInput: + supported = evidence_status is FailureEvidenceStatus.SUPPORTED + critical = f"observation-{task_id}" if supported else None + return RecordFailureInput( + workspace_root=root, + outcome=FailedOutcomeInput( + trace_id=trace_id, + task_id=task_id, + verifier_id="itsm-bench@v1", + evaluated_at=evaluated_at, + score=0.0, + evidence=(f"harbor://{task_id}/verifier/result",), + outcome_score_id=f"score-{trace_id}", + ), + evidence_status=evidence_status, + issue_type=issue_type if supported else None, + expected_outcome="Incident is closed.", + actual_outcome="Incident remains open.", + critical_observation_id=critical, + evidence_observation_ids=() if critical is None else (critical,), + root_cause=root_cause if supported else None, + counterfactual_action="Read state before finalizing." if supported else None, + inconclusive_reason=None if supported else "The trace omitted the final tool result.", + ) + + +def _record(service: FailureWorkspaceService, request: RecordFailureInput) -> str: + return service.record(request).artifact_id + + +def test_normalization_and_fingerprint_ignore_volatile_values() -> None: + first = ( + "Session /tmp/ofw-run-123/trace.json failed for request " + "request-id-Abc123Xyz987654 at line 7" + ) + second = ( + "Session /var/tmp/ofw-run-999/trace.json failed for request rq-9876543210AbCdEf at line 42" + ) + + assert normalize_root_cause(first) == normalize_root_cause(second) + assert "Abc123" not in normalize_root_cause(first) + assert failure_pattern_id(FailureType.CONTROL_FLOW_FAILURE, first) == failure_pattern_id( + FailureType.CONTROL_FLOW_FAILURE, + second, + ) + assert failure_pattern_id(FailureType.POLICY_FAILURE, first) != failure_pattern_id( + FailureType.CONTROL_FLOW_FAILURE, + first, + ) + assert "supercalifragilistic" in normalize_root_cause( + "The parser rejected the legitimate supercalifragilistic field." + ) + assert normalize_root_cause( + "Trace trace-01ARZ3NDEKTSV4RRFFQ69G5FAV failed." + ) == normalize_root_cause("Trace trace-01BX5ZZKBKACTAV9WEVGEMMVRZ failed.") + assert normalize_root_cause("Run 01ARZ3NDEKTSV4RRFFQ69G5FAV failed.") == normalize_root_cause( + "Run 01BX5ZZKBKACTAV9WEVGEMMVRZ failed." + ) + + +def test_fingerprint_uses_the_complete_normalized_root_cause() -> None: + shared = "The agent repeated a recoverable operation without checking state. " * 4 + + assert len(shared) > 200 + assert failure_pattern_id( + FailureType.CONTROL_FLOW_FAILURE, + shared + "It then finalized early.", + ) != failure_pattern_id( + FailureType.CONTROL_FLOW_FAILURE, + shared + "It then corrupted state.", + ) + + +def test_mines_exact_patterns_and_keeps_inconclusive_diagnoses_separate( + tmp_path: Path, +) -> None: + root = _prepared_workspace(tmp_path) + recorder = FailureWorkspaceService(FileFailureWorkspace()) + first = _record( + recorder, + _request( + root, + trace_id="trace-1", + task_id="task-1", + evaluated_at=_EVALUATED_AT, + root_cause="The agent stopped after attempt 1 in /tmp/run-1/state.json.", + ), + ) + second = _record( + recorder, + _request( + root, + trace_id="trace-2", + task_id="task-2", + evaluated_at=_EVALUATED_AT + timedelta(minutes=1), + root_cause="The agent stopped after attempt 9 in /var/tmp/run-2/state.json.", + ), + ) + third = _record( + recorder, + _request( + root, + trace_id="trace-3", + task_id="task-3", + evaluated_at=_EVALUATED_AT + timedelta(minutes=2), + root_cause="The agent ignored the required approval.", + issue_type=FailureType.POLICY_FAILURE, + ), + ) + inconclusive = _record( + recorder, + _request( + root, + trace_id="trace-4", + task_id="task-4", + evaluated_at=_EVALUATED_AT + timedelta(minutes=3), + root_cause=None, + evidence_status=FailureEvidenceStatus.INCONCLUSIVE, + issue_type=None, + ), + ) + request = MineFailurePatternsInput( + workspace_root=root, + artifact_ids=(third, inconclusive, second, first), + ) + + result = FailurePatternMiningService(FileFailureWorkspace()).mine(request) + + assert result.status is FailurePatternMiningStatus.SUCCESS + assert result.ordering is FailurePatternOrdering.OCCURRENCES_TASKS_LATEST_FINGERPRINT + assert result.source_artifact_count == 4 + assert result.inconclusive_artifact_ids == (inconclusive,) + assert len(result.patterns) == 2 + repeated, policy = result.patterns + assert repeated.occurrence_count == 2 + assert repeated.issue_type is FailureType.CONTROL_FLOW_FAILURE + assert repeated.task_ids == ("task-1", "task-2") + assert repeated.trace_ids == ("trace-1", "trace-2") + assert repeated.artifact_ids == tuple(sorted((first, second))) + assert repeated.first_evaluated_at == _EVALUATED_AT + assert repeated.last_evaluated_at == _EVALUATED_AT + timedelta(minutes=1) + assert policy.occurrence_count == 1 + assert result.artifacts == ( + failure_pattern_id( + FailureType.CONTROL_FLOW_FAILURE, + "The agent stopped after attempt 1 in /tmp/run-1/state.json.", + ), + failure_pattern_id( + FailureType.POLICY_FAILURE, + "The agent ignored the required approval.", + ), + ) + + +def test_reader_loads_only_explicit_artifact_ids(tmp_path: Path) -> None: + root = _prepared_workspace(tmp_path) + recorder = FailureWorkspaceService(FileFailureWorkspace()) + selected = _record( + recorder, + _request( + root, + trace_id="trace-selected", + task_id="task-selected", + evaluated_at=_EVALUATED_AT, + root_cause="Selected failure.", + ), + ) + _record( + recorder, + _request( + root, + trace_id="trace-unselected", + task_id="task-unselected", + evaluated_at=_EVALUATED_AT, + root_cause="Unselected failure.", + ), + ) + + result = FailurePatternMiningService(FileFailureWorkspace()).mine( + MineFailurePatternsInput(workspace_root=root, artifact_ids=(selected,)) + ) + + assert result.source_artifact_count == 1 + assert result.patterns[0].artifact_ids == (selected,) + + +def test_missing_or_tampered_artifact_fails_closed(tmp_path: Path) -> None: + root = _prepared_workspace(tmp_path) + recorder = FailureWorkspaceService(FileFailureWorkspace()) + artifact_id = _record( + recorder, + _request( + root, + trace_id="trace-1", + task_id="task-1", + evaluated_at=_EVALUATED_AT, + root_cause="The agent finalized early.", + ), + ) + service = FailurePatternMiningService(FileFailureWorkspace()) + + with pytest.raises(FailurePatternMiningError) as missing: + service.mine( + MineFailurePatternsInput( + workspace_root=root, + artifact_ids=("11111111-1111-1111-1111-111111111111",), + ) + ) + assert missing.value.code is FailurePatternMiningErrorCode.ARTIFACT_NOT_FOUND + + artifact_path = root / ".workspace/failures" / f"{artifact_id}.json" + artifact_path.write_text('{"schema_version":1}', encoding="utf-8") + with pytest.raises(FailurePatternMiningError) as invalid: + service.mine(MineFailurePatternsInput(workspace_root=root, artifact_ids=(artifact_id,))) + assert invalid.value.code is FailurePatternMiningErrorCode.INVALID_ARTIFACT + + +def test_reader_rejects_path_shaped_artifact_ids_at_both_seams(tmp_path: Path) -> None: + root = _prepared_workspace(tmp_path) + recorder = FailureWorkspaceService(FileFailureWorkspace()) + artifact_id = _record( + recorder, + _request( + root, + trace_id="trace-1", + task_id="task-1", + evaluated_at=_EVALUATED_AT, + root_cause="The agent finalized early.", + ), + ) + invalid = f"../{artifact_id}" + + with pytest.raises(ValidationError): + MineFailurePatternsInput(workspace_root=root, artifact_ids=(invalid,)) + with pytest.raises(FailurePatternMiningError) as raised: + FileFailureWorkspace().read(root, (invalid,)) + + assert raised.value.code is FailurePatternMiningErrorCode.INVALID_ARTIFACT + + +def test_reader_rejects_a_fifo_without_blocking(tmp_path: Path) -> None: + root = _prepared_workspace(tmp_path) + recorder = FailureWorkspaceService(FileFailureWorkspace()) + artifact_id = _record( + recorder, + _request( + root, + trace_id="trace-1", + task_id="task-1", + evaluated_at=_EVALUATED_AT, + root_cause="The agent finalized early.", + ), + ) + artifact_path = root / ".workspace/failures" / f"{artifact_id}.json" + artifact_path.unlink() + os.mkfifo(artifact_path) + script = ( + "import sys\n" + "from pathlib import Path\n" + "sys.path.insert(0, sys.argv[3])\n" + "from ofw.evaluation.failure_patterns import (" + "FailurePatternMiningError, FailurePatternMiningErrorCode, " + "FailurePatternMiningService, MineFailurePatternsInput)\n" + "from ofw.evaluation.failure_workspace import FileFailureWorkspace\n" + "request = MineFailurePatternsInput(workspace_root=Path(sys.argv[1]), " + "artifact_ids=(sys.argv[2],))\n" + "try:\n" + " FailurePatternMiningService(FileFailureWorkspace()).mine(request)\n" + "except FailurePatternMiningError as error:\n" + " expected = FailurePatternMiningErrorCode.READ_FAILED\n" + " raise SystemExit(0 if error.code is expected else 2)\n" + "raise SystemExit(3)\n" + ) + + subprocess.run( + ( + sys.executable, + "-c", + script, + str(root), + artifact_id, + str(Path(__file__).parents[1] / "src"), + ), + check=True, + timeout=2, + ) + + +def test_pattern_request_is_strict_bounded_and_immutable(tmp_path: Path) -> None: + root = _prepared_workspace(tmp_path) + request = MineFailurePatternsInput( + workspace_root=root, + artifact_ids=("11111111-1111-1111-1111-111111111111",), + ) + + with pytest.raises(ValidationError): + MineFailurePatternsInput( + workspace_root=root, + artifact_ids=(request.artifact_ids[0], request.artifact_ids[0]), + ) + with pytest.raises(ValidationError): + MineFailurePatternsInput( + workspace_root=root, + artifact_ids=tuple(f"00000000-0000-0000-0000-{index:012d}" for index in range(51)), + ) + with pytest.raises(ValidationError): + MineFailurePatternsInput( + workspace_root=Path("relative"), + artifact_ids=request.artifact_ids, + ) + with pytest.raises(ValidationError): + MineFailurePatternsInput.model_validate_json( + request.model_dump_json().removesuffix("}") + ',"unexpected":true}' + ) + with pytest.raises(ValidationError): + request.artifact_ids = () + + +def test_pattern_output_rejects_a_reader_contract_violation(tmp_path: Path) -> None: + root = _prepared_workspace(tmp_path) + request = MineFailurePatternsInput( + workspace_root=root, + artifact_ids=("11111111-1111-1111-1111-111111111111",), + ) + + class EmptyReader: + def read( + self, + workspace_root: Path, + artifact_ids: tuple[str, ...], + ) -> tuple[FailureDiagnosisRecord, ...]: + del workspace_root, artifact_ids + return () + + with pytest.raises(FailurePatternMiningError) as raised: + FailurePatternMiningService(EmptyReader()).mine(request) + + assert raised.value.code is FailurePatternMiningErrorCode.INVALID_READER_RESULT + + +def test_pattern_summary_is_immutable(tmp_path: Path) -> None: + root = _prepared_workspace(tmp_path) + recorder = FailureWorkspaceService(FileFailureWorkspace()) + artifact_id = _record( + recorder, + _request( + root, + trace_id="trace-1", + task_id="task-1", + evaluated_at=_EVALUATED_AT, + root_cause="The agent finalized early.", + ), + ) + result = FailurePatternMiningService(FileFailureWorkspace()).mine( + MineFailurePatternsInput(workspace_root=root, artifact_ids=(artifact_id,)) + ) + + with pytest.raises(ValidationError): + result.patterns[0].occurrence_count = 2 diff --git a/tests/test_openflywheel_mcp.py b/tests/test_openflywheel_mcp.py index 8948ca5..f2f2dd1 100644 --- a/tests/test_openflywheel_mcp.py +++ b/tests/test_openflywheel_mcp.py @@ -13,6 +13,12 @@ from mcp.types import Tool from ofw.evaluation.failure import FailureEvidenceStatus, FailureType +from ofw.evaluation.failure_patterns import ( + FailurePatternMiningObservation, + FailurePatternMiningStatus, + FailurePatternOrdering, + MineFailurePatternsInput, +) from ofw.evaluation.failure_workspace import ( FailedOutcomeInput, FailureRecordObservation, @@ -20,6 +26,7 @@ FailureWorkspaceErrorCode, FailureWorkspaceFailure, FailureWorkspaceService, + FileFailureWorkspace, RecordFailureInput, ) from ofw.evaluation.langfuse import ( @@ -108,6 +115,11 @@ def record_outcome( def record_failure(self, request: RecordFailureInput) -> FailureRecordObservation: ... + def mine_failure_patterns( + self, + request: MineFailurePatternsInput, + ) -> FailurePatternMiningObservation: ... + class _FakeOutcomeStore: def __init__(self) -> None: @@ -240,6 +252,7 @@ def test_mcp_exposes_scoped_read_and_recording_tools() -> None: "get_span_context", "record_outcome", "record_failure", + "mine_failure_patterns", ] assert tuple(map(_annotation_flags, tools)) == ( (False, False, True), @@ -249,6 +262,7 @@ def test_mcp_exposes_scoped_read_and_recording_tools() -> None: (True, False, True), (False, False, True), (False, False, True), + (True, False, True), ) @@ -464,3 +478,29 @@ def test_record_failure_preserves_typed_workspace_errors( assert raised.value.code is FailureWorkspaceErrorCode.WRITE_FAILED assert str(raised.value) == f"write_failed: {_FAILURE_ARTIFACT_ID}" + + +def test_mine_failure_patterns_passes_one_bounded_object_to_the_service( + tmp_path: Path, +) -> None: + module = _module() + (tmp_path / "PROGRAM.md").write_text("# Program\n", encoding="utf-8") + (tmp_path / "experiment_config.yaml").write_text( + "benchmark: itsm-bench\n", + encoding="utf-8", + ) + recorded = FailureWorkspaceService(FileFailureWorkspace()).record( + _inconclusive_failure_request(tmp_path) + ) + request = MineFailurePatternsInput( + workspace_root=tmp_path, + artifact_ids=(recorded.artifact_id,), + ) + + result = module.mine_failure_patterns(request) + + assert result.status is FailurePatternMiningStatus.SUCCESS + assert result.ordering is FailurePatternOrdering.OCCURRENCES_TASKS_LATEST_FINGERPRINT + assert result.source_artifact_count == 1 + assert result.patterns == () + assert result.inconclusive_artifact_ids == (recorded.artifact_id,) diff --git a/tests/test_plugin_packaging.py b/tests/test_plugin_packaging.py index 99be14e..9653e81 100644 --- a/tests/test_plugin_packaging.py +++ b/tests/test_plugin_packaging.py @@ -28,7 +28,7 @@ def test_openflywheel_mcp_uses_pinned_portable_runtime() -> None: server = manifest.mcpServers["openflywheel"] assert ( - "git+https://github.com/divo12/OpenFlyWheel.git@0d4af6e0499c54765e2f7784b48b53baf96d1ad5" + "git+https://github.com/divo12/OpenFlyWheel.git@ab0ef62cbe1e6cddf0bfd8ec61374d10120c61aa" in server.args ) assert "openflywheel-mcp" in server.args diff --git a/tests/test_program_templates.py b/tests/test_program_templates.py index 2ec35fc..0557d16 100644 --- a/tests/test_program_templates.py +++ b/tests/test_program_templates.py @@ -29,3 +29,27 @@ def test_itsm_program_routes_failure_mining_to_local_workspace_artifacts( content = files("ofw.preparation.templates").joinpath("itsm.md").read_text(encoding="utf-8") assert required_instruction in content + + +@pytest.mark.parametrize( + "required_instruction", + ( + "$failure-pattern-miner", + "mine_failure_patterns", + "exact normalized root cause", + "not semantic clusters", + ), +) +def test_itsm_program_routes_recorded_diagnoses_to_bounded_pattern_mining( + required_instruction: str, +) -> None: + content = files("ofw.preparation.templates").joinpath("itsm.md").read_text(encoding="utf-8") + + assert required_instruction in content + + +def test_failure_pattern_miner_skill_is_packaged() -> None: + skill = Path(__file__).parents[1] / "plugins/openflywheel/skills/failure-pattern-miner/SKILL.md" + + assert skill.is_file() + assert "mine_failure_patterns" in skill.read_text(encoding="utf-8") diff --git a/tests/test_typing.py b/tests/test_typing.py index 191fe97..2f8004e 100644 --- a/tests/test_typing.py +++ b/tests/test_typing.py @@ -40,6 +40,16 @@ def test_namespace_exports_failure_diagnosis_contract() -> None: assert "FailureType" in package.__all__ +def test_namespace_exports_failure_pattern_contract() -> None: + assert "FailurePatternMiningError" in package.__all__ + assert "FailurePatternMiningErrorCode" in package.__all__ + assert "FailurePatternMiningObservation" in package.__all__ + assert "FailurePatternMiningStatus" in package.__all__ + assert "FailurePatternOrdering" in package.__all__ + assert "FailurePatternSummary" in package.__all__ + assert "MineFailurePatternsInput" in package.__all__ + + def test_namespace_exports_workspace_preparation_contract() -> None: assert "PreparationErrorCode" in package.__all__ assert "PreparationPhase" in package.__all__