diff --git a/src/agentops/cli/config_commands.py b/src/agentops/cli/config_commands.py index f435b444..80809ef0 100644 --- a/src/agentops/cli/config_commands.py +++ b/src/agentops/cli/config_commands.py @@ -6,7 +6,6 @@ import typer -from agentops.cli._planned import _planned_command from agentops.utils.logging import get_logger log = get_logger(__name__) @@ -15,15 +14,80 @@ @config_app.command("validate") -def cmd_config_validate() -> None: - """Validate configuration files (planned).""" - _planned_command("agentops config validate") +def cmd_config_validate( + config: Path | None = typer.Option( + None, + "--config", + "-c", + help="Path to run.yaml (default: .agentops/run.yaml).", + ), + directory: Path = typer.Option( + Path("."), + "--dir", + help="Workspace directory.", + ), +) -> None: + """Validate configuration files (run.yaml, bundle, dataset, data).""" + from agentops.services.validate import validate_config + + try: + result = validate_config(config_path=config, directory=directory) + except FileNotFoundError as exc: + typer.echo(f"Error: {exc}", err=True) + raise typer.Exit(code=1) from exc + + typer.echo(f"Files checked: {result.files_checked}") + for issue in result.issues: + marker = "ERROR" if issue.severity == "error" else "WARN" + typer.echo(f" [{marker}] {issue.file.name}: {issue.message}") + + if result.passed: + typer.echo("Validation: PASSED") + else: + typer.echo("Validation: FAILED") + raise typer.Exit(code=1) @config_app.command("show") -def cmd_config_show() -> None: - """Show merged runtime config (planned).""" - _planned_command("agentops config show") +def cmd_config_show( + config: Path | None = typer.Option( + None, + "--config", + "-c", + help="Path to run.yaml (default: .agentops/run.yaml).", + ), + directory: Path = typer.Option( + Path("."), + "--dir", + help="Workspace directory.", + ), +) -> None: + """Show resolved configuration from a run.yaml file.""" + from agentops.services.validate import show_config + + try: + result = show_config(config_path=config, directory=directory) + except (FileNotFoundError, ValueError) as exc: + typer.echo(f"Error: {exc}", err=True) + raise typer.Exit(code=1) from exc + + typer.echo(f"Run config: {result.run_config_path}") + typer.echo(f"Bundle: {result.bundle_name} ({result.bundle_path})") + typer.echo(f"Dataset: {result.dataset_name} ({result.dataset_path})") + if result.data_path: + rows = f" ({result.data_rows} rows)" if result.data_rows is not None else "" + typer.echo(f"Data: {result.data_path}{rows}") + typer.echo(f"Backend: {result.backend_type} (target={result.target})") + if result.model: + typer.echo(f"Model: {result.model}") + if result.agent_id: + typer.echo(f"Agent: {result.agent_id}") + typer.echo(f"Thresholds: {result.thresholds}") + typer.echo("") + typer.echo("Evaluators:") + for e in result.evaluators: + status = "enabled" if e["enabled"] else "disabled" + typer.echo(f" {e['name']} (source={e['source']}, {status})") @config_app.command("cicd") diff --git a/src/agentops/cli/dataset_commands.py b/src/agentops/cli/dataset_commands.py index c768963c..694c48ef 100644 --- a/src/agentops/cli/dataset_commands.py +++ b/src/agentops/cli/dataset_commands.py @@ -2,6 +2,8 @@ from __future__ import annotations +from pathlib import Path + import typer from agentops.cli._planned import _planned_command @@ -10,15 +12,59 @@ @dataset_app.command("validate") -def cmd_dataset_validate() -> None: - """Validate dataset files (planned).""" - _planned_command("agentops dataset validate") +def cmd_dataset_validate( + dataset: Path = typer.Argument(help="Path to dataset YAML config file."), +) -> None: + """Validate a dataset config and its JSONL data file.""" + from agentops.services.validate import validate_dataset + + try: + result = validate_dataset(dataset_path=dataset) + except (FileNotFoundError, ValueError) as exc: + typer.echo(f"Error: {exc}", err=True) + raise typer.Exit(code=1) from exc + + typer.echo(f"Files checked: {result.files_checked}") + for issue in result.issues: + marker = "ERROR" if issue.severity == "error" else "WARN" + typer.echo(f" [{marker}] {issue.file.name}: {issue.message}") + + if result.passed: + typer.echo("Validation: PASSED") + else: + typer.echo("Validation: FAILED") + raise typer.Exit(code=1) @dataset_app.command("describe") -def cmd_dataset_describe() -> None: - """Describe dataset schema and shape (planned).""" - _planned_command("agentops dataset describe") +def cmd_dataset_describe( + dataset: Path = typer.Argument(help="Path to dataset YAML config file."), +) -> None: + """Describe dataset schema, fields, and row count.""" + from agentops.services.validate import describe_dataset + + try: + desc = describe_dataset(dataset_path=dataset) + except (FileNotFoundError, ValueError) as exc: + typer.echo(f"Error: {exc}", err=True) + raise typer.Exit(code=1) from exc + + typer.echo(f"Dataset: {desc.name}") + if desc.description: + typer.echo(f"Description: {desc.description}") + typer.echo(f"Source: {desc.source_type}") + typer.echo(f"Format: {desc.format_type}") + typer.echo(f"Input field: {desc.input_field}") + typer.echo(f"Expected field: {desc.expected_field}") + if desc.context_field: + typer.echo(f"Context field: {desc.context_field}") + if desc.data_path: + typer.echo(f"Data file: {desc.data_path}") + typer.echo(f"Rows: {desc.row_count}") + if desc.fields: + typer.echo(f"Fields: {', '.join(desc.fields)}") + if desc.metadata: + typer.echo(f"Metadata: {desc.metadata}") @dataset_app.command("import") diff --git a/src/agentops/services/_workspace.py b/src/agentops/services/_workspace.py new file mode 100644 index 00000000..f01a94fb --- /dev/null +++ b/src/agentops/services/_workspace.py @@ -0,0 +1,21 @@ +"""Shared workspace resolution for CLI services.""" + +from __future__ import annotations + +from pathlib import Path + +_DEFAULT_AGENTOPS_DIR = ".agentops" + + +def resolve_workspace(directory: Path) -> Path: + """Resolve the .agentops workspace directory. + + Raises: + FileNotFoundError: If no .agentops directory exists. + """ + workspace = (directory / _DEFAULT_AGENTOPS_DIR).resolve() + if not workspace.is_dir(): + raise FileNotFoundError( + f"No .agentops workspace found at {workspace}. Run 'agentops init' first." + ) + return workspace diff --git a/src/agentops/services/browse.py b/src/agentops/services/browse.py index 93f777d9..37e0506a 100644 --- a/src/agentops/services/browse.py +++ b/src/agentops/services/browse.py @@ -9,23 +9,7 @@ from agentops.core.config_loader import load_bundle_config from agentops.core.models import RunResult - - -# --------------------------------------------------------------------------- -# Workspace resolution -# --------------------------------------------------------------------------- - -_DEFAULT_AGENTOPS_DIR = ".agentops" - - -def _resolve_workspace(directory: Path) -> Path: - """Resolve the .agentops workspace directory.""" - workspace = (directory / _DEFAULT_AGENTOPS_DIR).resolve() - if not workspace.is_dir(): - raise FileNotFoundError( - f"No .agentops workspace found at {workspace}. Run 'agentops init' first." - ) - return workspace +from agentops.services._workspace import resolve_workspace # --------------------------------------------------------------------------- @@ -54,7 +38,7 @@ class BundleListResult: def list_bundles(directory: Path = Path(".")) -> BundleListResult: """List all bundle YAML files in the workspace.""" - workspace = _resolve_workspace(directory) + workspace = resolve_workspace(directory) bundles_dir = workspace / "bundles" if not bundles_dir.is_dir(): @@ -103,7 +87,7 @@ class BundleDetail: def show_bundle(bundle_name: str, directory: Path = Path(".")) -> BundleDetail: """Load and return full details of a bundle by name.""" - workspace = _resolve_workspace(directory) + workspace = resolve_workspace(directory) bundles_dir = workspace / "bundles" # Try exact filename first, then search by bundle name @@ -190,7 +174,7 @@ class RunListResult: def list_runs(directory: Path = Path(".")) -> RunListResult: """List all past evaluation runs in the workspace.""" - workspace = _resolve_workspace(directory) + workspace = resolve_workspace(directory) results_dir = workspace / "results" if not results_dir.is_dir(): @@ -266,7 +250,7 @@ class RunDetail: def show_run(run_id: str, directory: Path = Path(".")) -> RunDetail: """Load and return full details of a past run.""" - workspace = _resolve_workspace(directory) + workspace = resolve_workspace(directory) results_dir = workspace / "results" run_dir = (results_dir / run_id).resolve() diff --git a/src/agentops/services/validate.py b/src/agentops/services/validate.py new file mode 100644 index 00000000..bca6c555 --- /dev/null +++ b/src/agentops/services/validate.py @@ -0,0 +1,453 @@ +"""Validation and inspection services for configs and datasets.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, List, Optional + +from agentops.core.config_loader import ( + load_bundle_config, + load_dataset_config, + load_run_config, +) +from agentops.services._workspace import resolve_workspace + + +# --------------------------------------------------------------------------- +# Shared types +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class ValidationIssue: + """A single validation finding.""" + + file: Path + severity: str # "error" | "warning" + message: str + + +@dataclass(frozen=True) +class ValidationResult: + """Result of a validation run.""" + + passed: bool + files_checked: int + issues: List[ValidationIssue] + + +# --------------------------------------------------------------------------- +# Path resolution (mirrors runner.py logic) +# --------------------------------------------------------------------------- + + +def _resolve_path(path_value: Path, base_dir: Path) -> Path: + """Resolve a relative path against a base directory.""" + if path_value.is_absolute(): + return path_value + candidate = (base_dir / path_value).resolve() + if candidate.exists(): + return candidate + fallback = (Path.cwd() / path_value).resolve() + if fallback.exists(): + return fallback + return candidate + + +# --------------------------------------------------------------------------- +# config validate +# --------------------------------------------------------------------------- + + +def validate_config( + config_path: Optional[Path] = None, + directory: Path = Path("."), +) -> ValidationResult: + """Validate the full configuration chain: run.yaml → bundle → dataset → data. + + Returns a ValidationResult with all issues found. + """ + issues: List[ValidationIssue] = [] + files_checked = 0 + + # Resolve run config path + if config_path is None: + workspace = resolve_workspace(directory) + config_path = workspace / "run.yaml" + + config_path = config_path.resolve() + + # --- Validate run.yaml --- + if not config_path.exists(): + issues.append( + ValidationIssue( + file=config_path, + severity="error", + message=f"Run config not found: {config_path}", + ) + ) + return ValidationResult(passed=False, files_checked=0, issues=issues) + + files_checked += 1 + try: + run_config = load_run_config(config_path) + except (ValueError, Exception) as exc: + issues.append( + ValidationIssue(file=config_path, severity="error", message=str(exc)) + ) + return ValidationResult( + passed=False, files_checked=files_checked, issues=issues + ) + + run_config_dir = config_path.parent + + # --- Validate bundle --- + bundle_path = _resolve_path(run_config.bundle.path, run_config_dir) + if not bundle_path.exists(): + issues.append( + ValidationIssue( + file=bundle_path, + severity="error", + message=f"Bundle file not found: {bundle_path}", + ) + ) + else: + files_checked += 1 + try: + bundle_config = load_bundle_config(bundle_path) + + # Semantic checks + enabled = [e for e in bundle_config.evaluators if e.enabled] + if not enabled: + issues.append( + ValidationIssue( + file=bundle_path, + severity="warning", + message="No evaluators are enabled", + ) + ) + + evaluator_names = {e.name for e in bundle_config.evaluators} + for threshold in bundle_config.thresholds: + if threshold.evaluator not in evaluator_names: + issues.append( + ValidationIssue( + file=bundle_path, + severity="warning", + message=( + f"Threshold references unknown evaluator " + f"'{threshold.evaluator}'" + ), + ) + ) + except (ValueError, Exception) as exc: + issues.append( + ValidationIssue(file=bundle_path, severity="error", message=str(exc)) + ) + + # --- Validate dataset --- + dataset_path = _resolve_path(run_config.dataset.path, run_config_dir) + if not dataset_path.exists(): + issues.append( + ValidationIssue( + file=dataset_path, + severity="error", + message=f"Dataset config not found: {dataset_path}", + ) + ) + else: + files_checked += 1 + try: + dataset_config = load_dataset_config(dataset_path) + data_issues, data_files = _validate_dataset_data( + dataset_config, dataset_path.parent + ) + issues.extend(data_issues) + files_checked += data_files + except (ValueError, Exception) as exc: + issues.append( + ValidationIssue(file=dataset_path, severity="error", message=str(exc)) + ) + + passed = not any(i.severity == "error" for i in issues) + return ValidationResult(passed=passed, files_checked=files_checked, issues=issues) + + +# --------------------------------------------------------------------------- +# config show +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class ConfigShowResult: + """Resolved configuration view.""" + + run_config_path: Path + bundle_path: Path + bundle_name: str + dataset_path: Path + dataset_name: str + data_path: Optional[Path] + backend_type: str + target: str + model: Optional[str] + agent_id: Optional[str] + evaluators: List[Dict[str, Any]] + thresholds: int + data_rows: Optional[int] + + +def show_config( + config_path: Optional[Path] = None, + directory: Path = Path("."), +) -> ConfigShowResult: + """Load and resolve all configuration to show the merged view.""" + if config_path is None: + workspace = resolve_workspace(directory) + config_path = workspace / "run.yaml" + + config_path = config_path.resolve() + run_config = load_run_config(config_path) + run_config_dir = config_path.parent + + bundle_path = _resolve_path(run_config.bundle.path, run_config_dir) + dataset_path = _resolve_path(run_config.dataset.path, run_config_dir) + + bundle_config = load_bundle_config(bundle_path) + dataset_config = load_dataset_config(dataset_path) + + # Resolve data file + data_path = _resolve_path(Path(dataset_config.source.path), dataset_path.parent) + data_rows = None + if data_path.exists(): + try: + lines = data_path.read_text(encoding="utf-8").strip().splitlines() + data_rows = len(lines) + except Exception: # noqa: BLE001 + pass + + evaluators = [ + { + "name": e.name, + "source": e.source, + "enabled": e.enabled, + } + for e in bundle_config.evaluators + ] + + return ConfigShowResult( + run_config_path=config_path, + bundle_path=bundle_path, + bundle_name=bundle_config.name, + dataset_path=dataset_path, + dataset_name=dataset_config.name, + data_path=data_path if data_path.exists() else None, + backend_type=run_config.backend.type, + target=run_config.backend.target or "agent", + model=run_config.backend.model, + agent_id=run_config.backend.agent_id, + evaluators=evaluators, + thresholds=len(bundle_config.thresholds), + data_rows=data_rows, + ) + + +# --------------------------------------------------------------------------- +# dataset validate +# --------------------------------------------------------------------------- + + +def validate_dataset( + dataset_path: Path, +) -> ValidationResult: + """Validate a dataset config and its JSONL data file.""" + issues: List[ValidationIssue] = [] + files_checked = 0 + + dataset_path = dataset_path.resolve() + if not dataset_path.exists(): + issues.append( + ValidationIssue( + file=dataset_path, + severity="error", + message=f"Dataset config not found: {dataset_path}", + ) + ) + return ValidationResult(passed=False, files_checked=0, issues=issues) + + files_checked += 1 + try: + dataset_config = load_dataset_config(dataset_path) + except (ValueError, Exception) as exc: + issues.append( + ValidationIssue(file=dataset_path, severity="error", message=str(exc)) + ) + return ValidationResult( + passed=False, files_checked=files_checked, issues=issues + ) + + data_issues, data_files = _validate_dataset_data( + dataset_config, dataset_path.parent + ) + issues.extend(data_issues) + files_checked += data_files + + passed = not any(i.severity == "error" for i in issues) + return ValidationResult(passed=passed, files_checked=files_checked, issues=issues) + + +def _validate_dataset_data( + dataset_config, base_dir: Path +) -> tuple[List[ValidationIssue], int]: + """Validate the JSONL data file referenced by a dataset config.""" + issues: List[ValidationIssue] = [] + files_checked = 0 + + data_path = _resolve_path(Path(dataset_config.source.path), base_dir) + + if not data_path.exists(): + issues.append( + ValidationIssue( + file=data_path, + severity="error", + message=f"Data file not found: {data_path}", + ) + ) + return issues, files_checked + + files_checked += 1 + input_field = dataset_config.format.input_field + expected_field = dataset_config.format.expected_field + + try: + lines = data_path.read_text(encoding="utf-8").strip().splitlines() + except Exception as exc: + issues.append( + ValidationIssue( + file=data_path, + severity="error", + message=f"Cannot read data file: {exc}", + ) + ) + return issues, files_checked + + if not lines: + issues.append( + ValidationIssue( + file=data_path, + severity="warning", + message="Data file is empty (0 rows)", + ) + ) + return issues, files_checked + + for line_num, line in enumerate(lines, start=1): + try: + row = json.loads(line) + except json.JSONDecodeError as exc: + issues.append( + ValidationIssue( + file=data_path, + severity="error", + message=f"Line {line_num}: invalid JSON — {exc}", + ) + ) + continue + + if not isinstance(row, dict): + issues.append( + ValidationIssue( + file=data_path, + severity="error", + message=f"Line {line_num}: expected JSON object, got {type(row).__name__}", + ) + ) + continue + + if input_field not in row: + issues.append( + ValidationIssue( + file=data_path, + severity="error", + message=f"Line {line_num}: missing required field '{input_field}'", + ) + ) + if expected_field not in row: + issues.append( + ValidationIssue( + file=data_path, + severity="error", + message=f"Line {line_num}: missing required field '{expected_field}'", + ) + ) + + return issues, files_checked + + +# --------------------------------------------------------------------------- +# dataset describe +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class DatasetDescription: + """Description of a dataset.""" + + name: str + path: Path + data_path: Optional[Path] + description: str + source_type: str + format_type: str + input_field: str + expected_field: str + context_field: Optional[str] + row_count: int + fields: List[str] + metadata: Dict[str, Any] + + +def describe_dataset(dataset_path: Path) -> DatasetDescription: + """Load a dataset config and describe its contents.""" + dataset_path = dataset_path.resolve() + dataset_config = load_dataset_config(dataset_path) + + data_path = _resolve_path(Path(dataset_config.source.path), dataset_path.parent) + + row_count = 0 + fields: List[str] = [] + + if data_path.exists(): + try: + lines = data_path.read_text(encoding="utf-8").strip().splitlines() + row_count = len(lines) + # Collect unique fields from all rows + all_fields: dict[str, None] = {} + for line in lines: + try: + row = json.loads(line) + if isinstance(row, dict): + for key in row: + all_fields[key] = None + except json.JSONDecodeError: + pass + fields = list(all_fields.keys()) + except Exception: # noqa: BLE001 + pass + + return DatasetDescription( + name=dataset_config.name, + path=dataset_path, + data_path=data_path if data_path.exists() else None, + description=dataset_config.description or "", + source_type=dataset_config.source.type, + format_type=dataset_config.format.type, + input_field=dataset_config.format.input_field, + expected_field=dataset_config.format.expected_field, + context_field=dataset_config.format.context_field, + row_count=row_count, + fields=fields, + metadata=dataset_config.metadata, + ) diff --git a/tests/unit/test_validate.py b/tests/unit/test_validate.py new file mode 100644 index 00000000..5a3a4807 --- /dev/null +++ b/tests/unit/test_validate.py @@ -0,0 +1,278 @@ +"""Tests for validation services (config validate/show, dataset validate/describe).""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +from agentops.cli.app import app +from agentops.services.validate import ( + describe_dataset, + show_config, + validate_config, + validate_dataset, +) +from agentops.utils.yaml import save_yaml + +runner = CliRunner() + + +def _create_workspace(tmp_path: Path) -> Path: + ws = tmp_path / ".agentops" + ws.mkdir() + (ws / "bundles").mkdir() + (ws / "datasets").mkdir() + (ws / "data").mkdir() + return ws + + +def _write_full_workspace(tmp_path: Path, *, bad_jsonl: bool = False) -> Path: + """Create a complete workspace with run.yaml, bundle, dataset, and JSONL.""" + ws = _create_workspace(tmp_path) + + save_yaml( + ws / "bundles" / "test_bundle.yaml", + { + "version": 1, + "name": "test_bundle", + "evaluators": [ + {"name": "CoherenceEvaluator", "source": "foundry", "enabled": True}, + ], + "thresholds": [ + {"evaluator": "CoherenceEvaluator", "criteria": ">=", "value": 3}, + ], + }, + ) + + save_yaml( + ws / "datasets" / "test_dataset.yaml", + { + "version": 1, + "name": "test_dataset", + "source": {"type": "file", "path": "../data/test.jsonl"}, + "format": { + "type": "jsonl", + "input_field": "input", + "expected_field": "expected", + }, + }, + ) + + if bad_jsonl: + (ws / "data" / "test.jsonl").write_text("not valid json\n", encoding="utf-8") + else: + (ws / "data" / "test.jsonl").write_text( + '{"input": "hello", "expected": "world"}\n' + '{"input": "foo", "expected": "bar"}\n', + encoding="utf-8", + ) + + save_yaml( + ws / "run.yaml", + { + "version": 1, + "bundle": {"path": "bundles/test_bundle.yaml"}, + "dataset": {"path": "datasets/test_dataset.yaml"}, + "backend": { + "type": "foundry", + "target": "model", + "model": "gpt-4.1", + }, + "output": {"write_report": True}, + }, + ) + + return ws + + +# --------------------------------------------------------------------------- +# validate_config tests +# --------------------------------------------------------------------------- + + +class TestValidateConfig: + def test_valid_config(self, tmp_path: Path) -> None: + _write_full_workspace(tmp_path) + result = validate_config(directory=tmp_path) + assert result.passed is True + assert result.files_checked >= 3 # run.yaml + bundle + dataset + jsonl + assert len([i for i in result.issues if i.severity == "error"]) == 0 + + def test_missing_run_yaml(self, tmp_path: Path) -> None: + _create_workspace(tmp_path) + result = validate_config(directory=tmp_path) + assert result.passed is False + + def test_missing_bundle(self, tmp_path: Path) -> None: + ws = _write_full_workspace(tmp_path) + (ws / "bundles" / "test_bundle.yaml").unlink() + result = validate_config(directory=tmp_path) + assert result.passed is False + assert any("Bundle file not found" in i.message for i in result.issues) + + def test_missing_dataset(self, tmp_path: Path) -> None: + ws = _write_full_workspace(tmp_path) + (ws / "datasets" / "test_dataset.yaml").unlink() + result = validate_config(directory=tmp_path) + assert result.passed is False + assert any("Dataset config not found" in i.message for i in result.issues) + + def test_bad_jsonl(self, tmp_path: Path) -> None: + _write_full_workspace(tmp_path, bad_jsonl=True) + result = validate_config(directory=tmp_path) + assert result.passed is False + assert any("invalid JSON" in i.message for i in result.issues) + + def test_threshold_unknown_evaluator(self, tmp_path: Path) -> None: + ws = _write_full_workspace(tmp_path) + save_yaml( + ws / "bundles" / "test_bundle.yaml", + { + "version": 1, + "name": "test_bundle", + "evaluators": [ + { + "name": "CoherenceEvaluator", + "source": "foundry", + "enabled": True, + }, + ], + "thresholds": [ + {"evaluator": "NonExistent", "criteria": ">=", "value": 3}, + ], + }, + ) + result = validate_config(directory=tmp_path) + assert any("unknown evaluator" in i.message for i in result.issues) + + +# --------------------------------------------------------------------------- +# show_config tests +# --------------------------------------------------------------------------- + + +class TestShowConfig: + def test_shows_config(self, tmp_path: Path) -> None: + _write_full_workspace(tmp_path) + result = show_config(directory=tmp_path) + assert result.bundle_name == "test_bundle" + assert result.dataset_name == "test_dataset" + assert result.backend_type == "foundry" + assert result.target == "model" + assert result.model == "gpt-4.1" + assert result.data_rows == 2 + assert len(result.evaluators) == 1 + + def test_no_workspace(self, tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError): + show_config(directory=tmp_path) + + +# --------------------------------------------------------------------------- +# validate_dataset tests +# --------------------------------------------------------------------------- + + +class TestValidateDataset: + def test_valid_dataset(self, tmp_path: Path) -> None: + ws = _write_full_workspace(tmp_path) + result = validate_dataset(ws / "datasets" / "test_dataset.yaml") + assert result.passed is True + assert result.files_checked == 2 # dataset.yaml + jsonl + + def test_missing_field(self, tmp_path: Path) -> None: + ws = _write_full_workspace(tmp_path) + (ws / "data" / "test.jsonl").write_text( + '{"input": "hello"}\n', encoding="utf-8" + ) + result = validate_dataset(ws / "datasets" / "test_dataset.yaml") + assert result.passed is False + assert any("missing required field" in i.message for i in result.issues) + + def test_missing_data_file(self, tmp_path: Path) -> None: + ws = _write_full_workspace(tmp_path) + (ws / "data" / "test.jsonl").unlink() + result = validate_dataset(ws / "datasets" / "test_dataset.yaml") + assert result.passed is False + assert any("Data file not found" in i.message for i in result.issues) + + def test_empty_data(self, tmp_path: Path) -> None: + ws = _write_full_workspace(tmp_path) + (ws / "data" / "test.jsonl").write_text("", encoding="utf-8") + result = validate_dataset(ws / "datasets" / "test_dataset.yaml") + assert result.passed is True # warning, not error + assert any("empty" in i.message for i in result.issues) + + +# --------------------------------------------------------------------------- +# describe_dataset tests +# --------------------------------------------------------------------------- + + +class TestDescribeDataset: + def test_describes_dataset(self, tmp_path: Path) -> None: + ws = _write_full_workspace(tmp_path) + desc = describe_dataset(ws / "datasets" / "test_dataset.yaml") + assert desc.name == "test_dataset" + assert desc.row_count == 2 + assert desc.input_field == "input" + assert desc.expected_field == "expected" + assert "input" in desc.fields + assert "expected" in desc.fields + + def test_not_found(self, tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError): + describe_dataset(tmp_path / "nonexistent.yaml") + + +# --------------------------------------------------------------------------- +# CLI tests +# --------------------------------------------------------------------------- + + +class TestConfigValidateCLI: + def test_valid(self, tmp_path: Path) -> None: + _write_full_workspace(tmp_path) + result = runner.invoke(app, ["config", "validate", "--dir", str(tmp_path)]) + assert result.exit_code == 0 + assert "PASSED" in result.stdout + + def test_invalid(self, tmp_path: Path) -> None: + _write_full_workspace(tmp_path, bad_jsonl=True) + result = runner.invoke(app, ["config", "validate", "--dir", str(tmp_path)]) + assert result.exit_code == 1 + assert "FAILED" in result.stdout + + +class TestConfigShowCLI: + def test_shows(self, tmp_path: Path) -> None: + _write_full_workspace(tmp_path) + result = runner.invoke(app, ["config", "show", "--dir", str(tmp_path)]) + assert result.exit_code == 0 + assert "test_bundle" in result.stdout + assert "CoherenceEvaluator" in result.stdout + + +class TestDatasetValidateCLI: + def test_valid(self, tmp_path: Path) -> None: + ws = _write_full_workspace(tmp_path) + result = runner.invoke( + app, + ["dataset", "validate", str(ws / "datasets" / "test_dataset.yaml")], + ) + assert result.exit_code == 0 + assert "PASSED" in result.stdout + + +class TestDatasetDescribeCLI: + def test_describes(self, tmp_path: Path) -> None: + ws = _write_full_workspace(tmp_path) + result = runner.invoke( + app, + ["dataset", "describe", str(ws / "datasets" / "test_dataset.yaml")], + ) + assert result.exit_code == 0 + assert "test_dataset" in result.stdout + assert "Rows: 2" in result.stdout