diff --git a/CHANGELOG.md b/CHANGELOG.md index 128c336d..24f9e16e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval- --- +## [0.126.4] - 2026-07-24 - train-only GEPA optimization + +### Fixed + +- Official GEPA optimization accepts an empty selection set, reuses the non-empty training set for candidate comparison, and labels that comparison as a training-set fallback. + ## [0.126.3] - 2026-07-24 - execution-bound optimizer resume ### Added diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 800dbbb2..98ffcee6 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "agent-eval-rpc" -version = "0.126.3" +version = "0.126.4" description = "Python RPC client, official optimizer bridge, and DSPy metric adapter for @tangle-network/agent-eval." readme = "README.md" requires-python = ">=3.10" diff --git a/clients/python/src/agent_eval_rpc/__init__.py b/clients/python/src/agent_eval_rpc/__init__.py index eeea42c1..81146922 100644 --- a/clients/python/src/agent_eval_rpc/__init__.py +++ b/clients/python/src/agent_eval_rpc/__init__.py @@ -53,7 +53,7 @@ try: __version__ = version("agent-eval-rpc") except PackageNotFoundError: - __version__ = "0.126.3" + __version__ = "0.126.4" __all__ = [ "Client", diff --git a/clients/python/src/agent_eval_rpc/gepa_bridge.py b/clients/python/src/agent_eval_rpc/gepa_bridge.py index 5cc9c7f3..3decdc88 100644 --- a/clients/python/src/agent_eval_rpc/gepa_bridge.py +++ b/clients/python/src/agent_eval_rpc/gepa_bridge.py @@ -121,14 +121,17 @@ def evaluate( raise ValueError("GEPA requested an invalid example") return evaluate_one(candidate, example) - comparison_set = input_value["selectionSet"] or input_value["trainSet"] + selection_set = input_value["selectionSet"] + comparison_set = selection_set or input_value["trainSet"] if not comparison_set: raise ValueError("GEPA selection re-score requires at least one example") rows = [evaluate_one(candidate, item) for item in comparison_set] return ( sum(score for score, _ in rows) / len(rows), { - "comparison": "selection-set", + "comparison": ( + "selection-set" if selection_set else "train-set-fallback" + ), "examples": [ {"id": item["id"], "score": score, "info": info} for item, (score, info) in zip(comparison_set, rows, strict=True) diff --git a/clients/python/src/agent_eval_rpc/gepa_bridge_contract.py b/clients/python/src/agent_eval_rpc/gepa_bridge_contract.py index 13b14b75..a7f9bab4 100644 --- a/clients/python/src/agent_eval_rpc/gepa_bridge_contract.py +++ b/clients/python/src/agent_eval_rpc/gepa_bridge_contract.py @@ -77,8 +77,8 @@ def _validate_input(value: dict[str, Any]) -> None: value.get("selectionSet"), list ): raise ValueError("GEPA bridge input requires trainSet and selectionSet arrays") - if not value["trainSet"] or not value["selectionSet"]: - raise ValueError("GEPA bridge requires non-empty trainSet and selectionSet") + if not value["trainSet"]: + raise ValueError("GEPA bridge requires a non-empty trainSet") validate_json_size(value["objective"], value["maxEvidenceChars"], "GEPA objective") validate_json_size( value.get("background", ""), diff --git a/clients/python/tests/test_gepa_bridge.py b/clients/python/tests/test_gepa_bridge.py index 6fbb3657..011c362b 100644 --- a/clients/python/tests/test_gepa_bridge.py +++ b/clients/python/tests/test_gepa_bridge.py @@ -102,7 +102,7 @@ def test_bridge_calls_gepa_and_writes_a_cost_report( "objective": "Return a better candidate.", "seedCandidate": "baseline", "trainSet": [{"id": "train", "data": {"prompt": "visible"}}], - "selectionSet": [{"id": "selection", "data": {"prompt": "also-visible"}}], + "selectionSet": [], "maxCandidateChars": 100, "maxEvidenceChars": 1_000, "outputDir": str(tmp_path / "external"), @@ -119,9 +119,18 @@ def __init__(self, **kwargs: Any) -> None: def fake_optimize(seed_candidate: str, **kwargs: Any) -> SimpleNamespace: calls["seedCandidate"] = seed_candidate calls["arguments"] = kwargs - score, info = kwargs["evaluator"]("better", kwargs["dataset"][0]) + score, info = kwargs["evaluator"]("better") assert score == 0.75 - assert info == {"source": "agent-eval"} + assert info == { + "comparison": "train-set-fallback", + "examples": [ + { + "id": "train", + "score": 0.75, + "info": {"source": "agent-eval"}, + } + ], + } return SimpleNamespace( best_candidate="better", best_score=score, @@ -156,7 +165,7 @@ def fake_post(*args: Any, **kwargs: Any) -> httpx.Response: assert calls["seedCandidate"] == "baseline" assert calls["arguments"]["dataset"] == [{"id": "train", "data": {"prompt": "visible"}}] - assert calls["arguments"]["valset"] == [{"id": "selection", "data": {"prompt": "also-visible"}}] + assert calls["arguments"]["valset"] == [] config = calls["config"] output_dir = config.pop("output_dir") state_dir = config.pop("run_dir") @@ -506,6 +515,18 @@ def test_input_requires_evaluation_identity(tmp_path: Path) -> None: gepa_bridge._validate_input(input_value) +def test_input_allows_train_only_optimization_but_requires_training_data( + tmp_path: Path, +) -> None: + input_value = _valid_input(tmp_path) + input_value["selectionSet"] = [] + gepa_bridge._validate_input(input_value) + + input_value["trainSet"] = [] + with pytest.raises(ValueError, match="non-empty trainSet"): + gepa_bridge._validate_input(input_value) + + def test_proxied_gepa_accepts_official_retry_configuration(tmp_path: Path) -> None: input_value = _valid_input(tmp_path) input_value["recipe"] = { diff --git a/clients/python/uv.lock b/clients/python/uv.lock index a602c59f..678d5e4b 100644 --- a/clients/python/uv.lock +++ b/clients/python/uv.lock @@ -22,7 +22,7 @@ conflicts = [[ [[package]] name = "agent-eval-rpc" -version = "0.126.3" +version = "0.126.4" source = { editable = "." } dependencies = [ { name = "filelock" }, diff --git a/package.json b/package.json index 18599ae7..c800fba6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-eval", - "version": "0.126.3", + "version": "0.126.4", "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.", "homepage": "https://github.com/tangle-network/agent-eval#readme", "repository": {