diff --git a/agenticevals/export.py b/agenticevals/export.py index 8acf5a1..bc2e216 100644 --- a/agenticevals/export.py +++ b/agenticevals/export.py @@ -2,6 +2,7 @@ import gzip import json +import math from pathlib import Path from typing import Iterable @@ -179,14 +180,21 @@ def _reward_rows(root: Path, rollouts: list[dict]) -> list[dict]: if dimensions_path.exists(): dimensions = json.loads(dimensions_path.read_text(encoding="utf-8")) for name in ["completion", "robustness", "communication", "safety"]: + value = float(dimensions.get(name, 0.0) or 0.0) + # Graduated dimensions (e.g. communication, partial completion) + # carry their fractional score for downstream RL/preference use; + # `partial` flags credit between 0 and 1 that a binary `passed` + # would otherwise discard. rows.append( { "run_id": rollout.get("run_id"), "environment": rollout.get("environment"), "item_id": rollout.get("item_id"), "component": name, - "passed": dimensions.get(name, 0.0) >= 1.0, - "value": dimensions.get(name, 0.0), + "passed": math.isclose(value, 1.0) or value > 1.0, + "partial": 0.0 < value < 1.0, + "score": value, + "value": value, "max_value": 1.0, "detail": dimensions.get("details", {}).get(name), "source_rollout_path": rollout.get("source_rollout_path"), diff --git a/tests/test_export.py b/tests/test_export.py new file mode 100644 index 0000000..8028a8f --- /dev/null +++ b/tests/test_export.py @@ -0,0 +1,40 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from agenticevals.export import _reward_rows + + +class RewardRowDimensionTests(unittest.TestCase): + def _rows(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + run = root / "r" + run.mkdir() + (run / "dimensions.json").write_text( + json.dumps( + {"completion": 1.0, "robustness": 1.0, "communication": 0.5, "safety": 1.0, "details": {}} + ), + encoding="utf-8", + ) + rollout = {"source_rollout_path": str(run / "rollout.json"), "reward": {}} + return _reward_rows(root, [rollout]) + + def test_partial_dimension_keeps_its_fractional_score(self): + rows = self._rows() + comm = next(r for r in rows if r["component"] == "communication") + self.assertEqual(comm["value"], 0.5) + self.assertEqual(comm["score"], 0.5) + self.assertFalse(comm["passed"]) + self.assertTrue(comm["partial"]) + + def test_fully_met_dimension_is_passed_not_partial(self): + rows = self._rows() + completion = next(r for r in rows if r["component"] == "completion") + self.assertTrue(completion["passed"]) + self.assertFalse(completion["partial"]) + + +if __name__ == "__main__": + unittest.main()