v2

agenticevals/export.py

@@ -2,6 +2,7 @@
 
 import gzip
 import json
+import math
 from pathlib import Path
 from typing import Iterable
 
@@ -179,14 +180,21 @@ def _reward_rows(root: Path, rollouts: list[dict]) -> list[dict]:
         if dimensions_path.exists():
             dimensions = json.loads(dimensions_path.read_text(encoding="utf-8"))
             for name in ["completion", "robustness", "communication", "safety"]:
+                value = float(dimensions.get(name, 0.0) or 0.0)
+                # Graduated dimensions (e.g. communication, partial completion)
+                # carry their fractional score for downstream RL/preference use;
+                # `partial` flags credit between 0 and 1 that a binary `passed`
+                # would otherwise discard.
                 rows.append(
                     {
                         "run_id": rollout.get("run_id"),
                         "environment": rollout.get("environment"),
                         "item_id": rollout.get("item_id"),
                         "component": name,
-                        "passed": dimensions.get(name, 0.0) >= 1.0,
-                        "value": dimensions.get(name, 0.0),
+                        "passed": math.isclose(value, 1.0) or value > 1.0,
+                        "partial": 0.0 < value < 1.0,
+                        "score": value,
+                        "value": value,
                         "max_value": 1.0,
                         "detail": dimensions.get("details", {}).get(name),
                         "source_rollout_path": rollout.get("source_rollout_path"),

tests/test_export.py

@@ -0,0 +1,40 @@
+import json
+import tempfile
+import unittest
+from pathlib import Path
+
+from agenticevals.export import _reward_rows
+
+
+class RewardRowDimensionTests(unittest.TestCase):
+    def _rows(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            run = root / "r"
+            run.mkdir()
+            (run / "dimensions.json").write_text(
+                json.dumps(
+                    {"completion": 1.0, "robustness": 1.0, "communication": 0.5, "safety": 1.0, "details": {}}
+                ),
+                encoding="utf-8",
+            )
+            rollout = {"source_rollout_path": str(run / "rollout.json"), "reward": {}}
+            return _reward_rows(root, [rollout])
+
+    def test_partial_dimension_keeps_its_fractional_score(self):
+        rows = self._rows()
+        comm = next(r for r in rows if r["component"] == "communication")
+        self.assertEqual(comm["value"], 0.5)
+        self.assertEqual(comm["score"], 0.5)
+        self.assertFalse(comm["passed"])
+        self.assertTrue(comm["partial"])
+
+    def test_fully_met_dimension_is_passed_not_partial(self):
+        rows = self._rows()
+        completion = next(r for r in rows if r["component"] == "completion")
+        self.assertTrue(completion["passed"])
+        self.assertFalse(completion["partial"])
+
+
+if __name__ == "__main__":
+    unittest.main()