Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions apps/presentation/site/src/lhtb-copy.json
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
["04", "Task evidence", "#scores"]
],
"resultEyebrow": "00 / Core result",
"resultTitle": "Higher mean reward than both baselines; no more strict solves than Plain.",
"resultTitle": "Higher mean reward than Plain and native Goal.",
"resultBody": "LoopX averages 0.4948, Plain 0.4218, and native Goal 0.4475. At reward ≥ 0.95, they solve 7, 7, and 4 tasks respectively. Mean reward captures partial progress; full acceptance remains a separate outcome.",
"summaryColumns": ["Execution", "Mean reward", "Solved ≥ 0.95", "Token usage", "USD"],
"armLabels": {
Expand Down Expand Up @@ -170,7 +170,7 @@
["04", "逐题核对", "#scores"]
],
"resultEyebrow": "00 / 核心结果",
"resultTitle": "均分高于两种基线,完整通过数尚未超过 Plain。",
"resultTitle": "均分高于 Plain 和原生 Goal。",
"resultBody": "LoopX 为 0.4948,Plain 为 0.4218,原生 Goal 为 0.4475。以 Reward ≥ 0.95 计通过,三者分别为 7、7、4 题。均分反映部分进展,完整验收还需单独看。",
"summaryColumns": ["执行方式", "平均 Reward", "通过 ≥ 0.95", "Token 用量", "USD"],
"armLabels": {
Expand Down
18 changes: 12 additions & 6 deletions benchmark/tests/test_publication_scope.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,16 +85,17 @@ def test_published_data_and_bilingual_tables_share_scope(exporters):
def test_lhtb_published_data_and_bilingual_copy_share_scope():
study = STUDY.parent / "LHTB" / "studies" / "five-arm-gpt56sol-max"
data = json.loads((study / "data.json").read_text())
arms = {
all_arms = {
"plain",
"native_goal",
"ssh_goal",
"legacy_heartbeat",
"new_heartbeat",
}
assert set(data["arms"]) == arms
main_comparison_arms = {"plain", "native_goal", "new_heartbeat"}
assert set(data["arms"]) == all_arms
assert len(data["tasks"]) == len({row["task"] for row in data["tasks"]}) == 46
assert all(set(row) == arms | {"task"} for row in data["tasks"])
assert all(set(row) == all_arms | {"task"} for row in data["tasks"])

for arm, summary in data["arms"].items():
rewards = [row[arm] for row in data["tasks"]]
Expand All @@ -105,6 +106,11 @@ def test_lhtb_published_data_and_bilingual_copy_share_scope():
localized_copy = json.loads((site / "lhtb-copy.json").read_text())
assert set(localized_copy) == {"en", "zh"}
for localized in localized_copy.values():
assert set(localized["armLabels"]) == arms
assert set(localized["armKinds"]) == arms
assert {row[0] for row in localized["mechanismRows"]} == arms
assert set(localized["armLabels"]) == all_arms
assert set(localized["armKinds"]) == all_arms
assert {row[0] for row in localized["mechanismRows"]} == main_comparison_arms

assert localized_copy["en"]["resultTitle"] == "Higher mean reward than Plain and native Goal."
assert "7, 7, and 4 tasks" in localized_copy["en"]["resultBody"]
assert localized_copy["zh"]["resultTitle"] == "均分高于 Plain 和原生 Goal。"
assert "7、7、4 题" in localized_copy["zh"]["resultBody"]
Loading