diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 267af1b..bc7d13c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -29,7 +29,7 @@ Branches are short-lived. Delete after merge. If a branch is older than a week, **Description.** Three things, a few lines each: - What changed and why - How you tested it (unit tests, and which tasks you ran) -- For anything that changes agent behavior: score and total tokens before/after on the same task subset. Tokens cost us points, so a change that raises score and doubles tokens needs a word of justification. +- For anything that changes agent behavior: score and total tokens before/after on the same task subset. Tokens only break ties on the leaderboard, but they cost time and most failures are timeouts, so a change that doubles tokens needs a word of justification. **Review.** Any teammate other than the author approves. Rotate so everyone learns every part of the code. Reviewers check logic and intent; don't hand-audit formatting (tooling does that). If you're asked to review, respond within a day. diff --git a/docs/experiments.md b/docs/experiments.md index 8becd64..194ef5d 100644 --- a/docs/experiments.md +++ b/docs/experiments.md @@ -3,6 +3,19 @@ Newest first. One entry per run we want to remember: what changed, what the numbers were, what we learned. Numbers come from `scripts/score.py`. +## 2026-10-07 — Scoring rule changed + +The token penalty is now `min(0.01, 0.01 × tokens per task / 100M)` +([Kaggle evaluation page](https://www.kaggle.com/competitions/OpenAgent-Coding/overview/evaluation)), +replacing `0.01 × total tokens / 1M`. It is capped below one solved task, so +it only breaks ties. `scripts/score.py` uses the new rule. + +Leaderboard rows in the entries below use the old rule. Under the new one our +penalty is about 0.00002 (after the fix: 0.85714 → 0.85712; medium: +0.76190 → 0.76189), so read them as TB score. The full-scale extrapolation in +the reasoning-effort entry no longer applies; the decision to stay on the +default is unchanged, since medium passes fewer tasks. + ## 2026-10-06 — Recovery turn after truncation (plan Phase 2 #10) ✗ - Code: branch `ruoshi/feat-truncation-recovery` (`a4229cb`, not merged). diff --git a/eval/scoring.py b/eval/scoring.py index 4ec61bd..9f923be 100644 --- a/eval/scoring.py +++ b/eval/scoring.py @@ -1,6 +1,12 @@ """Turn a Harbor jobs directory into the three numbers the leaderboard wants. - leaderboard_score = tb_score - 0.01 * (total_tokens / 1_000_000) + tokens_per_task = total_tokens / n_tasks + token_penalty = min(0.01, 0.01 * tokens_per_task / 100_000_000) + leaderboard_score = tb_score - token_penalty + +The penalty is capped just under the value of one solved task (1/89), so +tokens only break ties between agents that pass the same number of tasks. +Rule as of 2026-10-07: kaggle.com/competitions/OpenAgent-Coding/overview/evaluation Harbor writes one directory per trial, `__/result.json`, under `jobs//`. A task run several times has several trial directories; we @@ -22,7 +28,8 @@ from pydantic import BaseModel, Field -TOKEN_PENALTY_PER_MILLION = 0.01 +TOKEN_PENALTY_CAP = 0.01 +TOKENS_PER_TASK_AT_CAP = 100_000_000 SLOW_TURN_SEC = 40.0 SLOW_OUTPUT_TOKENS_PER_SEC = 30.0 @@ -132,6 +139,7 @@ class JobSummary(BaseModel): tasks: list[TaskSummary] tb_score: float total_tokens: float + tokens_per_task: float = 0.0 leaderboard_score: float median_sec_per_turn: float | None = None slow: list[Trial] = Field(default_factory=list) @@ -187,11 +195,15 @@ def summarize(trials: list[Trial], drop_infra: bool = False) -> JobSummary: tb_score = sum(t.reward for t in tasks) / len(tasks) if tasks else 0.0 total_tokens = sum(t.tokens for t in tasks) - penalty = TOKEN_PENALTY_PER_MILLION * (total_tokens / 1_000_000) + # Divide by the tasks we ran, not 89, so a dev slice gets the same + # per-task number a full run would. + tokens_per_task = total_tokens / len(tasks) if tasks else 0.0 + penalty = TOKEN_PENALTY_CAP * min(1.0, tokens_per_task / TOKENS_PER_TASK_AT_CAP) return JobSummary( tasks=tasks, tb_score=tb_score, total_tokens=total_tokens, + tokens_per_task=tokens_per_task, leaderboard_score=tb_score - penalty, median_sec_per_turn=_median( [t.seconds_per_turn for t in trials if t.seconds_per_turn is not None] @@ -217,9 +229,10 @@ def format_table(summary: JobSummary) -> str: ) lines += [ "-" * width, - f"TB score: {summary.tb_score:.4f}", + f"TB score: {summary.tb_score:.5f}", f"Total tokens: {summary.total_tokens:,.0f}", - f"Leaderboard score: {summary.leaderboard_score:.4f}", + f"Tokens per task: {summary.tokens_per_task:,.0f}", + f"Leaderboard score: {summary.leaderboard_score:.5f}", "", ] lines += _health_lines(summary) diff --git a/plan.md b/plan.md index ba90a26..dea65b8 100644 --- a/plan.md +++ b/plan.md @@ -1,7 +1,8 @@ # Plan -Leaderboard score = `TB score − 0.01 × (total tokens / 1M)`. Tokens cost -points, so every result reports both. Numbers and what we learned from each +Leaderboard score = `TB score − min(0.01, 0.01 × tokens per task / 100M)`. +The penalty is capped below one solved task (1/89), so tokens only break ties: +passing more tasks comes first. Every result still reports both. Numbers and what we learned from each run live in [docs/experiments.md](docs/experiments.md). ## How we test a change diff --git a/tests/test_scoring.py b/tests/test_scoring.py index b950e85..3cf9bdd 100644 --- a/tests/test_scoring.py +++ b/tests/test_scoring.py @@ -4,7 +4,7 @@ import pytest -from eval.scoring import load_trials, summarize +from eval.scoring import TOKEN_PENALTY_CAP, Trial, load_trials, summarize FIXTURE = Path(__file__).parent / "fixtures" / "job" @@ -37,14 +37,21 @@ def test_tb_score_is_mean_over_tasks_not_trials(summary): assert summary.tb_score == pytest.approx(0.5) -def test_leaderboard_score_subtracts_the_token_penalty(summary): +def test_leaderboard_score_subtracts_the_per_task_token_penalty(summary): expected_tokens = 736_295.5 + 128_000 assert summary.total_tokens == pytest.approx(expected_tokens) + assert summary.tokens_per_task == pytest.approx(expected_tokens / 2) assert summary.leaderboard_score == pytest.approx( - 0.5 - 0.01 * expected_tokens / 1_000_000 + 0.5 - 0.01 * (expected_tokens / 2) / 100_000_000 ) +def test_token_penalty_is_capped_below_one_solved_task(): + huge = Trial(task="a", reward=1.0, input_tokens=5_000_000_000) + assert summarize([huge]).leaderboard_score == pytest.approx(1.0 - 0.01) + assert TOKEN_PENALTY_CAP < 1 / 89 + + def test_empty_job_dir_is_not_an_error(tmp_path): empty = summarize(load_trials(tmp_path)) assert empty.tasks == []