From b5606875cd6058985814ac2fc7645bcf85545496 Mon Sep 17 00:00:00 2001 From: itseffi <15998472+itseffi@users.noreply.github.com> Date: Mon, 25 May 2026 13:32:30 +0200 Subject: [PATCH] v2 Co-Authored-By: Claude Opus 4.7 --- README.md | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 01b1918..9b7766f 100644 --- a/README.md +++ b/README.md @@ -10,22 +10,36 @@ task -> agent turns -> actions -> observations -> final state -> reward The goal is to evaluate whether an agent can move from user intent to valid actions, observed state changes, recovery when needed, and a useful final artifact. +
+ ```mermaid -flowchart LR - Task["Task / environment item"] --> Agent["Agent adapter"] - Agent -->|"shell · files · tool calls"| Computer["Computer runtime
local / sandbox / docker
shell · files · browser · tool dispatcher"] - Computer -->|"observations (tool results)"| Agent +flowchart TB + Task["Task or environment item"] --> Agent["Agent adapter"] + Agent -->|"shell · files · tool calls"| Computer["Computer runtime
local / sandbox / docker"] + Computer -->|"observations"| Agent Computer --> Svc["Mock services + workspace state"] Agent -. records .-> Traj[["Trajectory (trace)"]] Computer -. records .-> Traj Svc -. audit .-> Traj - Traj --> Verif["Verifiers
programmatic · state_check · tool_calls · trajectory_check · llm_rubric"] + + subgraph TaskRun["JSON task run — run_task"] + Verif["Verifiers
programmatic · state_check · tool_calls · trajectory_check · llm_rubric"] --> RJ["reward.json + reward-details.json"] + Dims["dimensions.json / task_score
separate summary, not the reward"] + end + + subgraph EnvRun["Environment rollout — evaluate / rollout"] + CR["Environment.compute_reward(item, result, ctx)"] --> RO["rollout.json
reward embedded"] + end + + Traj --> Verif Checks["Post-run checks + final state
commands · files · browser · git diff"] --> Verif - Verif --> Reward["reward.json + dimensions / task_score"] - Reward --> Out["Baselines · RL/preference exports · release gate"] - Traj --> Out + Computer --> CR + RJ --> Out["Baselines · RL / preference exports · release gate"] + RO --> Out ``` +
+ ## Quick Start ```bash