diff --git a/CHANGES b/CHANGES index a6c0799..6bd2cf5 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,8 @@ # Changelog +## v1.5.0 — 2026-08-21 +- Add built-in `cursor` agent preset (`agent -p --force --trust`) using Cursor Auto when `--model` is omitted; document comparing Auto to other agents (claude/codex/etc.), optional `cursor-composer` example, and `CURSOR_API_KEY` auth (README, `config.example.yaml`, Docker Compose) + ## v1.4.5 — 2026-05-22 - Fix: commit missing `token_accum` parameter in `run_agent_loop` that caused a TypeError when tool-use models ran queries diff --git a/README.md b/README.md index 22a9774..6c8ac37 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,44 @@ uv pip install -e ".[dev]" ## Configuration -Copy `config.example.yaml` to `agent-tester.yaml` (or `agent-tester.yml`) in your target repo to customize agents. Built-in presets are available for `claude`, `aider`, and `codex`. +Copy `config.example.yaml` to `agent-tester.yaml` (or `agent-tester.yml`) in your target repo to customize agents. Built-in presets are available for `claude`, `aider`, `codex`, and `cursor`. + +### Comparing Cursor Auto to another agent + +The `cursor` preset runs the [Cursor CLI](https://cursor.com/docs/cli/overview) in print mode without `--model`, which uses **Auto** (the model router). Run it alongside any other agent (a static Claude/Codex/Aider setup, or another Cursor invocation with `--model`) in the same pass: + +```bash +# Cursor Auto vs Claude +agent-tester run "Refactor the auth module" --agents cursor,claude + +# Cursor Auto vs Codex +agent-tester run "Refactor the auth module" --agents cursor,codex +``` + +You can also pin a Cursor model (or any other CLI agent) as a separate config entry — useful when you want two Cursor variants, or to show that any agent tool works the same way: + +```yaml +agents: + cursor: + command: "agent -p --force --trust {prompt}" + commit_style: auto + timeout: 600 + # Prefer exporting CURSOR_API_KEY in the shell / Docker env. + # Or set it here (do not commit secrets): + # env: + # CURSOR_API_KEY: "..." + + cursor-composer: + command: "agent -p --force --trust --model composer-2.5 {prompt}" + commit_style: auto + timeout: 600 +``` + +```bash +agent-tester run "Refactor the auth module" --agents cursor,cursor-composer +``` + +Do not pass Cursor's `-w`/`--worktree` flag — AgentTester already isolates each agent in its own git worktree. See [Cursor CLI authentication](#cursor-cli) for `CURSOR_API_KEY` / `agent login`. ### Config file discovery @@ -108,6 +145,29 @@ your-repo/.agent-tester/skills/style.md # adds a new skill for this project ## Authentication +### Cursor CLI + +The `cursor` shell agent uses the Cursor CLI (`agent`), not the `providers:` block. Authenticate in one of these ways: + +1. **Interactive:** run `agent login` once on the host (stores credentials for later runs). +2. **API key:** export `CURSOR_API_KEY` in the environment (recommended for CI / Docker / headless). AgentTester forwards the process environment to each agent; `docker-compose.yaml` also passes `CURSOR_API_KEY` through from the host. +3. **Per-agent config:** set `env.CURSOR_API_KEY` on an agent entry in YAML (useful for remote hosts). Prefer the environment over committing secrets to config. + +```bash +export CURSOR_API_KEY=your_api_key_here +agent-tester run "…" --agents cursor,claude +``` + +```yaml +agents: + cursor: + command: "agent -p --force --trust {prompt}" + env: + CURSOR_API_KEY: "your_api_key_here" # prefer env / Docker; do not commit +``` + +### Providers (evaluators and REPL models) + Define a `providers` block to share credentials across evaluators and REPL model agents. Each provider type reads credentials from a standard environment variable automatically — no `api_key_env` required unless you want to override the default. | `type` | Description | Default env var | Install | @@ -337,7 +397,7 @@ The command walks through two phases — select sessions to delete entirely, the ## Docker -Provider API keys are forwarded automatically from the host environment — set any of `ANTHROPIC_API_KEY`, `AZURE_OPENAI_API_KEY`, `GOOGLE_API_KEY`, `BEDROCK_API_KEY`, or the standard `AWS_*` variables before running. +Provider and agent API keys are forwarded automatically from the host environment — set any of `CURSOR_API_KEY`, `ANTHROPIC_API_KEY`, `AZURE_OPENAI_API_KEY`, `GOOGLE_API_KEY`, `BEDROCK_API_KEY`, or the standard `AWS_*` variables before running. ```bash # Open REPL against the current directory diff --git a/config.example.yaml b/config.example.yaml index 6945e46..91afcf7 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -1,6 +1,6 @@ # AgentTester configuration # Copy to agent-tester.yaml in your target repo. -# Entries here override built-in presets (claude, aider, codex). +# Entries here override built-in presets (claude, aider, codex, cursor). # # host: "localhost" (default) runs locally; set to "user@server" for SSH. @@ -23,6 +23,33 @@ agents: commit_style: auto timeout: 600 + # Cursor CLI (install: https://cursor.com/docs/cli/overview). + # Omitting --model uses Auto (the model router). Compare against any + # other agent, e.g.: + # agent-tester run "…" --agents cursor,claude + # Auth: export CURSOR_API_KEY, or `agent login`, or set env below. + # Do not pass Cursor's -w/--worktree — AgentTester already isolates + # each agent in its own worktree. + cursor: + command: "agent -p --force --trust {prompt}" + host: localhost + commit_style: auto + timeout: 600 + # env: + # CURSOR_API_KEY: "..." # prefer exporting in the shell / Docker + + # Optional: pin a Cursor model (same CLI, static --model). Shows that + # any agent tool can be added the same way — not required for Auto vs + # Claude/Codex comparisons. + # agent-tester run "…" --agents cursor,cursor-composer + # cursor-composer: + # command: "agent -p --force --trust --model composer-2.5 {prompt}" + # host: localhost + # commit_style: auto + # timeout: 600 + # env: + # CURSOR_API_KEY: "..." + # Example: agent running on a remote GPU box # remote-claude: # command: 'claude -p {prompt} --allowedTools "Bash,Read,Edit"' diff --git a/docker-compose.yaml b/docker-compose.yaml index 230f77b..a560df0 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -10,6 +10,7 @@ services: environment: - SSH_AUTH_SOCK=/ssh-agent # Pass API keys through to agents + - CURSOR_API_KEY - ANTHROPIC_API_KEY - OPENAI_API_KEY # Azure AI Foundry / Azure OpenAI diff --git a/pyproject.toml b/pyproject.toml index b9f9e7f..2dc43a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "agenttester" -version = "1.4.5" +version = "1.5.0" description = "Run a prompt against multiple coding agents in parallel and compare results" readme = "README.md" requires-python = ">=3.10" diff --git a/src/agenttester/cli.py b/src/agenttester/cli.py index ab356f5..a1be100 100644 --- a/src/agenttester/cli.py +++ b/src/agenttester/cli.py @@ -457,7 +457,7 @@ def list_agents( all_agents = load_config(config) console.print("[bold]Available agents:[/bold]\n") for name, agent in sorted(all_agents.items()): - preset_names = ("claude", "aider", "codex") + preset_names = ("claude", "aider", "codex", "cursor") tag = "[dim](preset)[/dim]" if name in preset_names else "" console.print(f" [bold]{name}[/bold] {tag}") console.print(f" command: [dim]{agent.command}[/dim]") diff --git a/src/agenttester/presets.py b/src/agenttester/presets.py index aabc774..89a79c0 100644 --- a/src/agenttester/presets.py +++ b/src/agenttester/presets.py @@ -22,4 +22,14 @@ "commit_style": "auto", "timeout": 600, }, + # Uses Cursor Auto (model router) when --model is omitted. + # Compare against any other agent (claude, codex, …) or a second + # Cursor entry with --model (see `agent models`). Auth via + # CURSOR_API_KEY or `agent login`. Do not pass Cursor's -w/--worktree — + # AgentTester already isolates worktrees. + "cursor": { + "command": "agent -p --force --trust {prompt}", + "commit_style": "auto", + "timeout": 600, + }, } diff --git a/tests/test_cli.py b/tests/test_cli.py index 08c3a6b..42f1d1e 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -45,6 +45,7 @@ def test_lists_presets(self) -> None: assert "claude" in result.output assert "aider" in result.output assert "codex" in result.output + assert "cursor" in result.output class TestRunValidation: diff --git a/tests/test_config.py b/tests/test_config.py index 9db1cf2..b8d8865 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -47,6 +47,7 @@ def test_includes_builtin_presets(self) -> None: assert "claude" in agents assert "aider" in agents assert "codex" in agents + assert "cursor" in agents def test_preset_types(self) -> None: agents = load_config() @@ -61,6 +62,14 @@ def test_aider_preset_has_manual_commit(self) -> None: agents = load_config() assert agents["aider"].commit_style == "manual" + def test_cursor_preset_uses_auto_router(self) -> None: + agents = load_config() + assert agents["cursor"].commit_style == "auto" + assert "--model" not in agents["cursor"].command + assert "agent -p" in agents["cursor"].command + assert "--force" in agents["cursor"].command + assert "--trust" in agents["cursor"].command + def test_presets_default_to_localhost(self) -> None: agents = load_config() for agent in agents.values(): @@ -125,11 +134,11 @@ def test_yaml_with_env(self, tmp_path: Path) -> None: def test_missing_config_file_returns_presets(self) -> None: agents = load_config(Path("/nonexistent/config.yaml")) assert "claude" in agents - assert len(agents) == 3 # only presets + assert len(agents) == 4 # only presets def test_none_config_returns_presets(self) -> None: agents = load_config(None) - assert len(agents) >= 3 + assert len(agents) >= 4 def test_loads_agent_tester_filename(self, tmp_path: Path) -> None: config_file = tmp_path / "agent-tester.yaml"