From bd2fc71e5834260b84bb5cdb83b2239e41f515a9 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 21 Aug 2026 03:52:02 +0000 Subject: [PATCH] Add Cursor Auto provider for the REPL Introduce type: cursor so models: can use Cursor CLI Auto/Router as a peer REPL backend. Cursor owns its agent loop and tools; AgentTester streams CLI output, resumes chat across turns, and points --workspace at each model's clone when --workdir is set. Co-authored-by: Steven Roomberg --- CHANGES | 3 + README.md | 49 +++- config.example.yaml | 13 ++ pyproject.toml | 2 +- src/agenttester/config.py | 7 + src/agenttester/providers/__init__.py | 2 + src/agenttester/providers/cursor.py | 322 ++++++++++++++++++++++++++ src/agenttester/repl.py | 35 +++ tests/test_config.py | 21 ++ tests/test_cursor_provider.py | 155 +++++++++++++ tests/test_repl.py | 50 ++++ 11 files changed, 657 insertions(+), 2 deletions(-) create mode 100644 src/agenttester/providers/cursor.py create mode 100644 tests/test_cursor_provider.py diff --git a/CHANGES b/CHANGES index 6bd2cf5..3abb8a1 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,8 @@ # Changelog +## v1.6.0 — 2026-08-21 +- Add `type: cursor` REPL provider (Cursor CLI) so Auto / Router works in `agent-tester repl` alongside other models; `/reset` clears the CLI chat session; document `models.cursor-auto` config + ## v1.5.0 — 2026-08-21 - Add built-in `cursor` agent preset (`agent -p --force --trust`) using Cursor Auto when `--model` is omitted; document comparing Auto to other agents (claude/codex/etc.), optional `cursor-composer` example, and `CURSOR_API_KEY` auth (README, `config.example.yaml`, Docker Compose) diff --git a/README.md b/README.md index 6c8ac37..091a0b7 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,13 @@ agent-tester run "Refactor the auth module" --agents cursor,claude agent-tester run "Refactor the auth module" --agents cursor,codex ``` +For interactive multi-model sessions, configure the same Auto backend under `models:` with `type: cursor` (see [Cursor CLI authentication](#cursor-cli)): + +```bash +agent-tester repl --workdir . +# then compare @cursor-auto against other configured models +``` + You can also pin a Cursor model (or any other CLI agent) as a separate config entry — useful when you want two Cursor variants, or to show that any agent tool works the same way: ```yaml @@ -147,7 +154,7 @@ your-repo/.agent-tester/skills/style.md # adds a new skill for this project ### Cursor CLI -The `cursor` shell agent uses the Cursor CLI (`agent`), not the `providers:` block. Authenticate in one of these ways: +The `cursor` shell agent (`agent-tester run`) and the `type: cursor` REPL provider both use the Cursor CLI (`agent`). Authenticate in one of these ways: 1. **Interactive:** run `agent login` once on the host (stores credentials for later runs). 2. **API key:** export `CURSOR_API_KEY` in the environment (recommended for CI / Docker / headless). AgentTester forwards the process environment to each agent; `docker-compose.yaml` also passes `CURSOR_API_KEY` through from the host. @@ -156,6 +163,7 @@ The `cursor` shell agent uses the Cursor CLI (`agent`), not the `providers:` blo ```bash export CURSOR_API_KEY=your_api_key_here agent-tester run "…" --agents cursor,claude +agent-tester repl --workdir . ``` ```yaml @@ -164,8 +172,24 @@ agents: command: "agent -p --force --trust {prompt}" env: CURSOR_API_KEY: "your_api_key_here" # prefer env / Docker; do not commit + +providers: + cursor: + type: cursor + # api_key_env: CURSOR_API_KEY # default + # binary: agent # or cursor-agent + +models: + cursor-auto: + provider: cursor + model: auto # Cursor Auto / Router (omit --model) + # cursor-composer: + # provider: cursor + # model: composer-2.5 ``` +The REPL provider runs Cursor as a full agent (its own tools). With `--workdir`, each model still gets an isolated clone; Cursor is pointed at that clone via `--workspace`. `/reset` clears REPL history and the Cursor CLI chat session. + ### Providers (evaluators and REPL models) Define a `providers` block to share credentials across evaluators and REPL model agents. Each provider type reads credentials from a standard environment variable automatically — no `api_key_env` required unless you want to override the default. @@ -177,6 +201,7 @@ Define a `providers` block to share credentials across evaluators and REPL model | `bedrock` | AWS Bedrock Converse API | `BEDROCK_API_KEY` (api_key mode) | built-in; `pip install agenttester[aws]` for boto3 modes | | `azure` | Azure AI Foundry / Azure OpenAI Service | `AZURE_OPENAI_API_KEY` | built-in | | `vertex` | GCP Vertex AI (OpenAI-compatible endpoint) | `GOOGLE_API_KEY` | built-in | +| `cursor` | Cursor CLI agent (Auto / Router or a pinned model) | `CURSOR_API_KEY` | built-in; requires [Cursor CLI](https://cursor.com/docs/cli/overview) on PATH | Override the default for any provider or evaluator with `api_key_env: MY_CUSTOM_VAR`. @@ -257,6 +282,23 @@ providers: CLI tokens (Azure and GCP) are cached for 55 minutes to avoid extra subprocesses on every request. +### Cursor (REPL) + +```yaml +providers: + cursor: + type: cursor + # api_key_env: CURSOR_API_KEY + # binary: agent + +models: + cursor-auto: + provider: cursor + model: auto +``` + +`model: auto` (or `default` / `auto-smart`) uses Cursor Auto. Pin any other id from `agent models` the same way. See [Cursor CLI authentication](#cursor-cli). + ### Evaluators and REPL models Providers are referenced by name in `evaluators:` (for diff review) and `models:` (for the REPL): @@ -265,6 +307,8 @@ Providers are referenced by name in `evaluators:` (for diff review) and `models: providers: anthropic: type: anthropic + cursor: + type: cursor my-azure: type: azure endpoint: https://my-resource.openai.azure.com @@ -287,6 +331,9 @@ evaluation: max_aggregate_tokens: 2000 # aggregate is summarized before injection if too long models: + cursor-auto: + provider: cursor + model: auto claude-bedrock: provider: bedrock-sso model: anthropic.claude-3-5-sonnet-20241022-v2:0 diff --git a/config.example.yaml b/config.example.yaml index 91afcf7..82e1679 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -121,6 +121,19 @@ agents: # aws_access_key_id_env: MY_KEY_ID # explicit credential env vars # aws_secret_access_key_env: MY_SECRET # aws_session_token_env: MY_TOKEN # optional +# +# cursor: +# type: cursor +# # api_key_env: CURSOR_API_KEY # default; or use `agent login` +# # binary: agent # or cursor-agent +# +# models: +# cursor-auto: +# provider: cursor +# model: auto # Cursor Auto / Router +# # cursor-composer: +# # provider: cursor +# # model: composer-2.5 # --- LLM-based code quality evaluation --- # After each iteration, each evaluator independently reviews every agent's diff diff --git a/pyproject.toml b/pyproject.toml index 2dc43a3..1c3a493 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "agenttester" -version = "1.5.0" +version = "1.6.0" description = "Run a prompt against multiple coding agents in parallel and compare results" readme = "README.md" requires-python = ">=3.10" diff --git a/src/agenttester/config.py b/src/agenttester/config.py index 0f34010..1b9b975 100644 --- a/src/agenttester/config.py +++ b/src/agenttester/config.py @@ -14,6 +14,7 @@ AnthropicProvider, AzureProvider, BedrockProvider, + CursorProvider, OpenAICompatProvider, Provider, VertexProvider, @@ -245,6 +246,12 @@ def _build_named_provider(name: str, data: dict) -> Provider: api_key_env=data.get("api_key_env"), auth_method=data.get("auth_method", "api_key"), ) + if ptype == "cursor": + return CursorProvider( + api_key_env=data.get("api_key_env", "CURSOR_API_KEY"), + binary=data.get("binary", "agent"), + optimize_for=data.get("optimize_for"), + ) raise ValueError(f"Unknown provider type {ptype!r} for provider '{name}'") diff --git a/src/agenttester/providers/__init__.py b/src/agenttester/providers/__init__.py index 82a0e68..2b6a615 100644 --- a/src/agenttester/providers/__init__.py +++ b/src/agenttester/providers/__init__.py @@ -6,6 +6,7 @@ from .aws import BedrockProvider from .azure import AzureProvider from .base import Provider +from .cursor import CursorProvider from .gcp import VertexProvider from .openai_compat import OpenAICompatProvider @@ -13,6 +14,7 @@ "AnthropicProvider", "AzureProvider", "BedrockProvider", + "CursorProvider", "OpenAICompatProvider", "Provider", "VertexProvider", diff --git a/src/agenttester/providers/cursor.py b/src/agenttester/providers/cursor.py new file mode 100644 index 0000000..f29c922 --- /dev/null +++ b/src/agenttester/providers/cursor.py @@ -0,0 +1,322 @@ +"""Cursor CLI provider — Auto / Cursor Router for the REPL.""" + +from __future__ import annotations + +import asyncio +import json +import os +import shutil +from collections.abc import Callable +from typing import Any + +from .base import Provider + +# Models that mean "use Cursor Auto / Router" (omit --model or pass auto). +_AUTO_MODEL_IDS = frozenset({"auto", "default", "auto-smart", ""}) + + +class CursorProvider(Provider): + """Runs the Cursor CLI (``agent``) as a REPL model backend. + + Cursor owns its own agent loop and tools. AgentTester does not inject + OpenAI-style tool schemas; it streams the CLI's response and optionally + resumes the same CLI chat across turns. + + Config example:: + + providers: + cursor: + type: cursor + # api_key_env: CURSOR_API_KEY # default + # binary: agent # or cursor-agent + # optimize_for: balanced # cost | balanced | intelligence + + models: + cursor-auto: + provider: cursor + model: auto # Auto / Cursor Router + """ + + def __init__( + self, + api_key_env: str = "CURSOR_API_KEY", + binary: str = "agent", + optimize_for: str | None = None, + ) -> None: + self.api_key_env = api_key_env + self.binary = binary + self.optimize_for = optimize_for + self.workspace: str | None = None + self.session_id: str | None = None + + def reset_session(self) -> None: + """Drop CLI chat continuity (e.g. on ``/reset``).""" + self.session_id = None + + def _resolve_binary(self) -> str: + path = shutil.which(self.binary) + if path: + return path + # Common alternate install name + if self.binary == "agent": + alt = shutil.which("cursor-agent") + if alt: + return alt + raise RuntimeError( + f"Cursor CLI binary {self.binary!r} not found on PATH. " + "Install from https://cursor.com/docs/cli/overview" + ) + + def _env(self) -> dict[str, str]: + env = os.environ.copy() + if self.api_key_env and self.api_key_env not in env: + # Leave unset so `agent login` credentials still work + pass + return env + + def _prompt_from_messages(self, messages: list[dict], *, resume: bool) -> str: + """Build the CLI prompt from REPL message history. + + When resuming a CLI session, only the latest user turn is sent (Cursor + already has prior context). Otherwise system/skills + user turns are + concatenated so the first call has full context. + """ + if resume: + for msg in reversed(messages): + if msg.get("role") == "user": + content = msg.get("content") or "" + return content if isinstance(content, str) else str(content) + return "" + + parts: list[str] = [] + for msg in messages: + role = msg.get("role") + content = msg.get("content") or "" + if not isinstance(content, str): + content = str(content) + if not content.strip(): + continue + if role in ("system", "user"): + parts.append(content) + elif role == "assistant": + parts.append(f"[previous assistant]\n{content}") + return "\n\n".join(parts) + + def _build_command( + self, + model: str, + prompt: str, + *, + mode: str, + stream: bool, + resume: bool, + ) -> list[str]: + cmd = [self._resolve_binary(), "-p", "--force", "--trust"] + if mode == "ask": + cmd.append("--mode=ask") + if stream: + cmd.extend(["--output-format", "stream-json", "--stream-partial-output"]) + else: + cmd.extend(["--output-format", "json"]) + if resume and self.session_id: + cmd.extend(["--resume", self.session_id]) + if self.workspace: + cmd.extend(["--workspace", self.workspace]) + + model_id = (model or "auto").strip() + # auto-smart is the SDK Router id; CLI accepts Auto via omit / "auto". + if model_id == "auto-smart": + model_id = "auto" + if model_id not in _AUTO_MODEL_IDS: + cmd.extend(["--model", model_id]) + # optimize_for is SDK-only today; keep for future CLI support / docs. + _ = self.optimize_for + + api_key = os.environ.get(self.api_key_env, "") if self.api_key_env else "" + if api_key: + cmd.extend(["--api-key", api_key]) + + cmd.append(prompt) + return cmd + + @staticmethod + def _extract_text_from_event(event: dict[str, Any]) -> str: + if event.get("type") != "assistant": + return "" + message = event.get("message") or {} + content = message.get("content") + if isinstance(content, str): + return content + if isinstance(content, list): + parts: list[str] = [] + for block in content: + if isinstance(block, dict) and block.get("type") == "text": + parts.append(block.get("text") or "") + elif isinstance(block, str): + parts.append(block) + return "".join(parts) + return "" + + @staticmethod + def _usage_from_payload(data: dict[str, Any]) -> tuple[int, int]: + usage = data.get("usage") or {} + if not isinstance(usage, dict): + return 0, 0 + # Prefer explicit fields; total input ≈ uncached + cache read + cache write + input_tokens = int(usage.get("inputTokens") or usage.get("input_tokens") or 0) + cache_read = int( + usage.get("cacheReadTokens") or usage.get("cache_read_tokens") or 0 + ) + cache_write = int( + usage.get("cacheWriteTokens") or usage.get("cache_write_tokens") or 0 + ) + output_tokens = int( + usage.get("outputTokens") or usage.get("output_tokens") or 0 + ) + return input_tokens + cache_read + cache_write, output_tokens + + async def _run_cli( + self, + model: str, + messages: list[dict], + *, + mode: str, + stream: bool, + on_chunk: Callable[[str], None] | None = None, + update_session: bool = True, + ) -> dict: + resume = bool(update_session and self.session_id) + prompt = self._prompt_from_messages(messages, resume=resume) + if not prompt.strip(): + return { + "content": "", + "tool_calls": None, + "input_tokens": 0, + "output_tokens": 0, + } + + cmd = self._build_command( + model, prompt, mode=mode, stream=stream, resume=resume + ) + proc = await asyncio.create_subprocess_exec( + *cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env=self._env(), + cwd=self.workspace or None, + ) + assert proc.stdout is not None + assert proc.stderr is not None + + text_parts: list[str] = [] + input_tokens = 0 + output_tokens = 0 + result_text = "" + seen_partial = False + stderr = "" + + if stream: + while True: + raw = await proc.stdout.readline() + if not raw: + break + line = raw.decode("utf-8", errors="replace").strip() + if not line: + continue + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + etype = event.get("type") + if etype == "assistant": + # With --stream-partial-output: deltas have timestamp_ms and + # no model_call_id; skip duplicate flushes (docs). + has_ts = "timestamp_ms" in event + has_mc = "model_call_id" in event + if has_ts and not has_mc: + chunk = self._extract_text_from_event(event) + if chunk: + seen_partial = True + text_parts.append(chunk) + if on_chunk: + on_chunk(chunk) + elif not has_ts and not has_mc and not seen_partial: + chunk = self._extract_text_from_event(event) + if chunk: + text_parts.append(chunk) + if on_chunk: + on_chunk(chunk) + elif etype == "result": + result_text = event.get("result") or "" + sid = event.get("session_id") + if update_session and sid: + self.session_id = sid + in_t, out_t = self._usage_from_payload(event) + input_tokens += in_t + output_tokens += out_t + stderr_b = await proc.stderr.read() + stderr = stderr_b.decode("utf-8", errors="replace") + code = await proc.wait() + else: + stdout_b, stderr_b = await proc.communicate() + stderr = stderr_b.decode("utf-8", errors="replace") + code = proc.returncode if proc.returncode is not None else 0 + stdout = stdout_b.decode("utf-8", errors="replace").strip() + if stdout: + try: + data = json.loads(stdout) + except json.JSONDecodeError: + result_text = stdout + else: + result_text = data.get("result") or "" + sid = data.get("session_id") + if update_session and sid: + self.session_id = sid + input_tokens, output_tokens = self._usage_from_payload(data) + + if code != 0: + detail = stderr.strip() or f"exit code {code}" + raise RuntimeError(f"Cursor CLI failed: {detail}") + + content = "".join(text_parts) if text_parts else result_text + if not content and result_text: + content = result_text + return { + "content": content, + "tool_calls": None, + "input_tokens": input_tokens, + "output_tokens": output_tokens, + } + + async def async_call( + self, model: str, messages: list[dict], max_tokens: int + ) -> str: + """One-shot ask-mode call (no session resume) — used for branch naming.""" + del max_tokens # CLI manages its own limits + result = await self._run_cli( + model, + messages, + mode="ask", + stream=False, + update_session=False, + ) + return result.get("content") or "" + + async def async_stream_raw( + self, + model: str, + messages: list[dict], + max_tokens: int, + tools: list[dict] | None = None, + on_chunk: Callable[[str], None] | None = None, + ) -> dict: + """Agent-mode streaming call; resumes the CLI chat across REPL turns.""" + del max_tokens, tools # Cursor owns tools and token limits + return await self._run_cli( + model, + messages, + mode="agent", + stream=True, + on_chunk=on_chunk, + update_session=True, + ) diff --git a/src/agenttester/repl.py b/src/agenttester/repl.py index fd58974..a3d7e7a 100644 --- a/src/agenttester/repl.py +++ b/src/agenttester/repl.py @@ -31,6 +31,7 @@ from .providers import ( AnthropicProvider, BedrockProvider, + CursorProvider, OpenAICompatProvider, Provider, ) @@ -154,6 +155,9 @@ def _provider_label(m: Model) -> str: return m.provider.endpoint if isinstance(m.provider, BedrockProvider): return f"bedrock:{m.provider.region}" + if isinstance(m.provider, CursorProvider): + mid = m.model_id or "auto" + return f"cursor:{mid}" return type(m.provider).__name__.lower() @@ -235,6 +239,35 @@ async def _query_async( model.restore_messages(saved) return f"[error] {e}" + # Cursor CLI is a full agent: point it at this model's worktree and stream. + # Do not use AgentTester's OpenAI-style tool loop. + if isinstance(model.provider, CursorProvider): + if model.workdir: + model.provider.workspace = model.workdir + model.add_message("user", prompt) + parts: list[str] = [] + + def _on_cursor_chunk(chunk: str) -> None: + parts.append(chunk) + if on_event: + on_event("chunk", chunk) + + try: + result = await model.provider.async_stream_raw( + model.model_id, + model.messages, + model.max_tokens, + on_chunk=_on_cursor_chunk, + ) + reply = result.get("content") or "".join(parts) + model.input_tokens += result.get("input_tokens", 0) + model.output_tokens += result.get("output_tokens", 0) + except Exception as e: + model.pop_message() + return f"[error] {e}" + model.add_message("assistant", reply) + return reply + streaming_providers = (AnthropicProvider, BedrockProvider, OpenAICompatProvider) if isinstance(model.provider, streaming_providers): # Streaming, no tool use — stream text chunks directly. @@ -1024,6 +1057,8 @@ async def _iterate_run( if raw == "/reset": for model in models.values(): model.messages = list(seed) + if isinstance(model.provider, CursorProvider): + model.provider.reset_session() console.print("[dim]Context cleared.[/dim]\n") continue diff --git a/tests/test_config.py b/tests/test_config.py index b8d8865..ba0f913 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -19,6 +19,7 @@ from agenttester.providers import ( AnthropicProvider, BedrockProvider, + CursorProvider, OpenAICompatProvider, ) @@ -610,6 +611,26 @@ def test_github_provider_custom_api_key_env(self, tmp_path: Path) -> None: assert isinstance(prov, OpenAICompatProvider) assert prov.api_key_env == "MY_GH_TOKEN" + def test_cursor_provider(self, tmp_path: Path) -> None: + config_file = tmp_path / "cfg.yaml" + config_file.write_text( + "providers:\n" + " cursor:\n" + " type: cursor\n" + " binary: cursor-agent\n" + " optimize_for: balanced\n" + "evaluators:\n" + " - name: cursor-auto\n" + " provider: cursor\n" + " model: auto\n" + ) + evaluators, _ = load_evaluators_and_eval_config(config_file) + prov = evaluators[0].provider + assert isinstance(prov, CursorProvider) + assert prov.api_key_env == "CURSOR_API_KEY" + assert prov.binary == "cursor-agent" + assert prov.optimize_for == "balanced" + class TestGetReportsDir: def test_default_is_global_config_dir(self, tmp_path: Path) -> None: diff --git a/tests/test_cursor_provider.py b/tests/test_cursor_provider.py new file mode 100644 index 0000000..888d221 --- /dev/null +++ b/tests/test_cursor_provider.py @@ -0,0 +1,155 @@ +"""Tests for CursorProvider.""" + +from __future__ import annotations + +import json +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from agenttester.providers.cursor import CursorProvider + + +def test_usage_sums_cache_fields() -> None: + p = CursorProvider() + assert p._usage_from_payload( + { + "usage": { + "inputTokens": 7, + "cacheReadTokens": 100, + "cacheWriteTokens": 20, + "outputTokens": 50, + } + } + ) == (127, 50) + + +def test_prompt_resume_uses_last_user_only() -> None: + p = CursorProvider() + msgs = [ + {"role": "system", "content": "skills"}, + {"role": "user", "content": "first"}, + {"role": "assistant", "content": "ok"}, + {"role": "user", "content": "second"}, + ] + assert p._prompt_from_messages(msgs, resume=True) == "second" + assert "skills" in p._prompt_from_messages(msgs, resume=False) + assert "first" in p._prompt_from_messages(msgs, resume=False) + + +def test_build_command_omits_model_for_auto() -> None: + p = CursorProvider() + with patch.object(p, "_resolve_binary", return_value="/bin/agent"): + cmd = p._build_command("auto", "hi", mode="agent", stream=True, resume=False) + assert "--model" not in cmd + assert "hi" in cmd + assert "--force" in cmd + assert "--trust" in cmd + + +def test_build_command_pins_static_model_and_resume() -> None: + p = CursorProvider() + p.session_id = "sess-1" + p.workspace = "/tmp/wt" + with patch.object(p, "_resolve_binary", return_value="/bin/agent"): + cmd = p._build_command( + "composer-2.5", "hi", mode="agent", stream=True, resume=True + ) + assert cmd[cmd.index("--model") + 1] == "composer-2.5" + assert cmd[cmd.index("--resume") + 1] == "sess-1" + assert cmd[cmd.index("--workspace") + 1] == "/tmp/wt" + + +def test_auto_smart_maps_to_auto() -> None: + p = CursorProvider() + with patch.object(p, "_resolve_binary", return_value="/bin/agent"): + cmd = p._build_command( + "auto-smart", "hi", mode="agent", stream=False, resume=False + ) + assert "--model" not in cmd + + +def test_reset_session_clears_id() -> None: + p = CursorProvider() + p.session_id = "x" + p.reset_session() + assert p.session_id is None + + +@pytest.mark.asyncio +async def test_async_stream_raw_parses_ndjson() -> None: + p = CursorProvider() + events = [ + json.dumps( + { + "type": "assistant", + "timestamp_ms": 1, + "message": { + "role": "assistant", + "content": [{"type": "text", "text": "Hel"}], + }, + } + ), + json.dumps( + { + "type": "assistant", + "timestamp_ms": 2, + "message": { + "role": "assistant", + "content": [{"type": "text", "text": "lo"}], + }, + } + ), + json.dumps( + { + "type": "result", + "subtype": "success", + "result": "Hello", + "session_id": "abc", + "usage": {"inputTokens": 1, "outputTokens": 2}, + } + ), + ] + lines = [f"{e}\n".encode() for e in events] + [b""] + idx = [0] + + async def _readline() -> bytes: + if idx[0] < len(lines): + line = lines[idx[0]] + idx[0] += 1 + return line + return b"" + + mock_stdout = MagicMock() + mock_stdout.readline = _readline + mock_stderr = MagicMock() + mock_stderr.read = AsyncMock(return_value=b"") + + mock_proc = MagicMock() + mock_proc.stdout = mock_stdout + mock_proc.stderr = mock_stderr + mock_proc.wait = AsyncMock(return_value=0) + + async def _fake_exec(*_a, **_k): + return mock_proc + + with ( + patch( + "agenttester.providers.cursor.asyncio.create_subprocess_exec", + _fake_exec, + ), + patch.object(p, "_resolve_binary", return_value="/bin/agent"), + ): + chunks: list[str] = [] + result = await p.async_stream_raw( + "auto", + [{"role": "user", "content": "hi"}], + 128, + on_chunk=chunks.append, + ) + + assert result["content"] == "Hello" + assert result["input_tokens"] == 1 + assert result["output_tokens"] == 2 + assert p.session_id == "abc" + assert chunks == ["Hel", "lo"] diff --git a/tests/test_repl.py b/tests/test_repl.py index e0f1c61..d99ddba 100644 --- a/tests/test_repl.py +++ b/tests/test_repl.py @@ -12,6 +12,7 @@ from agenttester.providers import ( AnthropicProvider, BedrockProvider, + CursorProvider, OpenAICompatProvider, ) from agenttester.repl import ( @@ -191,6 +192,23 @@ def test_explicit_models_section_with_named_bedrock_provider( assert m.provider.aws_profile == "my-profile" assert m.model_id == "anthropic.claude-3-5-sonnet-20241022-v2:0" + def test_explicit_models_section_with_cursor_provider(self, tmp_path: Path) -> None: + cfg = tmp_path / "agent-tester.yaml" + cfg.write_text( + yaml.dump( + { + "providers": {"cursor": {"type": "cursor"}}, + "models": { + "cursor-auto": {"provider": "cursor", "model": "auto"}, + }, + } + ) + ) + m = load_models(cfg)["cursor-auto"] + assert isinstance(m.provider, CursorProvider) + assert m.model_id == "auto" + assert m.provider.api_key_env == "CURSOR_API_KEY" + def test_explicit_models_section_with_inline_endpoint(self, tmp_path: Path) -> None: cfg = tmp_path / "agent-tester.yaml" cfg.write_text( @@ -276,6 +294,38 @@ async def test_openai_provider_uses_async_stream_raw(self) -> None: provider.async_stream_raw.assert_called_once() assert result == "streamed reply" + async def test_cursor_provider_skips_agent_loop_uses_cli(self) -> None: + provider = MagicMock(spec=CursorProvider) + provider.async_stream_raw = AsyncMock( + return_value={ + "content": "cursor reply", + "tool_calls": None, + "input_tokens": 3, + "output_tokens": 4, + } + ) + provider.workspace = None + executor = MagicMock(spec=ToolExecutor) + executor.workdir = "/tmp/cursor-wt" + model = Model( + name="m", + model_id="auto", + provider=provider, + tool_executor=executor, + ) + with patch( + "agenttester.repl.run_agent_loop", + new_callable=AsyncMock, + ) as mock_loop: + result = await _query_async(model, "hi") + mock_loop.assert_not_called() + assert provider.workspace == "/tmp/cursor-wt" + provider.async_stream_raw.assert_called_once() + assert result == "cursor reply" + assert model.input_tokens == 3 + assert model.output_tokens == 4 + assert model.messages[-1] == {"role": "assistant", "content": "cursor reply"} + # --------------------------------------------------------------------------- # _run_one