From 7182940abf82fe2a2d150e9a3682c96028024f56 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 05:22:54 +0000 Subject: [PATCH] feat(agent-cli): add ori code harness for terminal-bench Co-Authored-By: Louis Vichy --- src/benchmarks/agent-cli/harness.ts | 191 +++++++++++++- src/benchmarks/agent-cli/schema.ts | 8 +- .../terminal-bench/ori-solver.test.ts | 241 ++++++++++++++++++ test/helpers/terminal-bench-sandbox.ts | 1 + 4 files changed, 439 insertions(+), 2 deletions(-) diff --git a/src/benchmarks/agent-cli/harness.ts b/src/benchmarks/agent-cli/harness.ts index 173450b..abb6861 100644 --- a/src/benchmarks/agent-cli/harness.ts +++ b/src/benchmarks/agent-cli/harness.ts @@ -27,6 +27,8 @@ export const DEFAULT_PRIME_AGENT_PACKAGE = export const DEFAULT_OMP_PACKAGE = "@oh-my-pi/pi-coding-agent@18.1.2" as const; +export const ORI_CODE_PACKAGE = "ori" as const; + export const OMP_BUN_VERSION = "bun-v1.3.14" as const; export const BUN_RELEASE_URL = @@ -145,6 +147,17 @@ function buildOmpImageSteps(agentPackage: string): string[] { ]; } +function buildOriCodeImageSteps(agentPackage: string): string[] { + if (agentPackage !== ORI_CODE_PACKAGE) { + throw new Error( + `ori code ships inside the ori CLI and takes no agentPackage; got ${JSON.stringify(agentPackage)} (pin the CLI with oriChannel instead)` + ); + } + return [ + "RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certificates git", + ]; +} + function buildBootstrapScript(opts: { oriInstallUrl: string; oriChannel: OriChannel; @@ -152,10 +165,14 @@ function buildBootstrapScript(opts: { }): string { const channelPrefix = opts.oriChannel === "stable" ? "" : `ORI_CHANNEL=${opts.oriChannel} `; + const versionCheck = + opts.binaryName === "ori" + ? "ori --version" + : `ori --version && ${opts.binaryName} --version`; return [ "set -euo pipefail", `curl -fsSL ${opts.oriInstallUrl} | ${channelPrefix}ORI_INSTALL_DIR=${ORI_INSTALL_DIR} bash`, - `ori --version && ${opts.binaryName} --version`, + versionCheck, ].join("\n"); } @@ -518,11 +535,47 @@ const OMP_HARNESS: OriHarnessDef = { parseRun: parseJsonAgentStream, }; +const ORI_CODE_HARNESS: OriHarnessDef = { + id: "code", + defaultPackage: ORI_CODE_PACKAGE, + binaryName: "ori", + remoteLogPath: "/logs/agent/ori-code.txt", + imageBuildSteps: (options) => buildOriCodeImageSteps(options.agentPackage), + buildBootstrapScript: (options) => + buildBootstrapScript({ ...options, binaryName: "ori" }), + buildRunScript: (options) => + [ + "set -euo pipefail", + "export HOME=/root", + "mkdir -p /logs/agent", + ...(options.hasSystemPrompt || options.hasAppendSystemPrompt + ? [ + 'echo "ori code does not support system prompt overrides" >&2', + "exit 2", + ] + : []), + ...(options.hasAllowedTools || options.hasDisallowedTools + ? [ + 'echo "ori code does not support tool allow/deny lists" >&2', + "exit 2", + ] + : []), + 'ori code --model "$TB_MODEL" \\', + ` --reasoning-effort ${options.reasoningEffort} \\`, + " --approvals self-drive \\", + " --output jsonl \\", + ` --prompt-file ${options.instructionPath} \\`, + ` 2>&1 > = { claude: CLAUDE_HARNESS, pi: ORI_PI_HARNESS, "prime-agent": PRIME_AGENT_HARNESS, omp: OMP_HARNESS, + code: ORI_CODE_HARNESS, }; export function getOriHarness(agent: OriAgent): OriHarnessDef { @@ -538,6 +591,142 @@ function reasoningTokensOf(usage: Record): number { return typeof reasoningTokens === "number" ? reasoningTokens : 0; } +function oriCodeFailureStatus(payload: unknown): string | undefined { + const failure = isRecord(payload) ? payload["failure"] : undefined; + const code = optionalStringField(failure, "code"); + const message = optionalStringField(failure, "message"); + if (code === undefined) { + return message; + } + return message === undefined ? code : `${code}: ${message}`; +} + +function parseOriCodeStream(stdout: string): OriAgentRun { + let inputTokens = 0; + let outputTokens = 0; + let totalCost = 0; + let turns = 0; + let toolCalls = 0; + let isError = false; + let apiErrorStatus: string | undefined; + let finalText: string | undefined; + let generationTimeMs: number | undefined; + let model: string | undefined; + let pendingText = ""; + const generationIds: string[] = []; + const assistantMessages: ModelMessage[] = []; + const responseItems: ResponseItem[] = []; + const flushText = () => { + if (pendingText.length === 0) { + return; + } + finalText = pendingText; + assistantMessages.push( + definedValues({ + role: MessageRole.Assistant, + content: pendingText, + model, + }) + ); + pendingText = ""; + }; + for (const line of stdout.split("\n")) { + const trimmed = line.trim(); + if (!trimmed.startsWith("{")) { + continue; + } + const parsed = Either.try(() => JSON.parse(trimmed)); + if (Either.isLeft(parsed) || !isRecord(parsed.right)) { + continue; + } + const envelope = parsed.right; + responseItems.push(envelope); + if (envelope["kind"] === "result") { + if (envelope["ok"] !== true) { + isError = true; + apiErrorStatus ??= optionalStringField(envelope["error"], "message"); + } + continue; + } + const streamEvent = envelope["event"]; + if (!isRecord(streamEvent) || streamEvent["type"] !== "runtime.event") { + continue; + } + const event = streamEvent["event"]; + if (!isRecord(event)) { + continue; + } + const payload = event["payload"]; + model = optionalStringField(event, "model") ?? model; + switch (event["type"]) { + case "assistant.text.delta": { + pendingText += optionalStringField(payload, "delta") ?? ""; + break; + } + case "tool.started": { + flushText(); + toolCalls++; + break; + } + case "turn.succeeded": { + flushText(); + turns++; + generationTimeMs = + (generationTimeMs ?? 0) + + (optionalNumberField(payload, "durationMs") ?? 0); + const usage = isRecord(payload) ? payload["usage"] : undefined; + if (isRecord(usage)) { + inputTokens += numberField(usage, "inputTokens"); + outputTokens += numberField(usage, "outputTokens"); + totalCost += numberField(usage, "costUsd"); + const ids = usage["generationIds"]; + if (Array.isArray(ids)) { + for (const id of ids) { + if (typeof id === "string" && !generationIds.includes(id)) { + generationIds.push(id); + } + } + } + } + break; + } + case "turn.failed": + case "session.failed": + case "runtime.error": { + flushText(); + isError = true; + apiErrorStatus = oriCodeFailureStatus(payload) ?? apiErrorStatus; + break; + } + default: { + break; + } + } + } + flushText(); + const hasTokens = inputTokens + outputTokens !== 0; + return { + usage: hasTokens + ? { + inputTokens, + outputTokens, + totalTokens: inputTokens + outputTokens, + reasoningTokens: 0, + totalCost, + } + : undefined, + generationIds, + generationTimeMs, + finalText, + assistantMessages, + responseItems, + isError, + apiErrorStatus, + turns: turns > 0 ? turns : undefined, + toolCalls, + }; +} + function parseJsonAgentStream(stdout: string): OriAgentRun { let inputTokens = 0; let outputTokens = 0; diff --git a/src/benchmarks/agent-cli/schema.ts b/src/benchmarks/agent-cli/schema.ts index 57fc926..92f711d 100644 --- a/src/benchmarks/agent-cli/schema.ts +++ b/src/benchmarks/agent-cli/schema.ts @@ -1,6 +1,12 @@ import type { ValueOf } from "../../internal/guards"; -export const ORI_AGENTS = ["pi", "claude", "prime-agent", "omp"] as const; +export const ORI_AGENTS = [ + "pi", + "claude", + "prime-agent", + "omp", + "code", +] as const; export type OriAgent = ValueOf; diff --git a/src/benchmarks/terminal-bench/ori-solver.test.ts b/src/benchmarks/terminal-bench/ori-solver.test.ts index 969f237..67c776f 100644 --- a/src/benchmarks/terminal-bench/ori-solver.test.ts +++ b/src/benchmarks/terminal-bench/ori-solver.test.ts @@ -53,6 +53,7 @@ import { NVM_INSTALL_SHA256, NVM_INSTALL_URL, OMP_BUN_VERSION, + ORI_CODE_PACKAGE, ORI_HARNESSES, } from "../agent-cli/harness"; import type { OriHarnessDef } from "../agent-cli/harness"; @@ -1812,3 +1813,243 @@ describe("terminal-bench omp via ori", () => { expect(steps.join("\n")).toContain('bun install -g "file:///opt/omp.tgz"'); }); }); + +describe("terminal-bench ori code", () => { + const CODE_GENERATION_IDS = [ + "gen-1788844773-8CTCvdudI7aQLUsPgGbd", + "gen-1788844774-0D2JBvRDSsgDglI9Fn8G", + ]; + const runtimeEvent = ( + type: string, + payload: Record + ): string => + JSON.stringify({ + event: { + type: "runtime.event", + event: { + type, + harness: "ori", + model: "meta/muse-spark-1.3", + runId: "run-1", + turnId: "turn-1", + sessionId: "session-1", + payload, + }, + }, + kind: "event", + }); + const CODE_STREAM = [ + JSON.stringify({ + event: { type: "audit.event", audit: { name: "command.received" } }, + kind: "event", + }), + runtimeEvent("run.started", { prompt: "Respond with exactly OK" }), + runtimeEvent("turn.started", { prompt: "Respond with exactly OK" }), + runtimeEvent("reasoning.delta", { delta: "[REDACTED]" }), + runtimeEvent("tool.started", { + name: "bash", + toolCallId: "call-1", + input: { command: "ls" }, + }), + runtimeEvent("tool.succeeded", { name: "bash", toolCallId: "call-1" }), + runtimeEvent("assistant.text.delta", { delta: "O" }), + runtimeEvent("assistant.text.delta", { delta: "K" }), + runtimeEvent("turn.succeeded", { + durationMs: 3662, + usage: { + cacheCreationTokens: 0, + cacheReadTokens: 3185, + contextTokens: 3603, + costUsd: 0.010407, + generationIds: [...CODE_GENERATION_IDS, CODE_GENERATION_IDS[0]], + inputTokens: 7073, + model: "meta/muse-spark-1.3", + outputTokens: 256, + }, + }), + JSON.stringify({ kind: "result", ok: true, sessionId: "session-1" }), + ].join("\n"); + + async function runCode( + opts?: Partial, + execCalls?: ExecCalls + ): Promise { + const layer = makeTerminalBenchFakeSandboxLayer({ + reward: 1, + testOutput: "1 passed", + agentEventStream: CODE_STREAM, + agentExitCode: 0, + ...(execCalls !== undefined && { execCalls }), + }); + const solverLayer = layerEffect(Solver)( + gen(function* () { + const sessionFactory = yield* SandboxSession; + return Solver.of( + oriSolver( + sessionFactory, + { ...SOLVER_OPTS, ...opts }, + getOriHarness("code") + ) + ); + }) + ); + return runPromise( + gen(function* () { + const solver = yield* Solver; + return yield* solver(sampleState()); + }).pipe( + provide( + layerMergeAll( + solverLayer.pipe(layerProvide(layer)), + noopProgressLayer, + noopCheckpointLayer + ) + ) + ) + ); + } + + it("is registered as an ori agent", () => { + expect(ORI_AGENTS).toContain("code"); + expect(getOriHarness("code").binaryName).toBe("ori"); + expect(getOriHarness("code").defaultPackage).toBe(ORI_CODE_PACKAGE); + expect(getOriHarness("code").remoteLogPath).toBe( + "/logs/agent/ori-code.txt" + ); + }); + + it("parses usage, cost, generation IDs, text, turns and tool calls", async () => { + const finalState = await runCode(); + expect(finalState.output?.completion).toBe("OK"); + expect(finalState.output?.usage).toEqual({ + inputTokens: 7073, + outputTokens: 256, + totalTokens: 7329, + reasoningTokens: 0, + totalCost: 0.010407, + }); + expect(finalState.sample.metadata?.["generationIds"]).toEqual( + CODE_GENERATION_IDS + ); + expect(finalState.sample.metadata?.["agent"]).toBe("code"); + expect(finalState.sample.metadata?.["agentTurns"]).toBe(1); + expect(finalState.sample.metadata?.["agentToolCalls"]).toBe(1); + expect(finalState.messages.at(-1)).toEqual({ + role: "assistant", + content: "OK", + model: "meta/muse-spark-1.3", + }); + }); + + it("launches headless ori code with jsonl output and self-drive approvals", async () => { + const execCalls: ExecCalls = []; + await runCode( + { model: "meta/muse-spark-1.3", agentReasoningEffort: "high" }, + execCalls + ); + const agentCall = execCalls.find((call) => + call.argv[2]?.includes("ori code") + ); + expect(agentCall?.env["TB_MODEL"]).toBe("meta/muse-spark-1.3"); + expect(agentCall?.argv[2]).toBe( + [ + "set -euo pipefail", + "export HOME=/root", + "mkdir -p /logs/agent", + 'ori code --model "$TB_MODEL" \\', + " --reasoning-effort high \\", + " --approvals self-drive \\", + " --output jsonl \\", + " --prompt-file /instruction.md \\", + ` 2>&1 { + const base = { + instructionPath: "/instruction.md", + logPath: "/logs/agent/ori-code.txt", + reasoningEffort: "medium" as const, + hasSystemPrompt: false, + hasAppendSystemPrompt: false, + hasAllowedTools: false, + hasDisallowedTools: false, + isolateAgentConfig: false, + }; + const withPrompt = ORI_HARNESSES.code.buildRunScript({ + ...base, + hasAppendSystemPrompt: true, + }); + expect(withPrompt).toContain( + 'echo "ori code does not support system prompt overrides" >&2\nexit 2' + ); + expect(withPrompt.indexOf("exit 2")).toBeLessThan( + withPrompt.indexOf("ori code --model") + ); + const withTools = ORI_HARNESSES.code.buildRunScript({ + ...base, + hasDisallowedTools: true, + }); + expect(withTools).toContain( + 'echo "ori code does not support tool allow/deny lists" >&2\nexit 2' + ); + }); + + it("marks failed turns and non-ok results as errors", () => { + const failedTurn = ORI_HARNESSES.code.parseRun( + [ + runtimeEvent("assistant.text.delta", { delta: "partial" }), + runtimeEvent("turn.failed", { + failure: { code: "provider_error", message: "rate limited" }, + }), + JSON.stringify({ + kind: "result", + ok: false, + error: { code: "turn_failed", message: "turn failed" }, + }), + ].join("\n") + ); + expect(failedTurn.isError).toBe(true); + expect(failedTurn.apiErrorStatus).toBe("provider_error: rate limited"); + expect(failedTurn.finalText).toBe("partial"); + expect(failedTurn.usage).toBeUndefined(); + expect(failedTurn.turns).toBeUndefined(); + + const pendingInteraction = ORI_HARNESSES.code.parseRun( + JSON.stringify({ + kind: "result", + ok: false, + error: { code: "interaction_pending", message: "question pending" }, + }) + ); + expect(pendingInteraction.isError).toBe(true); + expect(pendingInteraction.apiErrorStatus).toBe("question pending"); + }); + + it("installs only the ori CLI and rejects agent package overrides", () => { + const steps = ORI_HARNESSES.code.imageBuildSteps({ + agentPackage: ORI_CODE_PACKAGE, + }); + const dockerfile = steps.join("\n"); + expect(dockerfile).toContain("apt-get install"); + expect(dockerfile).not.toContain(DEFAULT_AGENT_RUNTIME_URL); + expect(dockerfile).not.toContain("npm install"); + expect(dockerfile).not.toContain("bun install"); + expect(() => + ORI_HARNESSES.code.imageBuildSteps({ agentPackage: "ori@1.2.3" }) + ).toThrow(/takes no agentPackage/); + expect( + ORI_HARNESSES.code.buildBootstrapScript({ + oriInstallUrl: "https://openrouter.ai/labs/ori/install.sh", + oriChannel: "alpha", + }) + ).toBe( + [ + "set -euo pipefail", + "curl -fsSL https://openrouter.ai/labs/ori/install.sh | ORI_CHANNEL=alpha ORI_INSTALL_DIR=/usr/local/bin bash", + "ori --version", + ].join("\n") + ); + }); +}); diff --git a/test/helpers/terminal-bench-sandbox.ts b/test/helpers/terminal-bench-sandbox.ts index c30bc34..7d0c7c4 100644 --- a/test/helpers/terminal-bench-sandbox.ts +++ b/test/helpers/terminal-bench-sandbox.ts @@ -35,6 +35,7 @@ const AGENT_COMMAND_MARKERS = [ "ori claude", "ori prime-agent", "ori omp", + "ori code", ] as const; const ORI_INSTALL_MARKER = "ORI_INSTALL_DIR=" as const;