diff --git a/src/harness/core.test.ts b/src/harness/core.test.ts new file mode 100644 index 0000000..b4615cb --- /dev/null +++ b/src/harness/core.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from "bun:test"; + +import { + isRetryableModelError, + isSystemicModelError, + ModelError, +} from "./core"; + +describe("isRetryableModelError", () => { + it("treats 408 request timeout as retryable", () => { + const error = new ModelError({ status: 408, message: "Request Timeout" }); + expect(isRetryableModelError(error)).toBe(true); + }); + + it("treats 429 rate limit as retryable", () => { + const error = new ModelError({ status: 429, message: "Too Many Requests" }); + expect(isRetryableModelError(error)).toBe(true); + }); + + it("treats 5xx server errors as retryable", () => { + for (const status of [500, 502, 503, 504]) { + const error = new ModelError({ + status, + message: `Server error ${status}`, + }); + expect(isRetryableModelError(error)).toBe(true); + } + }); + + it("does not treat client errors other than 408 and 429 as retryable", () => { + for (const status of [400, 401, 403, 404, 422]) { + const error = new ModelError({ + status, + message: `Client error ${status}`, + }); + expect(isRetryableModelError(error)).toBe(false); + } + }); + + it("does not treat undefined status as retryable", () => { + const error = new ModelError({ message: "Unknown error" }); + expect(isRetryableModelError(error)).toBe(false); + }); +}); + +describe("isSystemicModelError", () => { + it("treats 401, 403, and 404 as systemic", () => { + for (const status of [401, 403, 404]) { + const error = new ModelError({ status, message: `Systemic ${status}` }); + expect(isSystemicModelError(error)).toBe(true); + } + }); + + it("treats undefined status as systemic", () => { + const error = new ModelError({ message: "Unclassified error" }); + expect(isSystemicModelError(error)).toBe(true); + }); + + it("does not treat 408, 429, or 5xx as systemic", () => { + for (const status of [408, 429, 500, 502, 503, 504]) { + const error = new ModelError({ status, message: `Transient ${status}` }); + expect(isSystemicModelError(error)).toBe(false); + } + }); +}); diff --git a/src/harness/core.ts b/src/harness/core.ts index 25afb17..e5f0b21 100644 --- a/src/harness/core.ts +++ b/src/harness/core.ts @@ -238,7 +238,9 @@ export class ModelError extends TaggedError("ModelError")< export function isRetryableModelError(error: ModelError): boolean { return ( - error.status === 429 || (error.status !== undefined && error.status >= 500) + error.status === 408 || + error.status === 429 || + (error.status !== undefined && error.status >= 500) ); } diff --git a/src/harness/run.test.ts b/src/harness/run.test.ts index 422a4aa..85b99f3 100644 --- a/src/harness/run.test.ts +++ b/src/harness/run.test.ts @@ -324,6 +324,48 @@ describe("runBenchmark", () => { expect(result.metrics.skippedQuestions).toBe(1); expect(result.metrics.accuracy).toBe(1); }); + it("scores a sample as Skipped (excluded from accuracy) when the model exhausts 408 retries", async () => { + const service: ModelService = { + generate: (messages) => { + const userMsg = + messages.find((m) => m.role === MessageRole.User)?.content ?? ""; + if (userMsg.includes("Q1")) { + return effectFail( + new ModelError({ + message: "OpenRouter HTTP 408: request timed out", + status: 408, + }) + ); + } + return effectSucceed({ + completion: "Answer: B", + message: { role: MessageRole.Assistant, content: "Answer: B" }, + generationTimeMs: 100, + }); + }, + }; + const model = { service, layer: layerSucceed(Model, Model.of(service)) }; + const solver = generate(model.service, { + temperature: 0, + reasoningEffort: "high", + }); + const layers = mergeAll( + fakeDatasetLayer(SAMPLES), + layerSucceed(Solver, Solver.of(solver)), + layerSucceed(Scorer, Scorer.of(mcqScorer)), + model.layer, + noopProgressLayer, + noopCheckpointLayer + ); + const result = await runPromise( + runBenchmark({ epochs: 1, maxConcurrency: 2 }).pipe(provide(layers)) + ); + const skipped = result.sampleScores.find((s) => s.sampleId === "s-correct"); + expect(skipped?.score.value).toBe(ScoreValue.Skipped); + expect(result.metrics.totalQuestions).toBe(1); + expect(result.metrics.skippedQuestions).toBe(1); + expect(result.metrics.accuracy).toBe(1); + }); it("scores a sample as Incorrect (counted against accuracy) on a non-retryable model error", async () => { const service: ModelService = { generate: (messages) => { diff --git a/src/runtime/retry.test.ts b/src/runtime/retry.test.ts index a44bf61..a1616f8 100644 --- a/src/runtime/retry.test.ts +++ b/src/runtime/retry.test.ts @@ -25,6 +25,8 @@ const rateLimit = new ModelError({ status: 429, message: "429" }); const serverError = new ModelError({ status: 503, message: "503" }); +const timeoutError = new ModelError({ status: 408, message: "408" }); + const clientError = new ModelError({ status: 400, message: "400" }); const warnSpies: { @@ -128,7 +130,16 @@ describe("rateLimitRetrySchedule", () => { const result = await runHarnessPromise(flaky.pipe(retry(schedule))); expect(result).toBe(3); }); - it("does not retry non-retryable 4xx (other than 429)", async () => { + it("retries transient 408 timeout errors", async () => { + let attempts = 0; + const flaky = suspend(() => { + attempts++; + return attempts < 3 ? fail(timeoutError) : succeed(attempts); + }); + const result = await runHarnessPromise(flaky.pipe(retry(schedule))); + expect(result).toBe(3); + }); + it("does not retry non-retryable 4xx (other than 408 and 429)", async () => { let attempts = 0; const program = suspend(() => { attempts++;