fix(harness): treat HTTP 408 timeouts as retryable and score exhausted timeouts as Skipped

src/harness/core.test.ts

@@ -0,0 +1,65 @@
+import { describe, expect, it } from "bun:test";
+
+import {
+  isRetryableModelError,
+  isSystemicModelError,
+  ModelError,
+} from "./core";
+
+describe("isRetryableModelError", () => {
+  it("treats 408 request timeout as retryable", () => {
+    const error = new ModelError({ status: 408, message: "Request Timeout" });
+    expect(isRetryableModelError(error)).toBe(true);
+  });
+
+  it("treats 429 rate limit as retryable", () => {
+    const error = new ModelError({ status: 429, message: "Too Many Requests" });
+    expect(isRetryableModelError(error)).toBe(true);
+  });
+
+  it("treats 5xx server errors as retryable", () => {
+    for (const status of [500, 502, 503, 504]) {
+      const error = new ModelError({
+        status,
+        message: `Server error ${status}`,
+      });
+      expect(isRetryableModelError(error)).toBe(true);
+    }
+  });
+
+  it("does not treat client errors other than 408 and 429 as retryable", () => {
+    for (const status of [400, 401, 403, 404, 422]) {
+      const error = new ModelError({
+        status,
+        message: `Client error ${status}`,
+      });
+      expect(isRetryableModelError(error)).toBe(false);
+    }
+  });
+
+  it("does not treat undefined status as retryable", () => {
+    const error = new ModelError({ message: "Unknown error" });
+    expect(isRetryableModelError(error)).toBe(false);
+  });
+});
+
+describe("isSystemicModelError", () => {
+  it("treats 401, 403, and 404 as systemic", () => {
+    for (const status of [401, 403, 404]) {
+      const error = new ModelError({ status, message: `Systemic ${status}` });
+      expect(isSystemicModelError(error)).toBe(true);
+    }
+  });
+
+  it("treats undefined status as systemic", () => {
+    const error = new ModelError({ message: "Unclassified error" });
+    expect(isSystemicModelError(error)).toBe(true);
+  });
+
+  it("does not treat 408, 429, or 5xx as systemic", () => {
+    for (const status of [408, 429, 500, 502, 503, 504]) {
+      const error = new ModelError({ status, message: `Transient ${status}` });
+      expect(isSystemicModelError(error)).toBe(false);
+    }
+  });
+});

src/harness/core.ts

@@ -238,7 +238,9 @@ export class ModelError extends TaggedError("ModelError")<
 
 export function isRetryableModelError(error: ModelError): boolean {
   return (
-    error.status === 429 || (error.status !== undefined && error.status >= 500)
+    error.status === 408 ||
+    error.status === 429 ||
+    (error.status !== undefined && error.status >= 500)
   );
 }
 

src/harness/run.test.ts

@@ -324,6 +324,48 @@ describe("runBenchmark", () => {
     expect(result.metrics.skippedQuestions).toBe(1);
     expect(result.metrics.accuracy).toBe(1);
   });
+  it("scores a sample as Skipped (excluded from accuracy) when the model exhausts 408 retries", async () => {
+    const service: ModelService = {
+      generate: (messages) => {
+        const userMsg =
+          messages.find((m) => m.role === MessageRole.User)?.content ?? "";
+        if (userMsg.includes("Q1")) {
+          return effectFail(
+            new ModelError({
+              message: "OpenRouter HTTP 408: request timed out",
+              status: 408,
+            })
+          );
+        }
+        return effectSucceed({
+          completion: "Answer: B",
+          message: { role: MessageRole.Assistant, content: "Answer: B" },
+          generationTimeMs: 100,
+        });
+      },
+    };
+    const model = { service, layer: layerSucceed(Model, Model.of(service)) };
+    const solver = generate(model.service, {
+      temperature: 0,
+      reasoningEffort: "high",
+    });
+    const layers = mergeAll(
+      fakeDatasetLayer(SAMPLES),
+      layerSucceed(Solver, Solver.of(solver)),
+      layerSucceed(Scorer, Scorer.of(mcqScorer)),
+      model.layer,
+      noopProgressLayer,
+      noopCheckpointLayer
+    );
+    const result = await runPromise(
+      runBenchmark({ epochs: 1, maxConcurrency: 2 }).pipe(provide(layers))
+    );
+    const skipped = result.sampleScores.find((s) => s.sampleId === "s-correct");
+    expect(skipped?.score.value).toBe(ScoreValue.Skipped);
+    expect(result.metrics.totalQuestions).toBe(1);
+    expect(result.metrics.skippedQuestions).toBe(1);
+    expect(result.metrics.accuracy).toBe(1);
+  });
   it("scores a sample as Incorrect (counted against accuracy) on a non-retryable model error", async () => {
     const service: ModelService = {
       generate: (messages) => {

src/runtime/retry.test.ts

@@ -25,6 +25,8 @@ const rateLimit = new ModelError({ status: 429, message: "429" });
 
 const serverError = new ModelError({ status: 503, message: "503" });
 
+const timeoutError = new ModelError({ status: 408, message: "408" });
+
 const clientError = new ModelError({ status: 400, message: "400" });
 
 const warnSpies: {
@@ -128,7 +130,16 @@ describe("rateLimitRetrySchedule", () => {
     const result = await runHarnessPromise(flaky.pipe(retry(schedule)));
     expect(result).toBe(3);
   });
-  it("does not retry non-retryable 4xx (other than 429)", async () => {
+  it("retries transient 408 timeout errors", async () => {
+    let attempts = 0;
+    const flaky = suspend(() => {
+      attempts++;
+      return attempts < 3 ? fail(timeoutError) : succeed(attempts);
+    });
+    const result = await runHarnessPromise(flaky.pipe(retry(schedule)));
+    expect(result).toBe(3);
+  });
+  it("does not retry non-retryable 4xx (other than 408 and 429)", async () => {
     let attempts = 0;
     const program = suspend(() => {
       attempts++;