Retry transient Electron release downloads

scripts/build-desktop-artifact.ts

@@ -17,6 +17,7 @@ import {
   buildMacAdaptiveIconSync,
 } from "./lib/mac-adaptive-icon.ts";
 import { resolveCatalogDependencies } from "./lib/resolve-catalog.ts";
+import { isTransientNetworkFailure } from "./fetch-with-network-retry.ts";
 import { DESKTOP_RELEASE_APP_ID } from "@threadlines/shared/desktopIdentity";
 import { fromYaml } from "@threadlines/shared/schemaYaml";
 
@@ -31,6 +32,7 @@ import * as Logger from "effect/Logger";
 import * as Option from "effect/Option";
 import * as Path from "effect/Path";
 import * as Schema from "effect/Schema";
+import * as Schedule from "effect/Schedule";
 import * as Stream from "effect/Stream";
 import { Command, Flag } from "effect/unstable/cli";
 import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process";
@@ -1178,7 +1180,7 @@ const buildDesktopArtifact = Effect.fn("buildDesktopArtifact")(function* (
   }
 
   yield* Effect.log("[desktop-artifact] Installing staged production dependencies...");
-  yield* runCommand(
+  const electronBuilderCommand = runCommand(
     ChildProcess.make({
       cwd: stageAppDir,
       ...commandOutputOptions(options.verbose),
@@ -1236,6 +1238,20 @@ const buildDesktopArtifact = Effect.fn("buildDesktopArtifact")(function* (
       shell: process.platform === "win32",
     })`${vpBinary} exec --filter @threadlines/desktop -- electron-builder --projectDir ${stageAppDir} ${platformConfig.cliFlag} --${options.arch} --publish never`,
   );
+  yield* electronBuilderCommand.pipe(
+    Effect.tapError((error) =>
+      isTransientNetworkFailure(error)
+        ? Effect.logWarning(
+            "[desktop-artifact] Electron packaging hit a transient network failure; bounded retry policy active.",
+          )
+        : Effect.void,
+    ),
+    Effect.retry({
+      schedule: Schedule.spaced("2 seconds"),
+      times: 2,
+      while: isTransientNetworkFailure,
+    }),
+  );
 
   const stageDistDir = path.join(stageAppDir, "dist");
   if (!(yield* fs.exists(stageDistDir))) {

scripts/ensure-electron-binary.ts

@@ -17,6 +17,8 @@ import { fileURLToPath } from "node:url";
 import * as Console from "effect/Console";
 import * as Effect from "effect/Effect";
 
+import { fetchWithNetworkRetry } from "./fetch-with-network-retry.ts";
+
 const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
 const desktopRequire = createRequire(resolve(repoRoot, "apps/desktop/package.json"));
 const electronPackagePath = desktopRequire.resolve("electron/package.json");
@@ -110,7 +112,13 @@ async function installElectronBinary(): Promise<void> {
   try {
     log(`Downloading ${artifactUrl}`);
 
-    const response = await fetch(artifactUrl, { signal: AbortSignal.timeout(300_000) });
+    const response = await fetchWithNetworkRetry(artifactUrl, {
+      onRetry: ({ attempt, delayMs, error, maxAttempts }) => {
+        log(
+          `Electron download attempt ${attempt}/${maxAttempts} failed (${String(error)}); retrying in ${delayMs}ms`,
+        );
+      },
+    });
     if (!response.ok) {
       throw new Error(
         `Electron artifact download failed: ${response.status} ${response.statusText}`,

scripts/fetch-with-network-retry.test.ts

@@ -0,0 +1,65 @@
+import { assert, expect, it } from "vite-plus/test";
+
+import { fetchWithNetworkRetry, isTransientNetworkFailure } from "./fetch-with-network-retry.ts";
+
+it("identifies the external download failures seen in release jobs", () => {
+  assert.equal(isTransientNetworkFailure(new Error("socket hang up")), true);
+  assert.equal(isTransientNetworkFailure(new TypeError("fetch failed")), true);
+  assert.equal(isTransientNetworkFailure(new Error("UND_ERR_SOCKET: other side closed")), true);
+  assert.equal(isTransientNetworkFailure(new Error("checksum mismatch")), false);
+});
+
+it("retries transient network failures with a fresh attempt", async () => {
+  const response = new Response("electron artifact");
+  const networkFailure = new TypeError("fetch failed");
+  let fetchAttempts = 0;
+  const fetchImplementation = (async () => {
+    fetchAttempts += 1;
+    if (fetchAttempts < 3) {
+      throw networkFailure;
+    }
+    return response;
+  }) as typeof fetch;
+  const sleepDelays: number[] = [];
+  const retries: Array<{ attempt: number; delayMs: number; maxAttempts: number }> = [];
+
+  const result = await fetchWithNetworkRetry("https://example.test/electron.zip", {
+    fetchImplementation,
+    maxAttempts: 3,
+    retryDelayMs: 250,
+    sleep: async (delayMs) => {
+      sleepDelays.push(delayMs);
+    },
+    onRetry: ({ attempt, delayMs, maxAttempts }) => {
+      retries.push({ attempt, delayMs, maxAttempts });
+    },
+  });
+
+  assert.strictEqual(result, response);
+  assert.equal(fetchAttempts, 3);
+  assert.deepStrictEqual(sleepDelays, [250, 500]);
+  assert.deepStrictEqual(retries, [
+    { attempt: 1, delayMs: 250, maxAttempts: 3 },
+    { attempt: 2, delayMs: 500, maxAttempts: 3 },
+  ]);
+});
+
+it("fails after the bounded number of attempts", async () => {
+  const networkFailure = new TypeError("other side closed");
+  let fetchAttempts = 0;
+  const fetchImplementation = (async () => {
+    fetchAttempts += 1;
+    throw networkFailure;
+  }) as typeof fetch;
+
+  await expect(
+    fetchWithNetworkRetry("https://example.test/electron.zip", {
+      fetchImplementation,
+      maxAttempts: 3,
+      retryDelayMs: 0,
+      sleep: async () => {},
+    }),
+  ).rejects.toBe(networkFailure);
+
+  assert.equal(fetchAttempts, 3);
+});

scripts/fetch-with-network-retry.ts

@@ -0,0 +1,69 @@
+export interface FetchWithNetworkRetryOptions {
+  readonly maxAttempts?: number;
+  readonly retryDelayMs?: number;
+  readonly timeoutMs?: number;
+  readonly fetchImplementation?: typeof fetch;
+  readonly sleep?: (delayMs: number) => Promise<void>;
+  readonly onRetry?: (input: {
+    readonly attempt: number;
+    readonly delayMs: number;
+    readonly error: unknown;
+    readonly maxAttempts: number;
+  }) => void;
+}
+
+const defaultSleep = (delayMs: number): Promise<void> =>
+  new Promise((resolve) => setTimeout(resolve, delayMs));
+
+const TRANSIENT_NETWORK_ERROR_PATTERNS = [
+  "eai_again",
+  "econnreset",
+  "etimedout",
+  "fetch failed",
+  "other side closed",
+  "socket hang up",
+  "status code 502",
+  "status code 503",
+  "status code 504",
+  "und_err_socket",
+] as const;
+
+export function isTransientNetworkFailure(error: unknown): boolean {
+  const message = error instanceof Error ? `${error.name}: ${error.message}` : String(error);
+  const normalizedMessage = message.toLowerCase();
+  return TRANSIENT_NETWORK_ERROR_PATTERNS.some((pattern) => normalizedMessage.includes(pattern));
+}
+
+export async function fetchWithNetworkRetry(
+  url: string,
+  options: FetchWithNetworkRetryOptions = {},
+): Promise<Response> {
+  const {
+    maxAttempts = 3,
+    retryDelayMs = 1_000,
+    timeoutMs = 300_000,
+    fetchImplementation = fetch,
+    sleep = defaultSleep,
+    onRetry,
+  } = options;
+
+  if (!Number.isInteger(maxAttempts) || maxAttempts < 1) {
+    throw new Error(`maxAttempts must be a positive integer, got ${maxAttempts}`);
+  }
+
+  for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
+    try {
+      return await fetchImplementation(url, { signal: AbortSignal.timeout(timeoutMs) });
+    } catch (error) {
+      if (attempt === maxAttempts) {
+        throw error;
+      }
+
+      const delayMs = retryDelayMs * attempt;
+      onRetry?.({ attempt, delayMs, error, maxAttempts });
+      await sleep(delayMs);
+    }
+  }
+
+  throw new Error("Electron artifact download exhausted its retry attempts");
+}