From 9463dab58da46ffd7feda62e3f20654e40975641 Mon Sep 17 00:00:00 2001 From: badcuban <108198679+badcuban@users.noreply.github.com> Date: Wed, 12 Aug 2026 18:18:15 -0400 Subject: [PATCH] Retry transient Electron release downloads --- scripts/build-desktop-artifact.ts | 18 ++++++- scripts/ensure-electron-binary.ts | 10 +++- scripts/fetch-with-network-retry.test.ts | 65 ++++++++++++++++++++++ scripts/fetch-with-network-retry.ts | 69 ++++++++++++++++++++++++ 4 files changed, 160 insertions(+), 2 deletions(-) create mode 100644 scripts/fetch-with-network-retry.test.ts create mode 100644 scripts/fetch-with-network-retry.ts diff --git a/scripts/build-desktop-artifact.ts b/scripts/build-desktop-artifact.ts index d3e4fc603..a007782e6 100644 --- a/scripts/build-desktop-artifact.ts +++ b/scripts/build-desktop-artifact.ts @@ -17,6 +17,7 @@ import { buildMacAdaptiveIconSync, } from "./lib/mac-adaptive-icon.ts"; import { resolveCatalogDependencies } from "./lib/resolve-catalog.ts"; +import { isTransientNetworkFailure } from "./fetch-with-network-retry.ts"; import { DESKTOP_RELEASE_APP_ID } from "@threadlines/shared/desktopIdentity"; import { fromYaml } from "@threadlines/shared/schemaYaml"; @@ -31,6 +32,7 @@ import * as Logger from "effect/Logger"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; +import * as Schedule from "effect/Schedule"; import * as Stream from "effect/Stream"; import { Command, Flag } from "effect/unstable/cli"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; @@ -1178,7 +1180,7 @@ const buildDesktopArtifact = Effect.fn("buildDesktopArtifact")(function* ( } yield* Effect.log("[desktop-artifact] Installing staged production dependencies..."); - yield* runCommand( + const electronBuilderCommand = runCommand( ChildProcess.make({ cwd: stageAppDir, ...commandOutputOptions(options.verbose), @@ -1236,6 +1238,20 @@ const buildDesktopArtifact = Effect.fn("buildDesktopArtifact")(function* ( shell: process.platform === "win32", })`${vpBinary} exec --filter @threadlines/desktop -- electron-builder --projectDir ${stageAppDir} ${platformConfig.cliFlag} --${options.arch} --publish never`, ); + yield* electronBuilderCommand.pipe( + Effect.tapError((error) => + isTransientNetworkFailure(error) + ? Effect.logWarning( + "[desktop-artifact] Electron packaging hit a transient network failure; bounded retry policy active.", + ) + : Effect.void, + ), + Effect.retry({ + schedule: Schedule.spaced("2 seconds"), + times: 2, + while: isTransientNetworkFailure, + }), + ); const stageDistDir = path.join(stageAppDir, "dist"); if (!(yield* fs.exists(stageDistDir))) { diff --git a/scripts/ensure-electron-binary.ts b/scripts/ensure-electron-binary.ts index 2d7c533a8..b3327063b 100644 --- a/scripts/ensure-electron-binary.ts +++ b/scripts/ensure-electron-binary.ts @@ -17,6 +17,8 @@ import { fileURLToPath } from "node:url"; import * as Console from "effect/Console"; import * as Effect from "effect/Effect"; +import { fetchWithNetworkRetry } from "./fetch-with-network-retry.ts"; + const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const desktopRequire = createRequire(resolve(repoRoot, "apps/desktop/package.json")); const electronPackagePath = desktopRequire.resolve("electron/package.json"); @@ -110,7 +112,13 @@ async function installElectronBinary(): Promise { try { log(`Downloading ${artifactUrl}`); - const response = await fetch(artifactUrl, { signal: AbortSignal.timeout(300_000) }); + const response = await fetchWithNetworkRetry(artifactUrl, { + onRetry: ({ attempt, delayMs, error, maxAttempts }) => { + log( + `Electron download attempt ${attempt}/${maxAttempts} failed (${String(error)}); retrying in ${delayMs}ms`, + ); + }, + }); if (!response.ok) { throw new Error( `Electron artifact download failed: ${response.status} ${response.statusText}`, diff --git a/scripts/fetch-with-network-retry.test.ts b/scripts/fetch-with-network-retry.test.ts new file mode 100644 index 000000000..1575bd40a --- /dev/null +++ b/scripts/fetch-with-network-retry.test.ts @@ -0,0 +1,65 @@ +import { assert, expect, it } from "vite-plus/test"; + +import { fetchWithNetworkRetry, isTransientNetworkFailure } from "./fetch-with-network-retry.ts"; + +it("identifies the external download failures seen in release jobs", () => { + assert.equal(isTransientNetworkFailure(new Error("socket hang up")), true); + assert.equal(isTransientNetworkFailure(new TypeError("fetch failed")), true); + assert.equal(isTransientNetworkFailure(new Error("UND_ERR_SOCKET: other side closed")), true); + assert.equal(isTransientNetworkFailure(new Error("checksum mismatch")), false); +}); + +it("retries transient network failures with a fresh attempt", async () => { + const response = new Response("electron artifact"); + const networkFailure = new TypeError("fetch failed"); + let fetchAttempts = 0; + const fetchImplementation = (async () => { + fetchAttempts += 1; + if (fetchAttempts < 3) { + throw networkFailure; + } + return response; + }) as typeof fetch; + const sleepDelays: number[] = []; + const retries: Array<{ attempt: number; delayMs: number; maxAttempts: number }> = []; + + const result = await fetchWithNetworkRetry("https://example.test/electron.zip", { + fetchImplementation, + maxAttempts: 3, + retryDelayMs: 250, + sleep: async (delayMs) => { + sleepDelays.push(delayMs); + }, + onRetry: ({ attempt, delayMs, maxAttempts }) => { + retries.push({ attempt, delayMs, maxAttempts }); + }, + }); + + assert.strictEqual(result, response); + assert.equal(fetchAttempts, 3); + assert.deepStrictEqual(sleepDelays, [250, 500]); + assert.deepStrictEqual(retries, [ + { attempt: 1, delayMs: 250, maxAttempts: 3 }, + { attempt: 2, delayMs: 500, maxAttempts: 3 }, + ]); +}); + +it("fails after the bounded number of attempts", async () => { + const networkFailure = new TypeError("other side closed"); + let fetchAttempts = 0; + const fetchImplementation = (async () => { + fetchAttempts += 1; + throw networkFailure; + }) as typeof fetch; + + await expect( + fetchWithNetworkRetry("https://example.test/electron.zip", { + fetchImplementation, + maxAttempts: 3, + retryDelayMs: 0, + sleep: async () => {}, + }), + ).rejects.toBe(networkFailure); + + assert.equal(fetchAttempts, 3); +}); diff --git a/scripts/fetch-with-network-retry.ts b/scripts/fetch-with-network-retry.ts new file mode 100644 index 000000000..dee4089dd --- /dev/null +++ b/scripts/fetch-with-network-retry.ts @@ -0,0 +1,69 @@ +export interface FetchWithNetworkRetryOptions { + readonly maxAttempts?: number; + readonly retryDelayMs?: number; + readonly timeoutMs?: number; + readonly fetchImplementation?: typeof fetch; + readonly sleep?: (delayMs: number) => Promise; + readonly onRetry?: (input: { + readonly attempt: number; + readonly delayMs: number; + readonly error: unknown; + readonly maxAttempts: number; + }) => void; +} + +const defaultSleep = (delayMs: number): Promise => + new Promise((resolve) => setTimeout(resolve, delayMs)); + +const TRANSIENT_NETWORK_ERROR_PATTERNS = [ + "eai_again", + "econnreset", + "etimedout", + "fetch failed", + "other side closed", + "socket hang up", + "status code 502", + "status code 503", + "status code 504", + "und_err_socket", +] as const; + +export function isTransientNetworkFailure(error: unknown): boolean { + const message = error instanceof Error ? `${error.name}: ${error.message}` : String(error); + const normalizedMessage = message.toLowerCase(); + return TRANSIENT_NETWORK_ERROR_PATTERNS.some((pattern) => normalizedMessage.includes(pattern)); +} + +export async function fetchWithNetworkRetry( + url: string, + options: FetchWithNetworkRetryOptions = {}, +): Promise { + const { + maxAttempts = 3, + retryDelayMs = 1_000, + timeoutMs = 300_000, + fetchImplementation = fetch, + sleep = defaultSleep, + onRetry, + } = options; + + if (!Number.isInteger(maxAttempts) || maxAttempts < 1) { + throw new Error(`maxAttempts must be a positive integer, got ${maxAttempts}`); + } + + for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { + try { + return await fetchImplementation(url, { signal: AbortSignal.timeout(timeoutMs) }); + } catch (error) { + if (attempt === maxAttempts) { + throw error; + } + + const delayMs = retryDelayMs * attempt; + onRetry?.({ attempt, delayMs, error, maxAttempts }); + await sleep(delayMs); + } + } + + throw new Error("Electron artifact download exhausted its retry attempts"); +}