From 1ba8e42c6c72f4378cfd74bdf9b6d4733afb2873 Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Fri, 7 Aug 2026 11:01:48 +0200 Subject: [PATCH 1/2] ci(linux): guard the native payload, declare patchelf, drop the dead zsync glob MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three findings from an audit of the Linux build chain. None of them affects a published artifact today — they are the missing nets and one stale doc claim. before-pack.cjs asserted nothing on Linux. Its comment said "Linux ships no native addon of its own", written before the wgpu compositor addon and the PipeWire capture helper landed, and never revisited. macOS got a payload check for exactly this reason; Linux now gets the symmetric one, covering compositor_view.node, the five symbol-renamed ffmpeg .so files, the helper, whisper-stt-server and its ggml sidecars. The helper's ffmpeg/ subdirectory is checked separately, because a name match is not the property that matters: it has to be a directory holding the *unrenamed* libraries. That also catches a real collision — `fetch:ffmpeg` vendors the static ffmpeg binary to that exact path, so running it by hand replaces the directory with a file and produces a helper that cannot start. CI never sees it, `build:linux` only runs fetch:ffmpeg:sdk. Running `node scripts/before-pack.cjs` on Linux also fell through to the Windows branch and reported a missing D3D11 addon at a win32 path. patchelf is an unconditional dependency of build-linux-compositor-addon.mjs (resolvePatchelf throws without it) and was not in the apt line. It works because the ubuntu-24.04 image preinstalls it; declaring it stops the build depending on the runner image's contents. The release/**/*.zsync upload glob has matched nothing since the app-builder-lib 26.x bump — that version dropped zsync for an embedded block map, and there is no updater in this repo to consume one anyway. It stayed invisible because if-no-files-found: error evaluates the union of the patterns, so a dead glob alongside live ones never fails. --- .github/workflows/build.yml | 17 +- scripts/before-pack.cjs | 198 ++++++++++++++++-- .../engineering/ci-workflows.md | 2 +- 3 files changed, 196 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3c3f8293..8714639f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -439,8 +439,14 @@ jobs: - name: Setup Node.js uses: ./.github/actions/setup - - name: Install pacman build dependencies - run: sudo apt-get update && sudo apt-get install -y libarchive-tools + # bsdtar (libarchive-tools) is fpm's mtree generator for the pacman target. + # patchelf is what build-linux-compositor-addon.mjs renames the ffmpeg symbols + # with, so the addon cannot bind to Chromium's bundled ffmpeg — an unconditional + # dependency that resolvePatchelf() throws on. It happens to be preinstalled on + # the ubuntu-24.04 image, which is why this has worked; declaring it means the + # build stops depending on the runner image's contents. + - name: Install Linux packaging and native build dependencies + run: sudo apt-get update && sudo apt-get install -y libarchive-tools patchelf - name: Stage whisper-stt binaries shell: bash @@ -457,9 +463,14 @@ jobs: name: openscreen-linux path: | release/**/*.AppImage - release/**/*.zsync release/**/*.deb release/**/*.pacman + # No *.zsync: nothing produces one. zsync is electron-updater's delta format, + # this repo has no updater (no electron-updater, no autoUpdater, no + # latest-linux.yml), and app-builder-lib 26.x dropped zsync entirely in favour + # of the block map it embeds in the AppImage. The glob had matched nothing + # since the dependency bump, silently — `if-no-files-found: error` evaluates + # the union of these patterns, so one dead glob among live ones never fails. if-no-files-found: error retention-days: 30 diff --git a/scripts/before-pack.cjs b/scripts/before-pack.cjs index c38cea87..b94e1a41 100644 --- a/scripts/before-pack.cjs +++ b/scripts/before-pack.cjs @@ -41,6 +41,20 @@ const FIX = const FIX_MAC = "Rebuild it with:\n\n npm run build:native:compositor:mac\n\nor use `npm run build:mac`, which does that for you."; +const FIX_LINUX = + "Rebuild it with:\n\n npm run build:native:compositor:linux\n\nor use `npm run build:linux`, which does that for you."; + +const FIX_LINUX_HELPER = + "Rebuild it with:\n\n npm run build:native:linux\n\nor use `npm run build:linux`, which does that for you."; + +/** Everything the PipeWire capture helper is compiled from. */ +const HELPER_SOURCE_PATHS = [ + "electron/native/pipewire-capture/src", + "electron/native/pipewire-capture/csrc", + "electron/native/pipewire-capture/build.rs", + "electron/native/pipewire-capture/Cargo.toml", +].map((p) => path.join(ROOT, p)); + /** * Everything that has to be inside `electron/native/bin/darwin-/` for the .app to * work, keyed by what breaks when it is absent. @@ -89,6 +103,56 @@ const MAC_REQUIRED = [ }, ]; +/** + * The Linux counterpart of MAC_REQUIRED. It exists for the same reason: until this + * hook grew a Linux branch, `beforePack` asserted nothing at all on Linux — the + * comment said "Linux ships no native addon of its own", which stopped being true + * when the wgpu compositor addon and the PipeWire capture helper landed. + * + * `linux.extraResources` ships this directory wholesale (`filter: ["linux-*​/**"]`), + * so "present here" is the same thing as "present in the installed app". + * + * Note the two ffmpeg sets, which is why `ffmpeg/` is required separately below: + * the `.so` files sitting directly in this directory are the compositor's copies, + * with every symbol renamed to `osff_*` so the addon cannot bind to Chromium's + * bundled ffmpeg. The helper needs the *unrenamed* originals, which is what the + * `ffmpeg/` subdirectory holds. + */ +const LINUX_REQUIRED = [ + { + match: (name) => name === "compositor_view.node", + what: "the wgpu/Vulkan compositor addon", + breaks: "the preview renders nothing and every export falls back to the no-op compositor", + fix: FIX_LINUX, + }, + { + match: (name) => /^lib(avcodec|avformat|avutil|swresample|swscale)\.so\.\d+$/.test(name), + what: "the symbol-renamed LGPL ffmpeg shared objects the compositor links", + breaks: "the compositor addon cannot be loaded at all (ld.so error at require())", + fix: FIX_LINUX, + atLeast: 5, + }, + { + match: (name) => name === "openscreen-pipewire-helper", + what: "the PipeWire screen-capture helper", + breaks: "Wayland capture is unavailable and cursor recording throws", + fix: FIX_LINUX_HELPER, + }, + { + match: (name) => name === "whisper-stt-server", + what: "the whisper.cpp STT helper", + breaks: "transcription and captions fail with a developer error shown to end users", + fix: "Build it with:\n\n npm run build:whisper-binaries\n\nor stage CI's with `bash scripts/stage-whisper-stt.sh linux-x64`.", + }, + { + match: (name) => /^libggml.*\.so(\.\d+)*$/.test(name), + what: "the ggml backend shared objects the STT helper links", + breaks: "whisper-stt-server dies in ld.so before main(), so STT times out with no diagnostic", + fix: "Build it with:\n\n npm run build:whisper-binaries", + atLeast: 1, + }, +]; + /** electron-builder passes `context.arch` as a numeric enum; map it to our directory tag. */ function archTagFor(context) { const BY_INDEX = { 0: "ia32", 1: "x64", 2: "armv7l", 3: "arm64", 4: "universal" }; @@ -96,20 +160,22 @@ function archTagFor(context) { return name && name !== "universal" ? name : process.arch; } -function checkMacNativePayload(context) { - const tag = `darwin-${archTagFor(context)}`; - const dir = path.join(ROOT, "electron", "native", "bin", tag); +/** + * Shared by the macOS and Linux payload checks — same contract on both: the arch-tagged + * directory under electron/native/bin/ is what extraResources ships, so a missing entry + * here is a missing entry in the installed app. + */ +function checkNativePayload({ dir, required, osLabel, bundleNoun, emptyDirFix }) { if (!fs.existsSync(dir)) { throw new Error( - `Refusing to package: ${path.relative(ROOT, dir)} does not exist, so the .app would ` + + `Refusing to package: ${path.relative(ROOT, dir)} does not exist, so ${bundleNoun} would ` + "ship with no native modules at all.\n\n" + - `${FIX_MAC}\n\nThe STT helper and the capture helper are separate builds — see\n` + - "technical-documentation/engineering/build-and-packaging.md.", + emptyDirFix, ); } const present = fs.readdirSync(dir); - const missing = MAC_REQUIRED.filter( + const missing = required.filter( (req) => present.filter((name) => req.match(name)).length < (req.atLeast ?? 1), ); if (missing.length === 0) { @@ -123,7 +189,7 @@ function checkMacNativePayload(context) { ) .join("\n"); throw new Error( - `Refusing to package an incomplete macOS payload.\n\n` + + `Refusing to package an incomplete ${osLabel} payload.\n\n` + ` looked in: ${path.relative(ROOT, dir)}\n\n` + `Missing:\n${detail}\n\n` + "Every one of these fails silently or as an unactionable timeout in the installed\n" + @@ -131,6 +197,62 @@ function checkMacNativePayload(context) { ); } +function checkMacNativePayload(context) { + checkNativePayload({ + dir: path.join(ROOT, "electron", "native", "bin", `darwin-${archTagFor(context)}`), + required: MAC_REQUIRED, + osLabel: "macOS", + bundleNoun: "the .app", + emptyDirFix: `${FIX_MAC}\n\nThe STT helper and the capture helper are separate builds — see\ntechnical-documentation/engineering/build-and-packaging.md.`, + }); +} + +function checkLinuxNativePayload(context) { + const dir = path.join(ROOT, "electron", "native", "bin", `linux-${archTagFor(context)}`); + checkNativePayload({ + dir, + required: LINUX_REQUIRED, + osLabel: "Linux", + bundleNoun: "the package", + emptyDirFix: `${FIX_LINUX}\n\nThe capture helper and the STT helper are separate builds — see\ntechnical-documentation/engineering/build-and-packaging.md.`, + }); + + // Checked apart from LINUX_REQUIRED because "something named ffmpeg exists" is not the + // property that matters — it has to be a directory holding the *unrenamed* libraries. + // An empty one, or the wrong kind of entry, passes a name match and still ships a + // helper that cannot start. + const helperFfmpeg = path.join(dir, "ffmpeg"); + const isDir = fs.existsSync(helperFfmpeg) && fs.statSync(helperFfmpeg).isDirectory(); + if (fs.existsSync(helperFfmpeg) && !isDir) { + // `fetch:ffmpeg` vendors the *static* ffmpeg binary to exactly this path, while + // `build:native:linux` wants a directory here. They collide, and the loser is + // whichever ran first. CI never sees it — `build:linux` only runs + // `fetch:ffmpeg:sdk`, which does not write the executable — so this fires on + // local packaging after someone has run the full fetch by hand. + throw new Error( + `Refusing to package: ${path.relative(ROOT, helperFfmpeg)} is a file, not a directory.\n\n` + + "That path is where the PipeWire helper's ffmpeg libraries live, but the static\n" + + "ffmpeg binary that `npm run fetch:ffmpeg` vendors lands on the same name and\n" + + "overwrote it. Delete it and re-run:\n\n npm run build:native:linux\n\n" + + "(`npm run build:linux` uses fetch:ffmpeg:sdk, which does not write that file.)", + ); + } + const libs = isDir + ? fs.readdirSync(helperFfmpeg).filter((name) => /^lib(av|sw)\w+\.so\.\d+$/.test(name)) + : []; + if (libs.length === 0) { + throw new Error( + "Refusing to package an incomplete Linux payload.\n\n" + + ` looked in: ${path.relative(ROOT, helperFfmpeg)}\n\n` + + "Missing:\n - the PipeWire helper's own ffmpeg shared objects\n" + + " without it: openscreen-pipewire-helper dies in ld.so, so capture never starts\n" + + ` ${FIX_LINUX_HELPER.replace(/\n+/g, " ")}\n\n` + + "These are deliberately not the copies one level up: those have every symbol\n" + + "renamed to `osff_*` for the compositor addon, and the helper needs the originals.", + ); + } +} + /** Newest mtime under `target` (file or directory), or 0 if it does not exist. */ function newestMtimeMs(target) { let stat; @@ -149,24 +271,31 @@ function newestMtimeMs(target) { return newest; } -function checkCompositorAddonFreshness(addon = ADDON, fix = FIX, label = "D3D11") { +// `label` is a full noun ("D3D11 compositor addon", "PipeWire capture helper"): this now +// guards artifacts that are not all compositor addons. +function checkCompositorAddonFreshness( + addon = ADDON, + fix = FIX, + label = "D3D11 compositor addon", + sources, +) { if (!fs.existsSync(addon)) { throw new Error( - `Refusing to package: the ${label} compositor addon is missing.\n\n expected: ${addon}\n\n${fix}`, + `Refusing to package: the ${label} is missing.\n\n expected: ${addon}\n\n${fix}`, ); } const addonMs = fs.statSync(addon).mtimeMs; - const stale = SOURCE_PATHS.map((source) => ({ source, ms: newestMtimeMs(source) })).filter( - (entry) => entry.ms > addonMs, - ); + const stale = (sources ?? SOURCE_PATHS) + .map((source) => ({ source, ms: newestMtimeMs(source) })) + .filter((entry) => entry.ms > addonMs); if (stale.length === 0) { return; } const newest = stale.reduce((a, b) => (a.ms > b.ms ? a : b)); throw new Error( - `Refusing to package a stale ${label} compositor addon.\n\n` + + `Refusing to package a stale ${label}.\n\n` + ` addon: ${path.relative(ROOT, addon)}\n` + ` addon built: ${new Date(addonMs).toISOString()}\n` + ` newer source: ${path.relative(ROOT, newest.source)} (${new Date(newest.ms).toISOString()})\n\n` + @@ -189,10 +318,26 @@ exports.default = async function beforePack(context) { const tag = `darwin-${archTagFor(context)}`; const shipped = path.join(ROOT, "electron", "native", "bin", tag, "compositor_view.node"); checkMacNativePayload(context); - checkCompositorAddonFreshness(shipped, FIX_MAC, "Metal"); + checkCompositorAddonFreshness(shipped, FIX_MAC, "Metal compositor addon"); + return; + } + if (platform === "linux") { + const tag = `linux-${archTagFor(context)}`; + const dir = path.join(ROOT, "electron", "native", "bin", tag); + checkLinuxNativePayload(context); + checkCompositorAddonFreshness( + path.join(dir, "compositor_view.node"), + FIX_LINUX, + "wgpu/Vulkan compositor addon", + ); + checkCompositorAddonFreshness( + path.join(dir, "openscreen-pipewire-helper"), + FIX_LINUX_HELPER, + "PipeWire capture helper", + HELPER_SOURCE_PATHS, + ); return; } - // Linux ships no native addon of its own; nothing to assert. }; // Runnable on its own for debugging: `node scripts/before-pack.cjs` @@ -204,9 +349,28 @@ if (require.main === module) { checkCompositorAddonFreshness( path.join(ROOT, "electron", "native", "bin", tag, "compositor_view.node"), FIX_MAC, - "Metal", + "Metal compositor addon", ); console.log(`macOS native payload complete in electron/native/bin/${tag}, addon up to date.`); + } else if (process.platform === "linux") { + // Was falling through to the Windows branch below, so running this on Linux + // reported a missing D3D11 addon at a win32 path — noise, on the one platform + // where the hook now has something to say. + const tag = `linux-${process.arch}`; + const dir = path.join(ROOT, "electron", "native", "bin", tag); + checkLinuxNativePayload({ arch: undefined }); + checkCompositorAddonFreshness( + path.join(dir, "compositor_view.node"), + FIX_LINUX, + "wgpu/Vulkan compositor addon", + ); + checkCompositorAddonFreshness( + path.join(dir, "openscreen-pipewire-helper"), + FIX_LINUX_HELPER, + "PipeWire capture helper", + HELPER_SOURCE_PATHS, + ); + console.log(`Linux native payload complete in electron/native/bin/${tag}, addon up to date.`); } else { checkCompositorAddonFreshness(); console.log("compositor addon is up to date with its Rust sources."); diff --git a/technical-documentation/engineering/ci-workflows.md b/technical-documentation/engineering/ci-workflows.md index 512588d7..7807c473 100644 --- a/technical-documentation/engineering/ci-workflows.md +++ b/technical-documentation/engineering/ci-workflows.md @@ -101,7 +101,7 @@ A `v*` tag or manual dispatch starts platform builds. Dispatch accepts `arch` (` - `build-windows` runs `npm run build:win` and uploads `openscreen-windows` for 30 days. - `build-windows-store` runs `npm run build:win:store` and uploads `openscreen-windows-store` for 30 days. - `build-macos` is an `arm64`/`x64` matrix. It builds Vite/Electron and native helpers, packages and optionally signs the app, creates DMGs, notarizes every signed build including pre-releases, and uploads one artifact per architecture for 30 days. -- `build-linux` produces AppImage, zsync, deb, and pacman files and uploads `openscreen-linux` for 30 days. +- `build-linux` produces AppImage, deb, and pacman files and uploads `openscreen-linux` for 30 days. No zsync: that is electron-updater's delta format, this repo ships no updater, and app-builder-lib 26.x embeds a block map in the AppImage instead. - `publish-release` waits for Windows NSIS, macOS, and Linux jobs; the Store job is not a dependency. It checks the tag against `package.json`, downloads the NSIS/macOS/Linux artifacts, and creates or updates a GitHub release with `OPENSCREEN_RELEASE_TOKEN`. The build comments and package behavior refer to the local Whisper architecture documented in [transcription and captions](../architecture/transcription-and-captions.md). The STT model downloads to user data at runtime and is not a release-build asset. From 3f88b38e2d35a1155b939a61c12aee2887613834 Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Fri, 7 Aug 2026 12:01:29 +0200 Subject: [PATCH 2/2] ci(linux): require each compositor ffmpeg library family individually MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `atLeast: 5` portait sur le total de correspondances d'une regex combinée, pas sur la présence de chaque famille. Cinq copies versionnées d'une même bibliothèque — libavcodec.so.58 à .62 laissées par un build précédent — satisfaisaient le compte pendant qu'une autre manquait : le paquet passait la garde et le compositeur ne chargeait pas, exactement le mode de panne que cette garde existe pour attraper. Une exigence par famille, ce qui nomme aussi précisément celle qui manque dans le message d'erreur au lieu d'un « au moins 5 » opaque. Exercé contre des payloads fabriqués : le cas dégradé passait avec l'ancienne regex et est refusé avec la nouvelle, chacune des cinq familles est requise séparément, et le payload complet reste accepté. --- scripts/before-pack.cjs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/scripts/before-pack.cjs b/scripts/before-pack.cjs index b94e1a41..e017b401 100644 --- a/scripts/before-pack.cjs +++ b/scripts/before-pack.cjs @@ -125,13 +125,18 @@ const LINUX_REQUIRED = [ breaks: "the preview renders nothing and every export falls back to the no-op compositor", fix: FIX_LINUX, }, - { - match: (name) => /^lib(avcodec|avformat|avutil|swresample|swscale)\.so\.\d+$/.test(name), - what: "the symbol-renamed LGPL ffmpeg shared objects the compositor links", + // Une exigence par famille, plutôt qu'`atLeast: 5` sur une regex combinée. Le + // compte total était satisfait par cinq copies versionnées d'une même + // bibliothèque — libavcodec.so.58 à .62 laissées par un build précédent — + // pendant qu'une autre manquait. Le paquet passait alors la garde et le + // compositeur ne chargeait pas : exactement le mode de panne que cette garde + // existe pour attraper. + ...["avcodec", "avformat", "avutil", "swresample", "swscale"].map((library) => ({ + match: (name) => new RegExp(`^lib${library}\\.so\\.\\d+$`).test(name), + what: `the symbol-renamed lib${library} shared object the compositor links`, breaks: "the compositor addon cannot be loaded at all (ld.so error at require())", fix: FIX_LINUX, - atLeast: 5, - }, + })), { match: (name) => name === "openscreen-pipewire-helper", what: "the PipeWire screen-capture helper",