diff --git a/.github/workflows/build-whisper-stt.yml b/.github/workflows/build-whisper-stt.yml index f5a4dd2f..e63eba23 100644 --- a/.github/workflows/build-whisper-stt.yml +++ b/.github/workflows/build-whisper-stt.yml @@ -54,7 +54,16 @@ jobs: tag: darwin-x64 label: macOS x64 (CPU) vulkan: false - - os: ubuntu-latest + # PINNED, and not `ubuntu-latest` — see the long comment on build.yml's + # build-linux job. This leg produces the WORST offender of the two: built on + # 24.04, whisper-stt-server and the libggml backends needed GLIBC_2.38 and + # GLIBCXX_3.4.32, so they died in ld.so on Ubuntu 22.04, Debian 12 and RHEL 9 + # and transcription failed there with an unactionable developer error. The + # symbols were never asked for: `__isoc23_strtol` is glibc 2.38's C23 redirect + # and `_ZSt21ios_base_library_initv` is emitted into every TU that includes + # by GCC 13.2+. 22.04 has the CMake (3.22 >= the 3.20 required) and + # a C++20 compiler; nothing here uses a GCC 13-only library feature. + - os: ubuntu-22.04 arch: x64 tag: linux-x64 label: Linux x64 (Vulkan + CPU fallback) @@ -73,7 +82,10 @@ jobs: uses: ./.github/actions/setup - name: Install Ninja (Linux) - if: matrix.os == 'ubuntu-latest' + # `startsWith`, matching the macOS step below, rather than an exact image name: + # this was `== 'ubuntu-latest'`, so pinning the image would have skipped Ninja + # silently instead of failing. + if: startsWith(matrix.os, 'ubuntu') run: | sudo apt-get update sudo apt-get install -y ninja-build build-essential diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8ce503db..59ee3a15 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -490,7 +490,25 @@ jobs: build-linux: name: Linux packages - runs-on: ubuntu-latest + # PINNED, and not to `ubuntu-latest`. The linker binds every symbol to the newest + # version its BUILD machine offers, so the runner image silently decides the oldest + # distro these packages can run on — nothing in the source asks for any of it. + # On ubuntu-latest (24.04) that floor was glibc 2.38 / GLIBCXX_3.4.32, which put + # Ubuntu 22.04, Debian 12 and RHEL 9 out of range: whisper-stt-server and the ggml + # backends died in ld.so before main(), and compositor_view.node (2.35, from a single + # `hypotf`) died on RHEL 9 as well. Both fail SILENTLY — the app still launches, + # captions just report a developer error and the preview renders nothing — and no + # package format catches it, because the deb/rpm/pacman `depends` lists are + # hand-written in electron-builder.json5 and electron-builder passes fpm none of + # --rpm-autoreq*, so not even dnf generates the libc.so.6(GLIBC_2.38) requirement + # that would have refused the install. + # + # 22.04 is the oldest distro the README claims (it names it as the PipeWire + # baseline), and it is the binding one: glibc 2.35 and libstdc++6 from GCC 12 + # (GLIBCXX_3.4.30, CXXABI_1.3.13), against Debian 12's 2.36/3.4.30/1.3.13. + # scripts/before-pack.cjs enforces that ceiling on the built payload, so bumping + # this image alone cannot quietly raise the floor again. + runs-on: ubuntu-22.04 steps: - name: Checkout code uses: actions/checkout@v7 @@ -501,13 +519,28 @@ jobs: # bsdtar (libarchive-tools) is fpm's mtree generator for the pacman target. # rpmbuild (rpm) is what fpm shells out to for the rpm target; without it the # build fails at packaging, not at config parse. + - name: Install Linux packaging dependencies + run: sudo apt-get update && sudo apt-get install -y libarchive-tools rpm + # patchelf is what build-linux-compositor-addon.mjs renames the ffmpeg symbols # with, so the addon cannot bind to Chromium's bundled ffmpeg — an unconditional - # dependency that resolvePatchelf() throws on. It happens to be preinstalled on - # the ubuntu-24.04 image, which is why this has worked; declaring it means the - # build stops depending on the runner image's contents. - - name: Install Linux packaging and native build dependencies - run: sudo apt-get update && sudo apt-get install -y libarchive-tools rpm patchelf + # dependency that resolvePatchelf() throws on. + # + # NOT from apt, which is the whole reason this is its own step: 22.04 carries + # patchelf 0.14.3 and `--rename-dynamic-symbols` first shipped in 0.18.0, so the + # apt copy fails the rename outright. 0.18.0 is also exactly what the 24.04 image + # provided, so the renaming behaviour is unchanged from what already ships. + # resolvePatchelf() checks ~/.local/bin before /usr/bin, so this wins over + # whatever the image happens to carry. + - name: Install patchelf + run: | + curl -fsSLo /tmp/patchelf.tar.gz \ + https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-x86_64.tar.gz + echo "ce84f2447fb7a8679e58bc54a20dc2b01b37b5802e12c57eece772a6f14bf3f0 /tmp/patchelf.tar.gz" | sha256sum -c - + mkdir -p ~/.local/bin + tar -xzf /tmp/patchelf.tar.gz -C /tmp ./bin/patchelf + mv /tmp/bin/patchelf ~/.local/bin/patchelf + ~/.local/bin/patchelf --version - name: Stage whisper-stt binaries shell: bash diff --git a/scripts/before-pack.cjs b/scripts/before-pack.cjs index 83752c1f..26421796 100644 --- a/scripts/before-pack.cjs +++ b/scripts/before-pack.cjs @@ -299,6 +299,188 @@ function checkLinuxNativePayload(context) { "renamed to `osff_*` for the compositor addon, and the helper needs the originals.", ); } + + checkLinuxSymbolVersionFloor(dir); +} + +/** + * The highest versioned symbol a shipped ELF may require, per version prefix. + * + * The Linux counterpart of the Windows import-table guard (checkWinNoRedistDependency + * and importedDlls(), from #321 — this may land first, in which case they arrive with + * it), and the same failure that hook could not see: the linker binds each symbol to + * the newest version the BUILD machine + * offers, so the runner image silently decides the oldest distro the packages run on. + * It works on every developer machine and in CI by construction, and only the target + * distro tells the truth. + * + * Nothing in the source asks for any of it. Built on ubuntu-latest (24.04) the payload + * needed GLIBC_2.38 for four `__isoc23_strto*` — glibc 2.38's C23 redirect of `strtol` + * — GLIBCXX_3.4.32 for `_ZSt21ios_base_library_initv`, which GCC 13.2+ emits into every + * translation unit that includes , and GLIBC_2.35 for one `hypotf`, a symbol + * that has existed since 2.2.5 and whose newest version Rust's f32::hypot simply took. + * + * That shipped: on Ubuntu 22.04, Debian 12 and RHEL 9, whisper-stt-server and the ggml + * backends died in ld.so before main() (transcription and captions fail with a developer + * error), and on RHEL 9 compositor_view.node failed require() as well (no preview, and + * every export falls back to the no-op compositor). The app still LAUNCHES on all of + * them — Electron itself only needs 2.25 — so it reads as a broken app rather than a + * broken package, and nothing in any log says otherwise. No package format catches it + * either: the deb/rpm/pacman `depends` lists are hand-written in electron-builder.json5, + * and electron-builder passes fpm none of --rpm-autoreq*, so not even dnf generates the + * `libc.so.6(GLIBC_2.38)` requirement that would have refused the install. + * + * The ceiling is what the OLDEST distro the README claims actually provides. Ubuntu + * 22.04 is the binding one — glibc 2.35, libstdc++6 from GCC 12 — against Debian 12's + * 2.36 with the same libstdc++. Raising any of these is a decision to drop a distro + * from the README, not a build detail; the runners are pinned to match (build.yml's + * build-linux and build-whisper-stt.yml). + */ +const MAX_SYMBOL_VERSION = { GLIBC: "2.35", GLIBCXX: "3.4.30", CXXABI: "1.3.13" }; + +/** Dotted numeric compare, so 3.4.9 < 3.4.30 and 2.4 < 2.38 rather than by string. */ +function compareVersions(a, b) { + const left = a.split(".").map(Number); + const right = b.split(".").map(Number); + for (let i = 0; i < Math.max(left.length, right.length); i++) { + if ((left[i] ?? 0) !== (right[i] ?? 0)) return (left[i] ?? 0) - (right[i] ?? 0); + } + return 0; +} + +/** + * The highest version an ELF needs per prefix, as `{ GLIBC: "2.38", GLIBCXX: "3.4.32" }`. + * + * Reads `.gnu.version_r` (the version NEEDS) and deliberately not `.gnu.version_d` (the + * version DEFINITIONS): libc and libstdc++ define every version they ever shipped, so + * reading definitions would report a bundled library as needing itself. Parsed here + * rather than shelled out to readelf for the same reason importedDlls() does not use + * dumpbin — binutils is not installed on every machine that packages this. + * + * 64-bit little-endian only, which is every arch this ships (x86_64, aarch64). + */ +function neededSymbolVersions(file) { + const b = fs.readFileSync(file); + if (b.readUInt32BE(0) !== 0x7f454c46) throw new Error(`${file} is not an ELF binary`); + if (b[4] !== 2 || b[5] !== 1) throw new Error(`${file} is not 64-bit little-endian ELF`); + + const shoff = Number(b.readBigUInt64LE(0x28)); + const shentsize = b.readUInt16LE(0x3a); + const SHT_GNU_VERNEED = 0x6ffffffe; + + let section; + for (let i = 0; i < b.readUInt16LE(0x3c); i++) { + const sh = shoff + i * shentsize; + if (b.readUInt32LE(sh + 4) !== SHT_GNU_VERNEED) continue; + // sh_info is the Verneed count; sh_link is the string table these names live in. + const strtabHeader = shoff + b.readUInt32LE(sh + 0x28) * shentsize; + section = { + offset: Number(b.readBigUInt64LE(sh + 0x18)), + count: b.readUInt32LE(sh + 0x2c), + strtab: Number(b.readBigUInt64LE(strtabHeader + 0x18)), + }; + break; + } + // No such section means the binary needs no versioned symbols at all — legitimate + // for a fully static one, and nothing to check either way. + if (!section) return {}; + + const nameAt = (at) => + b.subarray(section.strtab + at, b.indexOf(0, section.strtab + at)).toString("latin1"); + + const highest = {}; + let verneed = section.offset; + for (let i = 0; i < section.count; i++) { + let vernaux = verneed + b.readUInt32LE(verneed + 8); + for (let j = 0; j < b.readUInt16LE(verneed + 2); j++) { + // "GLIBC_2.38" -> prefix GLIBC, version 2.38. Anything not shaped like that + // (there is none in practice) is skipped rather than guessed at. + const [, prefix, version] = + /^(.+)_(\d+(?:\.\d+)*)$/.exec(nameAt(b.readUInt32LE(vernaux + 8))) ?? []; + if (prefix && (!highest[prefix] || compareVersions(version, highest[prefix]) > 0)) { + highest[prefix] = version; + } + vernaux += b.readUInt32LE(vernaux + 12); + } + verneed += b.readUInt32LE(verneed + 12); + } + return highest; +} + +/** Every ELF under `dir`, recursively — the helper's ffmpeg sits in a subdirectory. */ +function elfFilesUnder(dir) { + const found = []; + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + found.push(...elfFilesUnder(full)); + continue; + } + if (!entry.isFile()) continue; + // By magic, not by extension: the helpers, whisper-stt-server and ffmpeg have none. + const magic = Buffer.alloc(4); + const fd = fs.openSync(full, "r"); + try { + fs.readSync(fd, magic, 0, 4, 0); + } finally { + fs.closeSync(fd); + } + if (magic.readUInt32BE(0) === 0x7f454c46) found.push(full); + } + return found; +} + +/** Nothing we ship may need a newer glibc or libstdc++ than MAX_SYMBOL_VERSION allows. */ +function checkLinuxSymbolVersionFloor(dir) { + const scanned = elfFilesUnder(dir).map((file) => ({ + name: path.relative(dir, file), + needs: neededSymbolVersions(file), + })); + + // The same assertion checkWinNoRedistDependency makes, for the same reason: a guard + // that silently stops looking reports "clean" for the rest of the project's life. + // Every dynamically linked binary in this payload needs versioned glibc symbols, so + // finding none anywhere means the parser broke. Asserted across the scan rather than + // per file, because a genuinely static binary legitimately has no .gnu.version_r. + if (!scanned.some((entry) => entry.needs.GLIBC)) { + throw new Error( + `Refusing to package: read no glibc symbol versions from any of the ${scanned.length} ` + + `ELF files in ${path.relative(ROOT, dir)}.\n\n` + + "Every one of them links glibc, so this is a bug in neededSymbolVersions()\n" + + "(scripts/before-pack.cjs), not a self-contained payload. Fix the parser — leaving\n" + + "it is how packages that cannot start on the supported distros get shipped again.", + ); + } + + const offenders = scanned + .map((entry) => ({ + name: entry.name, + bad: Object.entries(MAX_SYMBOL_VERSION) + .filter( + ([prefix, max]) => entry.needs[prefix] && compareVersions(entry.needs[prefix], max) > 0, + ) + .map(([prefix, max]) => `${prefix}_${entry.needs[prefix]} (max ${prefix}_${max})`), + })) + .filter((entry) => entry.bad.length > 0); + if (offenders.length === 0) { + return; + } + + throw new Error( + "Refusing to package binaries that need a newer glibc or libstdc++ than the oldest\n" + + "supported distro provides.\n\n" + + ` looked in: ${path.relative(ROOT, dir)}\n\n` + + `${offenders.map((o) => ` - ${o.name} needs ${o.bad.join(", ")}`).join("\n")}\n\n` + + "Almost certainly nothing asked for this: the linker binds each symbol to the newest\n" + + "version the build machine offers, so this means something was built on a newer image\n" + + "than the floor. On the target it dies in ld.so before main() or fails require(), while\n" + + "the app still launches — so it reads as a broken app, and no package format catches it.\n\n" + + "Build on the pinned runners: ubuntu-22.04 in .github/workflows/build.yml (build-linux)\n" + + "and build-whisper-stt.yml. To see which symbols pulled a version in:\n\n" + + " readelf -V \n" + + " readelf -W --dyn-syms | grep @GLIBC_2.38\n\n" + + "Raising MAX_SYMBOL_VERSION drops a distro the README claims to support.", + ); } /** Newest mtime under `target` (file or directory), or 0 if it does not exist. */