diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 923a5c435..76ac9ef39 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -89,10 +89,19 @@ jobs: build-macos: name: macOS ${{ matrix.arch }} DMG - # RELEASE-BRANCH-ONLY: 1.8.0 ships Windows-only. Do NOT let this `if: false` - # reach main when promoting, or every later release becomes Windows-only too. - if: false - runs-on: macos-latest + # Build each arch NATIVELY. `macos-latest` is Apple Silicon, and everything the + # 1.8.0 macOS path added keys off the HOST arch: fetch-ffmpeg-macos.mjs configures + # with `--arch=${process.arch}`, and build-macos-compositor-addon.mjs installs into + # `darwin-${process.arch}` and runs cargo without `--target`. So the x64 job on an + # arm64 runner produced arm64 output in darwin-arm64/, and packaging then failed + # with "Refusing to package an incomplete macOS payload — looked in darwin-x64". + # v1.7.0 shipped an x64 DMG because it had neither the compositor addon nor a + # vendored ffmpeg to build; both arrived with 1.8.0 and nobody could see the + # breakage while this job sat behind `if: false`. + # Running x64 on an Intel runner fixes it without threading a target arch through + # ffmpeg's configure, cargo and the output paths — four blind changes on a release + # branch, none of them testable without a Mac. + runs-on: ${{ matrix.arch == 'x64' && 'macos-15-intel' || 'macos-latest' }} strategy: fail-fast: false matrix: @@ -300,8 +309,6 @@ jobs: build-linux: name: Linux packages - # RELEASE-BRANCH-ONLY: see the note on build-macos above. - if: false runs-on: ubuntu-latest steps: - name: Checkout code @@ -337,11 +344,10 @@ jobs: publish-release: name: Publish GitHub release runs-on: ubuntu-latest - # RELEASE-BRANCH-ONLY: build-macos / build-linux are disabled for this - # Windows-only release. A skipped job in `needs` skips this one too, so they - # must come out of the list, not just be gated. Restore all three on main. needs: - build-windows + - build-macos + - build-linux if: ${{ (github.event_name == 'push' && github.ref_type == 'tag') || (github.event_name == 'workflow_dispatch' && github.event.inputs.release_tag != '') }} steps: - name: Checkout code @@ -409,23 +415,18 @@ jobs: path: artifacts/windows - name: Download macOS arm64 DMG - continue-on-error: true uses: actions/download-artifact@v4 with: name: openscreen-mac-arm64 path: artifacts/mac-arm64 - name: Download macOS x64 DMG - continue-on-error: true uses: actions/download-artifact@v4 with: name: openscreen-mac-x64 path: artifacts/mac-x64 - name: Download Linux packages - # RELEASE-BRANCH-ONLY: tolerate the missing artifact from the disabled - # Linux job (the macOS downloads below already do). Drop with the rest. - continue-on-error: true uses: actions/download-artifact@v4 with: name: openscreen-linux diff --git a/electron/native/whisper-stt/CMakeLists.txt b/electron/native/whisper-stt/CMakeLists.txt index e5358527e..b95baf7d1 100644 --- a/electron/native/whisper-stt/CMakeLists.txt +++ b/electron/native/whisper-stt/CMakeLists.txt @@ -127,6 +127,21 @@ set(HTTPLIB_USE_OPENSSL_IF_AVAILABLE OFF CACHE BOOL "" FORCE) set(HTTPLIB_USE_ZLIB_IF_AVAILABLE OFF CACHE BOOL "" FORCE) set(HTTPLIB_USE_BROTLI_IF_AVAILABLE OFF CACHE BOOL "" FORCE) +# ponytail: on Linux, ggml/whisper build as separate shared objects and CMake +# bakes the absolute build-tree `bin/` path into each RUNPATH. The build script +# then copies the binary and its libs side by side into +# `electron/native/bin/linux-x64/`, where that absolute path is meaningless the +# moment the build cache is wiped or the app is packaged on another machine. +# Resolve relative to the executable instead. `$ORIGIN/bin` keeps the binary in +# the build tree runnable too (it lands at the build root while the libs go to +# `bin/`). Set before MakeAvailable so the whisper/ggml targets inherit it. +# macOS is left alone: it needs @loader_path rather than $ORIGIN and is not +# testable from here. +if(UNIX AND NOT APPLE) + set(CMAKE_BUILD_WITH_INSTALL_RPATH ON) + set(CMAKE_INSTALL_RPATH "$ORIGIN:$ORIGIN/bin") +endif() + FetchContent_MakeAvailable(whisper httplib json) add_executable(whisper-stt-server src/main.cpp) diff --git a/package.json b/package.json index 6ebb2d7b8..f15735ddf 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "build:native:linux": "node scripts/build-linux-pipewire-helper.mjs", "build:win": "npm run build:native:win && npm run fetch:ffmpeg && npm run build:native:compositor && tsc && vite build && electron-builder --win --config.npmRebuild=false", "build:win:store": "npm run build:native:win && npm run fetch:ffmpeg && npm run build:native:compositor && tsc && vite build && electron-builder --win appx --config.npmRebuild=false", - "build:linux": "npm run build:native:linux && npm run build:native:compositor:linux && tsc && vite build && electron-builder --linux AppImage deb pacman --config.npmRebuild=false", + "build:linux": "npm run fetch:ffmpeg:sdk && npm run build:native:linux && npm run build:native:compositor:linux && tsc && vite build && electron-builder --linux AppImage deb pacman --config.npmRebuild=false", "build:whisper-binaries": "bash scripts/build-whisper-stt.sh", "test:whisper-stt": "node scripts/test-whisper-stt.mjs", "test": "vitest --run", @@ -66,7 +66,8 @@ "test:e2e:windows-native-checklist": "playwright test tests/e2e/windows-native-checklist.spec.ts", "prepare": "husky", "fetch:ffmpeg": "node scripts/fetch-ffmpeg.mjs", - "fetch:ffmpeg:mac": "node scripts/fetch-ffmpeg-macos.mjs" + "fetch:ffmpeg:mac": "node scripts/fetch-ffmpeg-macos.mjs", + "fetch:ffmpeg:sdk": "node scripts/fetch-ffmpeg.mjs --sdk-only" }, "dependencies": { "@fix-webm-duration/fix": "^1.0.1", diff --git a/scripts/fetch-ffmpeg-macos.mjs b/scripts/fetch-ffmpeg-macos.mjs index c3f946812..14330ac24 100644 --- a/scripts/fetch-ffmpeg-macos.mjs +++ b/scripts/fetch-ffmpeg-macos.mjs @@ -93,6 +93,19 @@ run( "--disable-static", "--disable-doc", "--disable-debug", + // ffmpeg's configure AUTO-DETECTS these from whatever the build machine happens + // to have installed, and a GitHub macOS runner has a large Homebrew prefix. That + // is how libavformat came to link /opt/homebrew/opt/libx11/lib/libX11.6.dylib and + // trip build-macos-compositor-addon.mjs's "no absolute build-machine paths" check, + // which is the check doing its job: such a dylib is unresolvable on a user's Mac. + // Disabled explicitly rather than left to chance, so the vendored tree depends on + // the source tarball and the SDK, never on the runner's incidental packages. + // None is used here: xlib/libxcb are X11 capture, sdl2 is only ffplay, and lzma + // only reaches decoders we do not ship (we decode mp4/webm). + "--disable-xlib", + "--disable-libxcb", + "--disable-sdl2", + "--disable-lzma", "--enable-videotoolbox", "--enable-audiotoolbox", "--disable-x86asm", diff --git a/scripts/fetch-ffmpeg.mjs b/scripts/fetch-ffmpeg.mjs index 44248e90f..1c42a92ce 100644 --- a/scripts/fetch-ffmpeg.mjs +++ b/scripts/fetch-ffmpeg.mjs @@ -95,11 +95,22 @@ const PINNED = { /** * The "-shared" sibling of PINNED, from the *same* release tag and source - * commit (n8.1.2-32-gcfa62de001) — same ffmpeg, just built with DLLs instead - * of static linking. Only the compositor addon needs this, and it's - * Windows-only (D3D11), so there's no linux/darwin entry here. + * commit (n8.1.2-32-gcfa62de001) — same ffmpeg, just built with shared libraries + * instead of static linking. + * + * Two consumers now: the Windows D3D11 compositor addon, and the Linux + * pipewire-capture helper, whose build.rs links against + * `crates/thirdparty/ffmpeg-linux64-lgpl-shared`. Nothing provisioned that tree, + * so `npm run build:linux` failed in cargo with "vendored ffmpeg headers are + * missing" — invisible for months because the Linux release job was disabled. + * darwin has no entry because BtbN publishes no macOS build; that path compiles + * ffmpeg from source (scripts/fetch-ffmpeg-macos.mjs). */ const SHARED_PINNED = { + "linux-x64": { + asset: "ffmpeg-n8.1.2-32-gcfa62de001-linux64-lgpl-shared-8.1.tar.xz", + sha256: "74ef679aa7e4f8cdbd5193da3d99bf220a679f64d35daf078397081b789f150e", + }, "win32-x64": { asset: "ffmpeg-n8.1.2-32-gcfa62de001-win64-lgpl-shared-8.1.zip", sha256: "23429f940316ea92e376f6946c0a1f1b9043c930f3bc068228461d65ae24f8b8", @@ -145,14 +156,19 @@ function run(cmd, args, opts = {}) { * (electron/media/ffmpegCapabilities.ts) went with the web export pipeline, and * nothing in the app spawns ffmpeg any more. */ -function assertLgpl(exePath) { +function assertLgpl(exePath, extraEnv) { const problems = []; + // A *shared* build's ffmpeg cannot resolve its own libav*.so without being told + // where they are, and a binary that fails to start prints nothing — which this + // function would read as "unrecognised licence" and refuse to vendor. The caller + // passes LD_LIBRARY_PATH for those; static builds pass nothing. + const opts = extraEnv ? { env: { ...process.env, ...extraEnv } } : {}; // `ffmpeg -L` prints the licence TEXT. This is the authoritative statement: // an LGPL build says "GNU Lesser General Public License", a GPL one says // "GNU General Public License". Note there is NO "License:" line in // `-version` — only `configuration:`. - const license = run(exePath, ["-hide_banner", "-L"]).stdout ?? ""; + const license = run(exePath, ["-hide_banner", "-L"], opts).stdout ?? ""; if (!/Lesser General Public License/i.test(license)) { const what = /General Public License/i.test(license) ? "GPL" : "unrecognised licence"; problems.push(`-L reports ${what}, not LGPL`); @@ -162,8 +178,8 @@ function assertLgpl(exePath) { // one `configuration:` line. Read both so a build that answers only one still // gets checked. const conf = - (run(exePath, ["-hide_banner", "-buildconf"]).stdout ?? "") + - (run(exePath, ["-hide_banner", "-version"]).stdout ?? ""); + (run(exePath, ["-hide_banner", "-buildconf"], opts).stdout ?? "") + + (run(exePath, ["-hide_banner", "-version"], opts).stdout ?? ""); for (const flag of ["--enable-gpl", "--enable-nonfree"]) { if (new RegExp(`(^|\\s)${flag}(\\s|$)`, "m").test(conf)) problems.push(`configured with ${flag}`); @@ -174,7 +190,11 @@ function assertLgpl(exePath) { // Belt and braces: whatever the flags claim, the binary must not actually // expose a GPL encoder. - const encoders = run(exePath, ["-hide_banner", "-encoders"]).stdout ?? ""; + // `opts` here too: without it a SHARED build cannot resolve its own libav*.so, + // prints nothing, and this check passes on an empty string — silently vouching + // for exactly the binaries it exists to reject. Measured: 0 bytes without the + // env, 229 encoders with it. + const encoders = run(exePath, ["-hide_banner", "-encoders"], opts).stdout ?? ""; for (const lib of ["libx264", "libx265"]) { if (new RegExp(`\\s${lib}\\s`).test(encoders)) problems.push(`exposes the ${lib} encoder`); } @@ -186,7 +206,7 @@ function assertLgpl(exePath) { "Bundling it would relicense OpenScreen under the GPL.", ); } - const ver = run(exePath, ["-hide_banner", "-version"]).stdout ?? ""; + const ver = run(exePath, ["-hide_banner", "-version"], opts).stdout ?? ""; return ver.split("\n")[0]?.trim() ?? ""; } @@ -248,6 +268,13 @@ function findExe(dir, name) { */ function ffmpegSdkDest() { const cratesDir = path.join(ROOT, "crates"); + // Linux is not in crates/.cargo/config.toml: its consumer is + // electron/native/pipewire-capture, whose build.rs defaults to this exact path + // (and honours FFMPEG_DIR when set). Mirror that default rather than adding a + // second place the two could disagree. + if (process.platform === "linux") { + return path.join(cratesDir, "thirdparty", "ffmpeg-linux64-lgpl-shared"); + } const configPath = path.join(cratesDir, ".cargo", "config.toml"); if (!fs.existsSync(configPath)) return null; // FFMPEG_DIR is declared `relative = true`, i.e. relative to crates/. @@ -280,7 +307,14 @@ function vendorFfmpegSdk(tmp, dest) { // target, and we only get here on --force or when it is genuinely absent. fs.rmSync(dest, { recursive: true, force: true }); fs.mkdirSync(path.dirname(dest), { recursive: true }); - fs.cpSync(root, dest, { recursive: true }); + // `verbatimSymlinks` matters on Linux, where BtbN ships lib/libavcodec.so -> + // libavcodec.so.62.28.102. WITHOUT it, cpSync RESOLVES each link and writes an + // absolute one pointing back into the extraction temp dir — which this function's + // caller deletes immediately after, leaving every dev symlink dangling. The + // headers then satisfy build.rs's assert while `-lavcodec` fails at link time, + // which is exactly how this presented. Windows has no symlinks here, so the flag + // is a no-op there. + fs.cpSync(root, dest, { recursive: true, verbatimSymlinks: true }); console.log(`Vendored ffmpeg SDK (include/ + lib/) -> ${dest}`); } @@ -295,14 +329,22 @@ function findDirContaining(dir, name) { return null; } -/** All `*.dll` files anywhere under `dir` (BtbN's shared builds nest a `bin/` under a versioned dir). */ -function findDlls(dir) { +/** What a shared ffmpeg library is called on this platform: `avcodec-62.dll` vs + * `libavcodec.so.62`. Both are what the app loads at runtime. */ +function isSharedLib(name) { + const n = name.toLowerCase(); + return process.platform === "win32" ? n.endsWith(".dll") : /\.so(\.\d+)*$/.test(n); +} + +/** Every shared ffmpeg library anywhere under `dir` (BtbN nests a `bin/` — Windows — + * or a `lib/` — Linux — under one versioned dir). */ +function findSharedLibs(dir) { const out = []; for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { const p = path.join(dir, entry.name); if (entry.isDirectory()) { - out.push(...findDlls(p)); - } else if (entry.name.toLowerCase().endsWith(".dll")) { + out.push(...findSharedLibs(p)); + } else if (isSharedLib(entry.name)) { out.push(p); } } @@ -349,46 +391,74 @@ async function fetchSharedDlls(tag, binDir) { // probe for any previously vendored DLL by name; re-download is driven by // --force same as the static exe, checked once we know what we'd extract. - const alreadyVendored = fs - .readdirSync(binDir, { withFileTypes: true }) - .some( - (e) => - e.isFile() && - e.name.toLowerCase().endsWith(".dll") && - e.name.toLowerCase().startsWith("av"), - ); + const alreadyVendored = + process.platform === "win32" && + fs + .readdirSync(binDir, { withFileTypes: true }) + .some((e) => e.isFile() && isSharedLib(e.name) && /^(lib)?av/i.test(e.name)); // The build-time SDK comes out of this same archive, so a tree that has the // DLLs but not the SDK must still re-download — otherwise we skip here and // the compositor build fails afterwards on the missing FFMPEG_DIR. const sdkDest = ffmpegSdkDest(); const sdkPresent = sdkDest == null || fs.existsSync(sdkDest); if (alreadyVendored && sdkPresent && !process.argv.includes("--force")) { - console.log(`\nShared ffmpeg DLLs already present in ${binDir}. Use --force to re-vendor.`); + console.log( + `\nShared ffmpeg libraries already present in ${binDir}. Use --force to re-vendor.`, + ); return; } - console.log(`\nFetching shared ffmpeg DLLs for the compositor addon (${tag})...`); + console.log(`\nFetching shared ffmpeg libraries (${tag})...`); const tmp = await downloadAndExtract(spec); try { - const exe = findExe(tmp, "ffmpeg.exe"); + const exeName = process.platform === "win32" ? "ffmpeg.exe" : "ffmpeg"; + const exe = findExe(tmp, exeName); if (!exe) - throw new Error(`ffmpeg.exe not found inside ${spec.asset} (needed to verify licence)`); + throw new Error(`${exeName} not found inside ${spec.asset} (needed to verify licence)`); + if (process.platform !== "win32") fs.chmodSync(exe, 0o755); // Same source commit as the static build, but configure flags are a // separate BtbN job — verify this artifact's licence independently // rather than assuming it matches. console.log("Verifying licence (shared build)..."); - const banner = assertLgpl(exe); + // BtbN lays the tree out as /bin/ffmpeg + /lib/*.so. + const sharedEnv = + process.platform === "win32" + ? undefined + : { LD_LIBRARY_PATH: path.join(path.dirname(exe), "..", "lib") }; + const banner = assertLgpl(exe, sharedEnv); console.log(banner); - const dlls = findDlls(tmp); - if (dlls.length === 0) throw new Error(`No .dll files found inside ${spec.asset}`); - - fs.mkdirSync(binDir, { recursive: true }); - for (const dll of dlls) { - fs.copyFileSync(dll, path.join(binDir, path.basename(dll))); + const libs = findSharedLibs(tmp); + if (libs.length === 0) throw new Error(`No shared ffmpeg libraries found inside ${spec.asset}`); + + // Windows only. There, the loader finds a DLL next to the .exe, so the runtime + // copies belong in binDir. On Linux that same directory is owned by the two + // native build scripts: build-linux-compositor-addon.mjs puts SYMBOL-RENAMED + // (osff_*) copies there so the addon cannot bind to Chromium's ffmpeg, and + // build-linux-pipewire-helper.mjs stages unrenamed ones in `binDir/ffmpeg/` + // for the helper's `$ORIGIN/ffmpeg` RUNPATH. Dropping a third, unrenamed set + // in binDir would overwrite the renamed ones under identical filenames and + // break the addon at load time. Linux takes the SDK below and nothing else. + if (process.platform !== "win32") { + console.log(`Skipping runtime copies into ${binDir} (owned by the native build scripts).`); + } else { + fs.mkdirSync(binDir, { recursive: true }); + for (const lib of libs) { + // Linux ships symlink chains (libavcodec.so -> .so.62 -> .so.62.x). Follow + // them: copyFileSync would dereference into three identical large files, and + // a dangling link would break the loader outright. + const dest = path.join(binDir, path.basename(lib)); + const st = fs.lstatSync(lib); + if (st.isSymbolicLink()) { + fs.rmSync(dest, { force: true }); + fs.symlinkSync(fs.readlinkSync(lib), dest); + } else { + fs.copyFileSync(lib, dest); + } + } + console.log(`Vendored ${libs.length} shared librar(ies) -> ${binDir}`); } - console.log(`Vendored ${dlls.length} DLL(s) -> ${binDir}`); if (sdkDest) vendorFfmpegSdk(tmp, sdkDest); console.log("LGPL verified: safe to ship with an MIT app."); } finally { @@ -417,6 +487,19 @@ async function main() { const binDir = path.join(ROOT, "electron", "native", "bin", tag); const dest = path.join(binDir, spec.exe); + // `--sdk-only` skips the standalone ffmpeg CLI and vendors just the build-time + // SDK. Linux needs it: the CLI lands at `/ffmpeg` as a FILE, while + // build-linux-pipewire-helper.mjs stages the helper's libraries into + // `/ffmpeg/` as a DIRECTORY — the name its `$ORIGIN/ffmpeg` RUNPATH is + // compiled against. One clobbers the other (`EEXIST: mkdir .../linux-x64/ffmpeg`). + // Nothing in the app spawns the CLI any more (see assertLgpl's note), and v1.7.0 + // shipped Linux packages without it, so on Linux it is dead weight AND a conflict. + if (process.argv.includes("--sdk-only")) { + console.log(`Skipping the standalone ffmpeg CLI (--sdk-only).`); + await fetchSharedDlls(tag, binDir); + return; + } + if (fs.existsSync(dest) && !process.argv.includes("--force")) { console.log(`Already present: ${dest}`); console.log(assertLgpl(dest)); @@ -446,9 +529,10 @@ async function main() { } } - if (process.platform === "win32") { - await fetchSharedDlls(tag, binDir); - } + // No platform guard: `fetchSharedDlls` returns early when SHARED_PINNED has no + // entry for this tag, so it self-gates. The guard that used to be here predated + // the Linux pin and silently skipped the pipewire helper's build-time SDK. + await fetchSharedDlls(tag, binDir); } main().catch((err) => {