Run browser tests only in Playwright

.github/workflows/CI.yml

@@ -176,106 +176,12 @@ jobs:
         shell: bash
         run: node scripts/preset-check.mjs --core packages/core${{ matrix.core == 'mt' && '-mt' || '' }} --preset ${{ matrix.preset }} | tee -a "$GITHUB_STEP_SUMMARY"
 
-  tests:
-    runs-on: ubuntu-latest
-    # A core stuck in a native loop never yields to the browser, so neither
-    # mocha's per-test timeout nor Ctrl-C can end the run: the page just
-    # stops answering and the job would sit until GitHub's 6 hour default.
-    # 20 minutes is well past the ~2 this job takes when green.
-    timeout-minutes: 20
-    needs:
-      - js
-      - build-core
-      - build-core-mt
-    steps:
-      - name: Checkout Source Code
-        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-      - name: Download ffmpeg-core
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          name: ffmpeg-core
-          path: packages/core/dist
-      - name: Download ffmpeg-core-mt
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          name: ffmpeg-core-mt
-          path: packages/core-mt/dist
-      - name: Setup pnpm
-        uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
-      - name: Setup Node.js
-        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
-        with:
-          node-version: 22
-          cache: pnpm
-      - name: Install dependencies
-        run: pnpm install --frozen-lockfile
-      - name: Build packages
-        run: pnpm build
-      - name: Install Chrome
-        id: chrome
-        uses: browser-actions/setup-chrome@48ad923757ca74d66703209fe939badbdf80f2f4 # v2.2.0
-        with:
-          chrome-version: stable
-      - name: Run tests
-        env:
-          CHROME_HEADLESS: 1
-          PUPPETEER_EXECUTABLE_PATH: ${{ steps.chrome.outputs.chrome-path }}
-          CHROME_FLAGS: "--headless --disable-gpu --no-sandbox --enable-features=SharedArrayBuffer,CrossOriginIsolation"
-        run: |
-          pnpm run serve &
-          sleep 5
-          curl -sfI http://localhost:3000/tests/ffmpeg-st.test.html | grep -i "cross-origin-embedder-policy: require-corp"
-
-          # One Chrome per page, all at once. Each page loads a whole core and
-          # runs it, so back to back they spend most of the job with three
-          # cores idle. Output is captured per page because concurrent
-          # browsers interleave on stdout, then replayed in a fixed order so
-          # the log still reads the same as a sequential run.
-          pids=""
-          for page in ffmpeg-core-st ffmpeg-st ffmpeg-mt ffmpeg-esm-fallback; do
-            npx mocha-headless-chrome --args="$CHROME_FLAGS" -a no-sandbox \
-              -f "http://localhost:3000/tests/$page.test.html" > "$page.log" 2>&1 &
-            pids="$pids $!:$page"
-          done
-          failed=""
-          for entry in $pids; do
-            page="${entry#*:}"
-            if wait "${entry%%:*}"; then status=passed; else status=FAILED; failed="$failed $page"; fi
-            echo "::group::$page ($status)"
-            cat "$page.log"
-            echo "::endgroup::"
-          done
-          if [ -n "$failed" ]; then
-            echo "::error::failed test pages:$failed"
-            # mocha-headless-chrome reports only pass/fail, so a page that
-            # wedges or fails inside a Web Worker is otherwise a black box.
-            # Re-open each one and dump its console for the log. One at a time
-            # would add a minute per failed page to a job that is already
-            # red, so they all run at once and each writes to its own file,
-            # replayed in a fixed order below for the same reason as above.
-            dump_pids=""
-            for page in $failed; do
-              timeout 100 node scripts/dump-test-page.mjs \
-                "http://localhost:3000/tests/$page.test.html" \
-                > "$page.dump.log" 2>&1 &
-              dump_pids="$dump_pids $!:$page"
-            done
-            for entry in $dump_pids; do
-              page="${entry#*:}"
-              wait "${entry%%:*}" || true
-              echo "::group::page output: $page"
-              cat "$page.dump.log"
-              echo "::endgroup::"
-            done
-            exit 1
-          fi
-
-  # Same Mocha pages as `tests`, in Chromium, Firefox and WebKit. plain pages
-  # have no COOP/COEP, so the st core runs without SharedArrayBuffer. Not a
-  # required check and never fails the workflow until the results are green.
+  # The Mocha pages in Chromium, Firefox and WebKit. plain pages have no
+  # COOP/COEP, so the st core runs without SharedArrayBuffer. Gated by `tests`.
+  # A core stuck in a native loop never yields to the browser, so the timeout
+  # stops the job from sitting until GitHub's 6 hour default.
   tests-browsers:
     runs-on: ubuntu-latest
-    continue-on-error: true
     timeout-minutes: 20
     strategy:
       fail-fast: false
@@ -311,13 +217,28 @@ jobs:
       - name: Build packages
         run: pnpm build
       - name: Install ${{ matrix.browser }}
-        run: pnpm exec playwright install --with-deps ${{ matrix.browser }}
+        # apt mirrors sometimes stall; retry instead of hitting the job timeout.
+        run: |
+          for i in 1 2 3; do
+            timeout 300 pnpm exec playwright install --with-deps ${{ matrix.browser }} && exit 0
+          done
+          exit 1
       - name: Run tests
         run: pnpm run test:browser:playwright ${{ matrix.browser }} ${{ matrix.mode }}
 
+  # The required check name; a matrix job cannot be required as a whole.
+  tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    needs: tests-browsers
+    if: ${{ always() }}
+    steps:
+      - name: Require every browser job to pass
+        run: test "${{ needs.tests-browsers.result }}" = success
+
   node-tests:
     runs-on: ubuntu-latest
-    # Same reason as the tests job: a core spinning natively cannot be
+    # Same reason as tests-browsers: a core spinning natively cannot be
     # interrupted, and mocha --exit only helps when it gets control back.
     timeout-minutes: 20
     needs:
@@ -428,7 +349,12 @@ jobs:
       - name: Build packages
         run: pnpm build
       - name: Install ${{ matrix.browser }}
-        run: pnpm exec playwright install --with-deps ${{ matrix.browser }}
+        # apt mirrors sometimes stall; retry instead of hitting the job timeout.
+        run: |
+          for i in 1 2 3; do
+            timeout 300 pnpm exec playwright install --with-deps ${{ matrix.browser }} && exit 0
+          done
+          exit 1
       - name: Run tests
         run: pnpm run test:browser:playwright ${{ matrix.browser }} jspi
 
@@ -599,7 +525,13 @@ jobs:
       - name: Build packages
         run: pnpm build
       - name: Install native FFmpeg
-        run: sudo apt-get update && sudo apt-get install -y ffmpeg time
+        # Retried for the same apt mirror stalls as the browser installs.
+        run: |
+          sudo apt-get update
+          for i in 1 2 3; do
+            sudo timeout 300 apt-get install -y --no-install-recommends ffmpeg time && exit 0
+          done
+          exit 1
       - name: Record native FFmpeg version
         run: ffmpeg -version | head -1
       - name: Benchmark native FFmpeg

AGENTS.md

@@ -69,7 +69,9 @@ Lessons from the shim, so they are not rediscovered:
   configure flags and linked libraries; `make prd PRESET=<name>`. The
   published packages always use `full`. See `apps/website/docs/presets.md`.
 - `apps/`: standalone examples, not part of the pnpm workspace.
-- `tests/`: browser and Node tests run against built cores. The helper tests
+- `tests/`: browser and Node tests run against built cores. Browser pages run
+  in Playwright with `pnpm run test:browser:playwright <chromium|firefox|webkit>
+  <isolated|plain|jspi>`. The helper tests
   that need no core (`ffmpeg-helpers-args.test.mjs`) run anywhere with `pnpm
   run test:node:helpers:args`.
 - `scripts/fate`, `scripts/bench`: run FFmpeg's FATE tests and benchmarks against

package.json

@@ -8,15 +8,7 @@
     "lint:root": "eslint tests",
     "check:doc-versions": "node scripts/check-doc-versions.mjs",
     "build": "pnpm --if-present -r run build",
-    "pretest": "pnpm run build",
     "serve": "node scripts/serve.mjs 3000",
-    "test": "server-test test:browser:server 3000 test:all",
-    "test:all": "npm-run-all test:browser:*:*",
-    "test:browser": "mocha-headless-chrome -a enable-features=SharedArrayBuffer",
-    "test:browser:core:st": "pnpm run test:browser -f http://localhost:3000/tests/ffmpeg-core-st.test.html",
-    "test:browser:ffmpeg:mt": "pnpm run test:browser -f http://localhost:3000/tests/ffmpeg-mt.test.html",
-    "test:browser:ffmpeg:st": "pnpm run test:browser -f http://localhost:3000/tests/ffmpeg-st.test.html",
-    "test:browser:server": "pnpm run serve",
     "test:browser:playwright": "node scripts/test-browser.mjs",
     "test:node": "mocha --exit --bail -t 60000",
     "test:node:core:mt": "pnpm run test:node --require tests/test-helper-mt.js tests/ffmpeg-core.test.js",
@@ -40,10 +32,6 @@
     "eslint": "^10.12.0",
     "globals": "^17.13.0",
     "mocha": "^12.0.3",
-    "mocha-headless-chrome": "^5.1.0",
-    "npm-run-all": "^4.1.5",
-    "playwright": "^1.63.0",
-    "puppeteer-core": "^24.43.1",
-    "start-server-and-test": "^3.0.12"
+    "playwright": "^1.63.0"
   }
 }