benchmark.yml

name: Benchmark runner

on:
  schedule:
    - cron: '30 2 * * 0'
  workflow_dispatch:

# Read-only by default, so the jobs that only measure cannot write to the
# repository whatever the repository's own default happens to be. The one job
# that publishes raises this for itself.
permissions:
  contents: read

jobs:
  # Fills pnpr's storage with everything the fixtures need, once, for the
  # three sample jobs to share.
  #
  # The warm-up is the slowest part of a run — some 5.4k packages and ~3 GB
  # for the current fixtures — and it is identical work in every job. Three
  # jobs doing it separately is three times the load on npmjs and bit for one
  # answer, and worse, it is three *different* answers whenever a package
  # publishes while the run is in flight: each job resolves its own view of
  # the graph and the three stop being samples of the same thing.
  #
  # Shared through the run's own cache key rather than an artifact: the entry
  # is written once and read three times within this run and never reused by
  # the next one, which is deliberate. A cache that survived the week would
  # freeze every packument in it, and the benchmark exists to measure the
  # latest of everything.
  populate-registry-cache:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          persist-credentials: false
      - uses: pnpm/setup@v2
      # No `GITHUB_TOKEN` here, unlike the measuring jobs: the only thing
      # that needs one is provisioning Yarn, whose binary comes from a
      # GitHub release, and this job provisions pnpm alone.
      - run: pnpm run populate-cache
      - uses: actions/cache/save@v4
        with:
          path: .pnpr-cache
          key: pnpr-storage-${{ github.run_id }}

  # The samples this run contributes, measured in parallel rather than one
  # after another. Three measurements of the same thing are what the published
  # number is a `min()` of, and they are independent of each other: each makes
  # its own temp directory, provisions its own package managers, and starts its
  # own registry on a port the system says is free. Run sequentially in one job
  # they took three times as long as a single measurement for no benefit.
  #
  # Three samples rather than one because the page reports the minimum per
  # package manager *per version*. Cutting to one sample wouldn't only add
  # noise, it would bias: a version released this week would be a min-of-one
  # while a long-lived one keeps its min-of-many, which reads as a regression
  # in whatever was released most recently.
  benchmark:
    needs: populate-registry-cache
    strategy:
      # One machine having a bad day shouldn't throw away the samples the
      # others already measured. The reporting job publishes whatever arrived.
      fail-fast: false
      matrix:
        sample: [1, 2, 3]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token
          fetch-depth: 0
      # The harness's own pnpm and the Node.js it runs on, both at the
      # versions package.json pins (`devEngines.packageManager` and
      # `devEngines.runtime`). The step also runs `pnpm install` for the
      # harness.
      #
      # The pnpm pin is an exact version, not a range, because this step
      # installs whatever the pin resolves to and then refuses to continue
      # unless `pnpm --version` reports that same version back — and it asks
      # inside this checkout, where pnpm hands off to the version
      # `pnpm-lock.yaml` pins under `packageManagerDependencies`. A range
      # resolves to each new release the day it lands while the lockfile still
      # pins the last one, and the two disagreeing is what fails this step. An
      # exact pin moves only when the lockfile moves with it.
      - uses: pnpm/setup@v2
      # fnm is compared against pnpm in the Node.js version management section.
      - name: Install fnm
        run: |
          curl -fsSL https://github.com/Schniz/fnm/releases/latest/download/fnm-linux.zip -o /tmp/fnm.zip
          unzip -o /tmp/fnm.zip -d /tmp/fnm
          mkdir -p "$HOME/.local/bin"
          install -m 755 /tmp/fnm/fnm "$HOME/.local/bin/fnm"
          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
          "$HOME/.local/bin/fnm" --version
      # The token authenticates pnpm's Yarn 6 release-list request against
      # GitHub's API, whose anonymous rate limit is shared across every job on
      # the runner's IP.
      # A miss here is not a failure: `BENCHMARK_PNPR_CACHE` naming a
      # directory that isn't there makes the run warm the registry itself,
      # the way it did before this was split out. It costs the job time, not
      # its numbers.
      - uses: actions/cache/restore@v4
        with:
          path: .pnpr-cache
          key: pnpr-storage-${{ github.run_id }}
      - run: pnpm run benchmark
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          BENCHMARK_PNPR_CACHE: ${{ github.workspace }}/.pnpr-cache
      # Only uploaded when the measurement finished. A run that died partway
      # recorded some scenarios and not others, and those are exactly the
      # numbers not to publish.
      - uses: actions/upload-artifact@v4
        with:
          name: samples-${{ matrix.sample }}
          path: |
            results
            versions.json
          retention-days: 3

  # Writes `benchmarks.json` from the samples the jobs above recorded. It
  # measures nothing itself — no package manager is installed here and no
  # registry is started — so it can't quietly substitute a number for one a
  # measuring run failed to record, and it costs seconds rather than minutes.
  report:
    needs: benchmark
    # `needs` alone would skip this job if any one sample failed, throwing away
    # the two that succeeded.
    if: ${{ !cancelled() }}
    runs-on: ubuntu-latest
    # The job commits the results it publishes back to the branch, which the
    # default read-only token can't do.
    permissions:
      contents: write
    steps:
      - uses: actions/checkout@v4
        with:
          persist-credentials: false
          fetch-depth: 0
      - uses: pnpm/setup@v2
      # Fails when no sample arrived at all, which is the right outcome: with
      # nothing measured this week, the alternative is republishing last week's
      # numbers under today's date.
      # Downloaded outside the working tree. The publish step below commits
      # whatever it finds there, and an artifact unpacked next to the results
      # is 3000 files it will happily commit alongside them.
      - uses: actions/download-artifact@v4
        with:
          pattern: samples-*
          path: ${{ runner.temp }}/samples
      - run: pnpm run merge-results "$RUNNER_TEMP/samples"
      - run: pnpm run report
      - name: Commit & Push changes
        # Pinned: this step is handed a token that can write to the repo, so
        # what runs in it shouldn't be able to change without a commit here.
        uses: actions-js/push@968f4695ca558093eadb24ad83cc5891f47e0cdc # v1.6
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          message: "chore: update benchmarks"
          branch: main