diff --git a/.github/workflows/plugin-ci.yml b/.github/workflows/plugin-ci.yml index 26f482a80..1d16ae3b0 100644 --- a/.github/workflows/plugin-ci.yml +++ b/.github/workflows/plugin-ci.yml @@ -139,11 +139,42 @@ jobs: with: path: plugin + # RESOLVE THE SIBLING REF BEFORE CHECKING IT OUT, because the ref every caller passes is + # frequently one that does not exist in busbar, and the resulting failure is unreadable. + # + # THE BUG THIS FIXES, and it is the reason several plugin repos have never merged a PR. Most + # callers pass `busbar_ref: ${{ github.ref_name }}`, meaning "test against the same-named + # busbar branch" - dev against dev, main against main. On a `pull_request` event + # `github.ref_name` is not a branch name at all: it is `/merge`. So EVERY pull request + # to those repos asked busbar for a branch called `5/merge`, and `actions/checkout` retried + # and died with "The process '/usr/bin/git' failed with exit code 1" and no explanation of + # which ref it wanted or why. The check has been red on every PR, for a reason the log does + # not state, which is indistinguishable from the repo's tests being broken. That is enough to + # stop a review culture forming. + # + # The same hole swallows feature-branch pushes: `ci/whatever` exists in the plugin repo and + # not in busbar. Fixed once here rather than in ten callers, which is what this file is for. + # A caller passing a REAL ref (a sha, `main`, `dev`) is unaffected; only unresolvable refs + # fall back, and the fallback announces itself. + - name: Resolve which busbar ref to check out + id: busbarref + run: | + set -euo pipefail + want="${{ inputs.busbar_ref }}" + if git ls-remote --exit-code --heads --tags https://github.com/GetBusbar/busbar.git "$want" >/dev/null 2>&1 \ + || printf '%s' "$want" | grep -qE '^[0-9a-f]{40}$'; then + echo "ref=$want" >> "$GITHUB_OUTPUT" + echo "Building against busbar@${want}." + else + echo "ref=dev" >> "$GITHUB_OUTPUT" + echo "::warning::busbar has no ref '${want}', so this build used busbar@dev instead. On a pull_request, \`github.ref_name\` is '/merge' rather than a branch name, which is the usual cause - pass \`busbar_ref: \${{ github.base_ref || github.ref_name }}\` from the caller to say what you meant. Falling back is deliberate: an unresolvable sibling ref used to fail the checkout with an unreadable git error and no mention of the ref, which read like the plugin's own tests were broken." + fi + - name: Checkout busbar (sibling path dependency) uses: actions/checkout@v7 with: repository: GetBusbar/busbar - ref: ${{ inputs.busbar_ref }} + ref: ${{ steps.busbarref.outputs.ref }} path: busbarAI - uses: dtolnay/rust-toolchain@stable diff --git a/.github/workflows/plugin-consumer-verify.yml b/.github/workflows/plugin-consumer-verify.yml new file mode 100644 index 000000000..6d6a5b7cb --- /dev/null +++ b/.github/workflows/plugin-consumer-verify.yml @@ -0,0 +1,552 @@ +name: plugin-consumer-verify + +# plugin-consumer-verify - the ONE consumer-side check every first-party plugin repo calls, the way +# they all already call plugin-ci.yml. It answers exactly one question per repo: +# +# DOES THE THING WE PUBLISHED ACTUALLY WORK WHEN A USER GETS IT? +# +# WHY THE FLEET NEEDED THIS. Not one plugin repo verified what it published. Every repo's release +# workflow ends with an assertion about ITS OWN upload step, from inside the run that did the +# uploading, which proves the run believed it succeeded and nothing else. Two failures came straight +# out of that gap: +# +# * headroom-hook's SHIPPED docker/bundle/config.yaml carries config shapes busbar 1.5.3 retired +# (`auth.admin_auth:` with INLINE module entries, which moved under `identity-providers:`). The +# published bundle therefore CANNOT BOOT: busbar exits 1 with "this looks like a busbar 1.x +# config" before it ever binds a port. The image built fine, pushed fine, and every workflow +# involved was green, because "it built" and "it runs" are different claims and only the first +# one was ever checked. +# * webrequest-hook v1.0.4 published as a zero-asset phantom. The tag exists, the Release object +# exists, and there is nothing in it to download. +# +# Both are the same shape and neither needs a clever check to catch. It needs SOMEBODY TO ACTUALLY +# FETCH THE PUBLISHED THING AND USE IT, from outside, after publication. +# +# THIS IS DELIBERATELY SMALLER THAN CORE'S verify-deploy.yml. A plugin has no install.sh, no +# Homebrew tap, no helm chart, no marketing site. It has a Release with tarballs in it, and +# sometimes a container image. So the whole check is: the Release is real and public, every platform +# archive it owes is present and downloadable through the moving `latest` pointer, one of those +# archives unpacks into a plugin busbar would actually accept, and - where the repo ships a runnable +# bundle - the published image BOOTS AND SERVES rather than merely existing. +# +# HOW TO ADOPT IT, in the calling repo's .github/workflows/consumer-verify.yml: +# +# name: consumer-verify +# on: +# release: { types: [published] } +# schedule: [{ cron: "41 9 * * *" }] +# workflow_dispatch: { inputs: { version: { required: false, type: string } } } +# permissions: { contents: read, issues: write, actions: read } +# jobs: +# verify: +# uses: GetBusbar/busbar/.github/workflows/plugin-consumer-verify.yml@main +# with: +# asset_prefix: busbar-store-postgres +# plugin_name: busbar-store-postgres-plugin +# plugin_alias: postgres +# plugin_kind: store +# +# and, in the repo's OWN release.yml, as the final job so a broken publish turns the RELEASE red: +# +# consumer-verification: +# needs: [verify-assets] +# if: ${{ !cancelled() }} +# uses: GetBusbar/busbar/.github/workflows/plugin-consumer-verify.yml@main +# with: { version: ${{ github.ref_name }}, asset_prefix: ..., ... } +# permissions: { contents: read, issues: write, actions: read } +# secrets: inherit + +on: + workflow_call: + inputs: + asset_prefix: + description: >- + The release asset basename before the version, i.e. the pack --out prefix. The published + asset is --.tar.gz. NOTE it is not always the same as + plugin_name: the store repos drop the trailing -plugin from the filename and the auth + repos keep it, so this is an input rather than something derived. + required: true + type: string + plugin_name: + description: "The manifest `name` the tarball must declare (pack --name), e.g. busbar-store-postgres-plugin" + required: true + type: string + plugin_alias: + description: "The manifest `alias` (pack --alias), e.g. postgres" + required: true + type: string + plugin_kind: + description: "store | auth | hook | secret - the manifest `kind` (pack --kind)" + required: true + type: string + version: + description: >- + Version to verify, with or without a leading v. Leave empty and the newest published + release of the CALLING repo is used, which is what the daily schedule wants. + required: false + type: string + default: "" + targets: + description: >- + Space-separated platform triples the release owes an archive for. The default is the + 5-target matrix every plugin repo's release.yml builds. Windows ships .zip in core but + .tar.gz here, because plugin-pack always writes a tarball. + required: false + type: string + default: "x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu x86_64-apple-darwin aarch64-apple-darwin x86_64-pc-windows-msvc" + require_signed: + description: >- + Assert the published manifest carries a signature. Default true, and it should stay true: + every plugin release workflow falls back to --allow-unsigned when BUSBAR_SIGN_KEY is + unset, so an unsigned publish looks identical to a signed one from inside the release run + and is refused by busbar at load time on the user's machine. Set false ONLY with a comment + saying why, so the exemption is a visible decision rather than a silent gap. + required: false + type: boolean + default: true + bundle_image: + description: >- + Docker repository of a runnable bundle this repo publishes, e.g. getbusbar/busbar-headroom. + Empty means the repo ships no bundle and the boot check is declared not-applicable rather + than silently skipped. When set, the image is pulled FRESH, its :latest is checked against + the version pin by manifest digest, and the container must BOOT AND SERVE. + required: false + type: string + default: "" + bundle_env: + description: >- + Space-separated KEY=VALUE pairs passed to `docker run -e` for the bundle boot. The bundle's + shipped config decides what is required; headroom's needs ANTHROPIC_KEY and + BUSBAR_ADMIN_TOKEN. Dummy values are correct here: this asserts BOOT and health, and + deliberately spends no real provider call. + required: false + type: string + default: "" + bundle_health_path: + description: "Path the booted bundle must serve" + required: false + type: string + default: "/healthz" + bundle_health_body: + description: "Exact body the health path must return" + required: false + type: string + default: "ok" + +permissions: + contents: read + issues: write + actions: read + +jobs: + consumer: + name: consumer check (fetch what we published and use it) + runs-on: ubuntu-latest + timeout-minutes: 25 + outputs: + version: ${{ steps.check.outputs.version }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + ASSET_PREFIX: ${{ inputs.asset_prefix }} + PLUGIN_NAME: ${{ inputs.plugin_name }} + PLUGIN_ALIAS: ${{ inputs.plugin_alias }} + PLUGIN_KIND: ${{ inputs.plugin_kind }} + IN_VERSION: ${{ inputs.version }} + TARGETS: ${{ inputs.targets }} + REQUIRE_SIGNED: ${{ inputs.require_signed }} + BUNDLE_IMAGE: ${{ inputs.bundle_image }} + BUNDLE_ENV: ${{ inputs.bundle_env }} + HEALTH_PATH: ${{ inputs.bundle_health_path }} + HEALTH_BODY: ${{ inputs.bundle_health_body }} + steps: + # No actions/checkout, on purpose and for the same reason core's verify-deploy.yml has none: + # this job must not be able to read the repository. Every byte it judges is fetched from the + # published Release or pulled from the registry, because a fix that is committed but never + # published is still broken for every user, and a working copy is exactly how that gets hidden. + - name: Does what we published actually work? + id: check + run: | + # `set +e` FIRST, AND IT IS LOAD-BEARING. GitHub runs every `run:` block as + # `bash -e {0}`, so errexit is ALREADY ON before line 1 and `set -uo pipefail` does not + # turn it off. Without this the job aborts at the first non-zero command, and "the bundle + # does not boot" would hide "the bundle does not boot AND two archives are missing". + set +e + set -uo pipefail + fail=0 + : > /tmp/findings.md + + record() { # record + echo "FAIL: $1 | expected: $2 | observed: $3" + { + echo "- **$1**" + echo " - expected: \`$2\`" + echo " - observed: \`$3\`" + echo " - $4" + } >> /tmp/findings.md + echo "::error::CONSUMER CHECK FAILED: $1 - expected '$2', observed '$3'. $4" + fail=1 + } + declared() { echo "NOT APPLICABLE: $1 - $2"; } + + # -- Which version -------------------------------------------------------------------- + V="${IN_VERSION#v}" + if [ -z "$V" ]; then + # From the RELEASES list, not from /releases/latest, which is one of the pointers this + # job is about to test. Asking a pointer what the answer is and then checking the + # pointer against its own answer is a check that can never fail. + V="$(gh api --paginate "repos/${REPO}/releases" \ + --jq '.[] | select(.draft==false and .prerelease==false) | .tag_name' 2>/dev/null \ + | sed 's/^v//' | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' | sort -V | tail -1)" + fi + if [ -z "${V:-}" ]; then + echo "::error::${REPO} has no published, non-draft release to verify. Either nothing has shipped yet, or - the failure this exists to catch - the release was created as a DRAFT and never promoted, which means it is invisible to every user while looking perfectly fine on the Actions tab." + exit 1 + fi + TAG="v$V" + echo "version=$V" >> "$GITHUB_OUTPUT" + echo "Verifying what ${REPO} published as ${TAG}, from the outside." + + # -- C1: the Release is REAL and PUBLIC ----------------------------------------------- + # A draft Release is the failure mode two repos in the fleet are one bug away from: + # headroom-hook and webrequest-hook both create the Release with --draft and rely on their + # verify-assets job running `gh release edit --draft=false` afterwards. If that job is + # skipped (a `needs:` on a failed job skips its dependent by DEFAULT, which is exactly how + # busbar's own verify-assets got skipped precisely when the release was broken), the tag + # exists, the run may even be green, and users see nothing at all. + meta="$(gh api "repos/${REPO}/releases/tags/${TAG}" 2>/dev/null || true)" + if [ -z "$meta" ]; then + record "GitHub Release ${TAG}" "a published Release" "" \ + "The tag was pushed but no Release object is visible. Fix: check the release workflow's create-release job." + echo "$fail" >/dev/null + else + is_draft="$(printf '%s' "$meta" | jq -r '.draft')" + if [ "$is_draft" = "true" ]; then + record "GitHub Release ${TAG} draft flag" "false (published)" "true (still a DRAFT)" \ + "A draft Release is INVISIBLE to users: nothing downloads, and \`gh release download\` fails. The release workflow creates it with --draft and promotes it in verify-assets, so verify-assets was skipped or failed. Fix: \`gh release edit ${TAG} --repo ${REPO} --draft=false --latest\`, then fix the promote step." + else + echo "PASS: Release ${TAG} exists and is published (not a draft)" + fi + fi + + # -- C2: the moving `latest` pointer resolves to it ----------------------------------- + # Users and tooling that do not pin a version follow /releases/latest. It 404s if the + # newest Release is not flagged latest and silently serves the PREVIOUS release otherwise. + loc="$(curl -fsS --max-time 30 -o /dev/null -w '%{redirect_url}' \ + "https://github.com/${REPO}/releases/latest" || true)" + if [ "${loc##*/releases/tag/}" = "$TAG" ]; then + echo "PASS: github.com/${REPO}/releases/latest -> ${TAG}" + else + record "${REPO} /releases/latest" ".../releases/tag/${TAG}" "${loc:-}" \ + "Anyone fetching this plugin without pinning a version gets a different release than ${TAG}. Fix: flag ${TAG} as latest (\`gh release edit ${TAG} --latest\`)." + fi + + # -- C3: every platform archive is present, sized and DOWNLOADABLE -------------------- + # THE PHANTOM GUARD, and the reason it is not a count. webrequest-hook v1.0.4 published + # with zero assets; a `>= 1` assertion would have caught that one but passes happily for a + # release missing four of five platforms, which is the busbar-1.5.3 shape one level down. + # A count can never see a MISSING PLATFORM. Only a name can. And a name in the asset list + # is still not a usable artifact: GitHub creates the row when the upload STARTS, so a + # truncated upload lists identically to a good one. Assert name, size and real bytes. + listed="$(printf '%s' "$meta" | jq -r '.assets[]? | "\(.name)\t\(.size)"' 2>/dev/null || true)" + if [ -z "$listed" ]; then + record "Release ${TAG} assets" "one archive per platform in: ${TARGETS}" "ZERO assets" \ + "PHANTOM RELEASE: the tag and the Release object exist and there is nothing in them to download. This is webrequest-hook v1.0.4 verbatim. Fix: every leg of the release build matrix failed to upload - check the cdylib build and the sibling busbar checkout pin in .busbar-ref." + fi + first_archive="" + for t in $TARGETS; do + a="${ASSET_PREFIX}-${V}-${t}.tar.gz" + sz="$(printf '%s\n' "$listed" | awk -F'\t' -v n="$a" '$1==n{print $2; exit}')" + if [ -z "${sz:-}" ]; then + record "Release asset ${a}" "present on ${TAG}" "" \ + "Users on ${t} get a 404. Fix: that target's leg of the release build matrix did not upload - it runs with fail-fast:false, so the other platforms succeeding tells you nothing about this one." + continue + fi + # A packed plugin is a compressed cdylib: hundreds of KB at the very least. 50 KiB is far + # below any real one and far above a truncated or header-only upload. + if [ "$sz" -lt 51200 ]; then + record "Release asset ${a} size" ">= 51200 bytes" "${sz} bytes" \ + "A truncated upload: it is listed as present and is useless to anyone who downloads it. Fix: re-upload the asset." + continue + fi + # Through /releases/latest/download/, not the pinned tag URL: that is the version-agnostic + # path, so this proves the moving pointer AND the asset in one request. + u="https://github.com/${REPO}/releases/latest/download/${a}" + code="$(curl -sSL --max-time 60 --range 0-0 -o /dev/null -w '%{http_code}' "$u" || echo 000)" + case "$code" in + 200|206) + echo "PASS: ${a} present (${sz} bytes) and downloadable through /releases/latest/download/" + [ -z "$first_archive" ] && first_archive="$a" + ;; + *) record "/releases/latest/download/${a}" "HTTP 200/206" "HTTP ${code}" \ + "The asset is listed on the Release but downloading it fails, so a user following the documented URL gets nothing. Fix: re-upload it." ;; + esac + done + + # -- C4: the archive is a plugin busbar would actually accept ------------------------- + # An archive that downloads is not an archive that LOADS. busbar refuses a plugin whose + # manifest is absent, whose declared sha256 does not bind the cdylib beside it, or (under + # default trust) that carries no signature - and every one of those is invisible from + # inside the release run that produced it. + if [ -n "$first_archive" ]; then + work="$(mktemp -d)" + if curl -fsSL --max-time 120 \ + "https://github.com/${REPO}/releases/latest/download/${first_archive}" \ + -o "${work}/p.tar.gz" && tar xzf "${work}/p.tar.gz" -C "$work" 2>/dev/null; then + if [ ! -s "${work}/manifest.json" ]; then + record "${first_archive} contents" "a manifest.json at the archive root" "" \ + "busbar rejects the plugin at load time with no manifest. Fix: the pack step produced a malformed archive." + else + py_out="$(python3 - "$work" "$PLUGIN_NAME" "$PLUGIN_ALIAS" "$PLUGIN_KIND" "$V" "$REQUIRE_SIGNED" <<'PY' + import glob, hashlib, json, os, sys + work, want_name, want_alias, want_kind, want_ver, require_signed = sys.argv[1:7] + m = json.load(open(os.path.join(work, "manifest.json"))) + out = [] + def chk(field, got, want): + out.append(("ok" if got == want else "bad", "manifest %s" % field, str(want), str(got))) + chk("name", m.get("name"), want_name) + chk("alias", m.get("alias"), want_alias) + chk("kind", m.get("kind"), want_kind) + chk("version", m.get("version"), want_ver) + libs = [p for p in glob.glob(os.path.join(work, "*")) + if os.path.basename(p) not in ("manifest.json", "p.tar.gz")] + if not libs: + out.append(("bad", "cdylib in the archive", "one shared library beside manifest.json", "none")) + else: + b = open(libs[0], "rb").read() + chk("sha256 binding to %s" % os.path.basename(libs[0]), + hashlib.sha256(b).hexdigest(), m.get("sha256")) + # The magic byte check is cheap and catches the whole family of "we packed the wrong + # file": a build script that packed a .d, an empty stub, or a text error message. + magic = "ELF" if b[:4] == b"\x7fELF" else ("Mach-O" if b[:4] in (b"\xcf\xfa\xed\xfe", b"\xce\xfa\xed\xfe") else ("PE" if b[:2] == b"MZ" else "not a shared library")) + out.append(("ok" if magic != "not a shared library" else "bad", + "packed file is a real shared library", "ELF/Mach-O/PE", magic)) + if require_signed == "true": + out.append(("ok" if m.get("signature") else "bad", "manifest signature", "a non-empty signature", m.get("signature") or "")) + for verdict, what, want, got in out: + print("\t".join((verdict, what, want, got))) + PY + )" || py_out="" + if [ -z "$py_out" ]; then + record "${first_archive} manifest" "a readable manifest.json" "" \ + "The published archive's manifest could not be read at all. Fix: re-run the pack step." + fi + while IFS=$'\t' read -r verdict what want got; do + [ -n "${verdict:-}" ] || continue + if [ "$verdict" = "ok" ]; then + echo "PASS: ${what} == ${got}" + else + record "${first_archive}: ${what}" "$want" "$got" \ + "busbar refuses to load this plugin on the user's machine. A signature is empty when BUSBAR_SIGN_KEY was unset and the pack step silently fell back to --allow-unsigned; a sha256 mismatch means the archive was rebuilt after signing. Fix: provision BUSBAR_SIGN_KEY in this repo and re-cut." + fi + done <<< "$py_out" + fi + else + record "${first_archive} download+extract" "a valid .tar.gz" "" \ + "The published bytes are not a readable gzip archive. Fix: re-upload the asset." + fi + rm -rf "$work" + fi + + # -- C5: THE BUNDLE MUST BOOT AND SERVE ----------------------------------------------- + # "It built" and "it runs" are different claims, and only the first was ever checked + # anywhere in this fleet. headroom-hook's published bundle proves the gap is not theoretical: + # its shipped docker/bundle/config.yaml still uses `auth.admin_auth:` with inline module + # entries, retired in busbar 1.5.3, so the container exits 1 during config load with "this + # looks like a busbar 1.x config" and never binds a port. The image builds, pushes, and + # every workflow involved goes green. + if [ -z "$BUNDLE_IMAGE" ]; then + declared "runnable bundle boot" "this repo publishes no container bundle (bundle_image is empty), so there is no image to boot. If it grows one, set bundle_image and this check starts applying automatically." + else + # DELETE THE LOCAL COPIES FIRST. `docker pull` is a no-op against a tag the daemon + # already has and `docker run`/`inspect` then use the LOCAL image, which reports a stale + # bundle as fresh. Nothing below may be allowed to answer from cache. + docker rmi -f "${BUNDLE_IMAGE}:${V}" "${BUNDLE_IMAGE}:latest" >/dev/null 2>&1 || true + if ! docker pull -q "${BUNDLE_IMAGE}:${V}" >/dev/null 2>&1; then + record "${BUNDLE_IMAGE}:${V}" "a pullable image" "" \ + "The bundle for ${TAG} was never pushed, so \`docker run ${BUNDLE_IMAGE}\` cannot give anyone this version. Fix: re-run the bundle build workflow for ${TAG}." + else + # `:latest` must be the same image, by DIGEST. A bundle whose :latest never moved hands + # every unpinned user the previous release, silently and indefinitely. + d_ver="$(docker image inspect "${BUNDLE_IMAGE}:${V}" --format '{{index .RepoDigests 0}}' 2>/dev/null | sed 's/.*@//')" + d_latest="" + docker pull -q "${BUNDLE_IMAGE}:latest" >/dev/null 2>&1 \ + && d_latest="$(docker image inspect "${BUNDLE_IMAGE}:latest" --format '{{index .RepoDigests 0}}' 2>/dev/null | sed 's/.*@//')" + if [ -n "$d_ver" ] && [ "$d_ver" = "$d_latest" ]; then + echo "PASS: ${BUNDLE_IMAGE}:latest == :${V} (${d_ver})" + else + record "${BUNDLE_IMAGE}:latest" "${d_ver:-} (the :${V} digest)" "${d_latest:-}" \ + "\`docker run ${BUNDLE_IMAGE}\` with no tag - what the docs show - serves a different release than ${TAG}. Fix: the bundle build must tag and push \`latest\` as well as the version; docker/metadata-action does NOT imply latest from a semver pattern." + fi + + # THE PORT MUST BE PROVEN FREE BEFORE THE PROBE, and this is not paranoia: probing a + # port that something else already answers on returns a cheerful 200 from the wrong + # process while the container under test is dead in the water. That happened while this + # check was being written and briefly reported a bundle as healthy that had exited 1. + port=18723 + if curl -s -m 3 -o /dev/null "http://127.0.0.1:${port}${HEALTH_PATH}" 2>/dev/null; then + echo "::error::port ${port} on the runner is already answering before the container starts, so a health probe against it would prove nothing. Refusing to report a possibly-false PASS." + fail=1 + else + envargs="" + for kv in $BUNDLE_ENV; do envargs="${envargs} -e ${kv}"; done + docker rm -f busbar-bundle-verify >/dev/null 2>&1 || true + # shellcheck disable=SC2086 + docker run -d --name busbar-bundle-verify -p "${port}:8080" $envargs \ + "${BUNDLE_IMAGE}:${V}" >/dev/null 2>&1 || true + body="" + for _ in $(seq 1 30); do + # A container that EXITED will never answer, so stop waiting the moment it dies + # rather than burning the full 60s and then reporting a timeout, which reads like a + # slow boot instead of a refusal to boot. + st="$(docker inspect -f '{{.State.Status}}' busbar-bundle-verify 2>/dev/null || echo missing)" + [ "$st" = "exited" ] || [ "$st" = "missing" ] && break + body="$(curl -fsS -m 3 "http://127.0.0.1:${port}${HEALTH_PATH}" 2>/dev/null || true)" + [ "$body" = "$HEALTH_BODY" ] && break + sleep 2 + done + st="$(docker inspect -f '{{.State.Status}} exit={{.State.ExitCode}}' busbar-bundle-verify 2>/dev/null || echo 'missing exit=?')" + if [ "$body" = "$HEALTH_BODY" ]; then + echo "PASS: ${BUNDLE_IMAGE}:${V} boots and serves '${HEALTH_BODY}' on ${HEALTH_PATH}" + else + echo "--- container log ---" + docker logs busbar-bundle-verify 2>&1 | head -40 || true + echo "---------------------" + record "${BUNDLE_IMAGE}:${V} boot" "serves '${HEALTH_BODY}' on ${HEALTH_PATH}" "container ${st}, body '${body:-}'" \ + "THE PUBLISHED BUNDLE DOES NOT RUN. A user who does \`docker run ${BUNDLE_IMAGE}\` gets a container that dies. The log is above; the known cause in this fleet is the bundle's own shipped config.yaml using config shapes a newer busbar retired, in which case busbar exits 1 with 'this looks like a busbar 1.x config' before binding a port. Fix: run \`busbar --migrate-config\` over docker/bundle/config.yaml, and pin .busbar-ref to the engine the bundle actually embeds." + fi + docker rm -f busbar-bundle-verify >/dev/null 2>&1 || true + fi + fi + fi + + { + echo "### Consumer check for \`${TAG}\`" + echo + if [ "$fail" = 0 ]; then + echo "What ${REPO} published as \`${TAG}\` downloads, unpacks into a plugin busbar accepts, and (where a bundle is published) boots and serves." + else + echo "**What ${REPO} published as \`${TAG}\` does NOT work for a user:**" + echo + cat /tmp/findings.md + fi + } >> "$GITHUB_STEP_SUMMARY" + + [ "$fail" = 0 ] + + # Same alert shape as core's verify-deploy.yml, and for the same reason: a red run in a repo + # nobody is watching is not a signal. Because this is a reusable workflow, `github.repository` is + # the CALLING repo, so the issue lands where the broken release lives. Idempotent by label: one + # open issue at a time, retitled and rewritten on every later failure, so a daily schedule updates + # it instead of filing thirty. + alert: + name: alert (open or update the release-broken issue) + needs: consumer + if: ${{ always() && needs.consumer.result == 'failure' }} + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: write + actions: read + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + VERSION: ${{ needs.consumer.outputs.version }} + LABEL: consumer-verification + steps: + - name: Open or update the single open release-broken issue + run: | + set +e # GitHub runs this as `bash -e {0}`; errexit is on before line 1. + set -uo pipefail + V="${VERSION:-unknown}" + TITLE="[release-broken] Published v${V} does not work for a consumer" + + # Read the findings out of the FAILED JOB'S OWN LOG rather than restating them. The job + # already printed `FAIL: | expected: | observed: ` and an `::error::` line + # naming the fix; re-deriving that here would be a second statement of the same fact, free + # to drift from the first, which is the exact defect shape this whole workflow exists to + # catch. Job logs are available over the API as soon as the JOB finishes, which it has. + : > /tmp/findings.md + jid="$(gh api "repos/${REPO}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}/jobs" \ + --jq '[.jobs[]? | select(.conclusion=="failure")][0].id' 2>/dev/null || true)" + if [ -n "${jid:-}" ] && gh api "repos/${REPO}/actions/jobs/${jid}/logs" > /tmp/job.log 2>/dev/null; then + sed -E 's/^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:.]+Z //' /tmp/job.log \ + | grep -E '^(FAIL:|::error::)' | sed -E 's/^::error::/DIAGNOSIS: /' \ + | head -40 | sed 's/^/ /' >> /tmp/findings.md + fi + [ -s /tmp/findings.md ] || echo " (the failed job's log was not retrievable over the API; open the run and read it there)" >> /tmp/findings.md + + { + echo "" + echo + echo "**What this repo published does not work when a user gets it.** Opened and" + echo "maintained automatically by \`GetBusbar/busbar/.github/workflows/plugin-consumer-verify.yml\`," + echo "which downloads the published artifact from the Release and uses it - it does not" + echo "read this repository, so a fix that is committed but not published still fails here." + echo + echo "This is **not a flaky test**. Every check reads what is published." + echo + echo "| | |" + echo "| --- | --- |" + echo "| version under test | \`v${V}\` |" + echo "| failing run | ${RUN_URL} |" + echo "| last checked | $(date -u '+%Y-%m-%d %H:%M UTC') |" + echo + echo "## What failed" + echo + cat /tmp/findings.md + echo + echo "## What to do" + echo + echo "Read the \`DIAGNOSIS:\` lines - each names the user-visible symptom and the fix." + echo "Fix it at the source (the release workflow, the bundle's shipped config), then" + echo "re-run the check. This issue is reused, not duplicated, and closes itself when a" + echo "run passes." + } > /tmp/issue-body.md + + gh label create "$LABEL" --repo "$REPO" --color B60205 \ + --description "A published release is broken for consumers (auto-filed)" >/dev/null 2>&1 || true + + existing="$(gh issue list --repo "$REPO" --label "$LABEL" --state open \ + --limit 1 --json number --jq '.[0].number // empty' 2>/dev/null || true)" + if [ -n "${existing:-}" ]; then + gh issue edit "$existing" --repo "$REPO" --title "$TITLE" --body-file /tmp/issue-body.md + gh issue comment "$existing" --repo "$REPO" \ + --body "Still failing as of $(date -u '+%Y-%m-%d %H:%M UTC') - ${RUN_URL}" + echo "::error::Consumer verification is FAILING for v${V}. Details in ${GITHUB_SERVER_URL}/${REPO}/issues/${existing}" + else + num="$(gh issue create --repo "$REPO" --title "$TITLE" --label "$LABEL" \ + --body-file /tmp/issue-body.md 2>/dev/null | tail -1)" + echo "::error::Consumer verification is FAILING for v${V}. Opened ${num}" + fi + + # Without this the issue is opened once and lives forever, and a stale open "release is broken" + # issue is worse than none: the next real breakage looks like the old one and gets ignored. + resolved: + name: close the release-broken issue when the check passes + needs: consumer + if: ${{ always() && needs.consumer.result == 'success' }} + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + issues: write + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + LABEL: consumer-verification + steps: + - name: Close any open release-broken issue + run: | + set +e # GitHub runs this as `bash -e {0}`; errexit is on before line 1. + set -uo pipefail + existing="$(gh issue list --repo "$REPO" --label "$LABEL" --state open \ + --limit 1 --json number --jq '.[0].number // empty' 2>/dev/null || true)" + [ -n "${existing:-}" ] || { echo "No open ${LABEL} issue. Nothing to close."; exit 0; } + gh issue comment "$existing" --repo "$REPO" \ + --body "Consumer verification is GREEN again as of $(date -u '+%Y-%m-%d %H:%M UTC'): the published release downloads, unpacks into a plugin busbar accepts, and boots where a bundle is published. Closing. Run: ${RUN_URL}" + gh issue close "$existing" --repo "$REPO" --reason completed + echo "Closed #${existing}." diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dbd3bf2f4..b92534448 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -519,3 +519,58 @@ jobs: fi done < .github/release-notify-targets.txt [ "$fail" = 0 ] || { echo "::error::one or more downstream dispatches failed (see warnings)"; exit 1; } + + # -- THE LAST STEP: DOES THE THING WE JUST PUBLISHED ACTUALLY WORK FOR A USER? ------------------- + # + # Everything above this line verifies what THIS WORKFLOW produced, from inside this workflow. The + # gate proves the tests passed. `verify-assets` proves the assets it uploaded are present and + # plausibly sized. `notify-downstream` proves the fan-out fired. Not one of them proves that + # `docker pull getbusbar/busbar`, `curl -fsSL https://getbusbar.com/install.sh | sh`, or + # `brew install getbusbar/busbar/busbar` gives a user this release. Those are different systems, in + # different repos, on different clocks. + # + # `verify-deploy.yml` already checked all of that, and thoroughly - but it fired on its OWN + # `release: published` trigger, BESIDE this workflow rather than as part of it. Nothing here waited + # on it and nothing reported its verdict as part of the release's status. So the release could be + # green while being unusable, with the only evidence a separate red run in a repo full of runs. + # That is not a hypothetical: 1.5.3 published five of its seven assets, `install.sh` returned 404 + # on Apple Silicon, `docker pull getbusbar/busbar` served the previous release, and every one of + # those was found by a human checking by hand. + # + # Calling it as a job makes its failure THIS RUN's failure, on the release's own status page, with + # the failing check named in this run's own job list. That is what "last step" has to mean. + # + # IT CANNOT GATE PUBLICATION, AND DOES NOT PRETEND TO. Consumer verification is post-publication by + # nature: you cannot pull an image that was never pushed or brew-install a formula the tap has not + # bumped. Drafting the release and promoting it only after verification would gate the + # asset-completeness class of defect but STILL not this one, because every downstream channel it + # checks (the tap, the chart, the site, /releases/latest) only moves once the release is public. + # So the value here is the verdict being impossible to miss, not a block: + # 1. this job turns the RELEASE RUN red, and + # 2. verify-deploy.yml's `alert` job opens/updates a labelled GitHub issue naming the failing + # check, its expected and observed values, and the run URL. + # + # `!cancelled()` FOR THE SAME REASON `verify-assets` HAS IT. A `needs:` on a failed job skips the + # dependent by default, so without this a failed `notify-downstream` (which fails outright while + # RELEASE_DISPATCH_TOKEN is unprovisioned - see its TODO) would SKIP consumer verification on every + # release. The one check that matters most must not be switched off by an unrelated red job + # upstream of it. + consumer-verification: + name: consumer verification (LAST STEP) + needs: [verify-assets, notify-downstream] + if: ${{ !cancelled() }} + # `./` resolves the reusable workflow AT THIS RUN'S REF, i.e. at the tag being released, so a + # release is verified by the verifier that shipped with it rather than by whatever is on the + # default branch today. + uses: ./.github/workflows/verify-deploy.yml + with: + version: ${{ github.ref_name }} + # Job-level permissions REPLACE the workflow-level block for a called workflow, and a called + # workflow can never exceed what it is granted here. `issues: write` is what lets the alert job + # file the issue; `actions: read` is what lets it read the failed job's log to quote the exact + # expected/observed values. + permissions: + contents: read + issues: write + actions: read + secrets: inherit diff --git a/.github/workflows/verify-deploy.yml b/.github/workflows/verify-deploy.yml index 052ce8662..77da03e7b 100644 --- a/.github/workflows/verify-deploy.yml +++ b/.github/workflows/verify-deploy.yml @@ -123,12 +123,45 @@ name: Verify deploy # object BEFORE the build matrix runs, and that event fires regardless of what the rest of the run # does, which is exactly the property needed: if a release object exists in public, it gets verified, # whether or not the workflow that made it finished happy. +# +# -- THIS IS THE LAST STEP OF THE RELEASE, NOT A BYSTANDER --------------------------------------- +# Until now every trigger above fired this workflow BESIDE the release. release.yml's own graph +# ended at `notify-downstream`, nothing in it waited on this, and nothing reported this verdict as +# part of the release's status. So a release could be green, fanned out to the Homebrew tap and the +# Helm chart, and simultaneously unusable, with the only evidence a separate red run in a repo full +# of runs. That is what happened to 1.5.3: it published five of seven assets, `install.sh` 404'd on +# Apple Silicon, and a human found it by hand. +# +# `workflow_call` fixes that. release.yml's FINAL job now calls this file directly, so a failing +# consumer check turns the RELEASE RUN red, on the release's own status, with the failure named in +# the release run's own job list. Two properties make this the right mechanism rather than "have +# release.yml poll for the separate run's conclusion": +# * `uses: ./.github/workflows/verify-deploy.yml` from a tag-triggered caller loads this file AT +# THE TAG, so the release is verified by the verifier that shipped with it. (The other triggers +# load it from the default branch - see the workflow_run note below for why that is harmless +# here: this file reads no repository state.) +# * A called workflow's failure is the caller job's failure. No polling, no timeout heuristics, +# no second source of truth about whether verification passed. +# +# CONSUMER VERIFICATION IS POST-PUBLICATION BY NATURE, and this does not pretend otherwise. You +# cannot `docker pull` an image that was never pushed, or `brew install` a formula the tap has not +# bumped. So this CANNOT gate publication and is not designed to. What it does instead: +# 1. makes the verdict part of the release's own status (RED on the release run), and +# 2. opens/updates a GitHub issue naming the failing check (see the `alert` job below), +# because a release that is published and unusable is a fact somebody has to be TOLD, and red alone +# is a signal only for whoever happens to be looking at that run. on: release: types: [published] workflow_run: workflows: ["Release", "Docker"] types: [completed] + workflow_call: + inputs: + version: + description: "Version to verify (e.g. 1.5.2, no leading v)" + required: true + type: string workflow_dispatch: inputs: version: @@ -138,14 +171,439 @@ on: # 13:17 UTC daily. Off the top of the hour on purpose: :00 cron slots are the most contended on # GitHub's shared scheduler and get delayed the most, and mid-day UTC lands in working hours for # both EU and US-East so a red rot alert is seen the day it fires, not the next morning. + # THE FULL SWEEP runs on this one only. - cron: "17 13 * * *" + # 3-HOURLY POINTER SWEEP (the `pointers` job only; `verify` is gated off it below). + # + # WHY A SECOND, FASTER CRON. "if docker or anything isn't latest we need to know right away." + # Daily means up to 24h of `docker pull getbusbar/busbar` handing users the previous release, + # which is exactly what happened across at least two releases when docker.yml's `tags:` block + # emitted `type=semver,pattern={{version}}` and nothing else - docker/metadata-action does NOT + # imply `latest` from a semver pattern, so `latest` stayed frozen wherever a human last put it + # while the comment above the block said "X.Y.Z + latest" the whole time. + # + # WHY 3 HOURS AND NOT HOURLY. The cost side is real but small: the pointer sweep is HEAD requests + # and small JSON reads (no image pull, no build, no brew, no boot), ~1-2 minutes of runner time. + # 3-hourly is 8 runs/day, under 20 minutes of runner time a day, and caps the window in which a + # stale default pointer can go unnoticed at 3h instead of 24h. Hourly would be 3x the runs for a + # 2h improvement on a number already inside the "someone notices this shift" range, and GitHub + # deprioritises high-frequency crons on shared runners, so the nominal interval would not be the + # real one anyway. The release-publication trigger is what makes the common case immediate; this + # cron exists for the case where a pointer rots WITHOUT a release, which is how the Docker `latest` + # freeze survived: nothing about our artifacts changed on the day it broke. + - cron: "23 */3 * * *" +# `issues: write` is for the `alert` job, which is the second half of "make a failure impossible to +# miss": red on the release run, AND an issue that comes and finds a human. `actions: read` lets +# that job read the FAILED job's own log through the API so the issue can quote the exact FAIL / +# ::error:: lines (the failing check, expected, observed) rather than saying "something went wrong, +# go read a log". permissions: contents: read + issues: write + actions: read jobs: + # -- MOVING POINTERS ----------------------------------------------------------------------------- + # A separate job from `verify`, on purpose, and it is the cheap one. + # + # THE DEFECT CLASS. Every channel below has a DEFAULT pointer: the thing a user gets when they do + # not name a version. `docker pull getbusbar/busbar`. `/releases/latest/download/...`. `brew + # install`. `pip install busbar-admin`. `uses: GetBusbar/validate-action@v1`. Each of those + # pointers is written by a DIFFERENT publish step in a DIFFERENT repo, and each one can silently + # fail to move while the version-pinned artifact beside it publishes perfectly. When that happens + # nothing is red anywhere: the pinned thing exists, the release is green, and users quietly get + # old code. `docker pull getbusbar/busbar` served the previous release to tens of thousands of + # pulls that way, across at least two releases. + # + # THE RULE THIS JOB ENFORCES: for every channel, the pointer a user gets by DEFAULT must resolve to + # the newest thing that channel's repo actually published. Two families, because busbar's + # distribution is deliberately mixed-model: + # * TRACKS BUSBAR'S VERSION - docker.io/ghcr.io `:latest`, github `/releases/latest`, the Homebrew + # tap, the helm chart's `appVersion`, the download page. These must equal the release under test. + # * INDEPENDENT SEMVER - the three SDKs (PyPI/npm/Go), the Terraform provider, validate-action's + # `@v1`. These do NOT mirror busbar's version and asserting they do is simply wrong (it was, and + # it red-failed check (e)). For these the invariant is registry-latest == that repo's own newest + # published tag, which catches the real defect ("we tagged it and the publish job never ran") + # without false-failing on the legitimate no-op-release case. + # + # DIGESTS, NOT TAG NAMES, AND NEVER A LOCAL IMAGE. The registry assertions compare MANIFEST DIGESTS + # over the Distribution API. They deliberately do not `docker run ... --version`: a local image + # cache will happily answer with the OLD image for the SAME tag and report a stale `latest` as + # fresh (or a fresh one as stale). This job pulls nothing at all, so it cannot be fooled that way; + # check (d) in the `verify` job, which does need a real image, deletes its local copy first. + pointers: + name: moving pointers (every default a user gets is the newest release) + runs-on: ubuntu-latest + # HEAD requests and small JSON reads only. If this has not finished in 20 minutes something is + # hanging, and on a 3-hourly cron a hung run must die well before the next one fires. + timeout-minutes: 20 + outputs: + version: ${{ steps.sweep.outputs.version }} + env: + DOCKERHUB_IMAGE: getbusbar/busbar + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Every default pointer must resolve to the newest published release + id: sweep + # THE ONLY `${{ }}` IN THIS STEP LIVES HERE, AND IT HAS TO. A `run:` block containing any + # `${{ }}` is compiled as ONE expression, and GitHub caps an expression at 21000 characters + # - which this script comfortably exceeds, so the whole WORKFLOW fails to parse with + # "Exceeded max expression length" and cannot even be dispatched. Hoisting the interpolation + # into `env:` leaves the script as a plain literal with no length limit. It is also the safer + # shape regardless: an event-supplied value reaches the shell as an environment variable + # rather than being pasted into the script text. + env: + SUPPLIED_VERSION: ${{ inputs.version || github.event.release.tag_name || '' }} + run: | + # `set +e` FIRST, AND IT IS LOAD-BEARING. GitHub runs every `run:` block as + # `bash -e {0}`, so errexit is ALREADY ON before the first line of the script: writing + # `set -uo pipefail` does not turn it off, it just leaves it on. Without the explicit + # `set +e` this sweep would abort at the first non-zero command and report ONE stale + # channel while hiding the rest, and "docker is stale" and "docker AND homebrew AND helm + # are stale" are different incidents. It cost the alert job a red run to find this. + set +e + set -uo pipefail + fail=0 + : > /tmp/pointer-failures.md + + record() { # record + echo "FAIL: $1 | expected: $2 | observed: $3" + { + echo "- **$1**" + echo " - expected: \`$2\`" + echo " - observed: \`$3\`" + echo " - $4" + } >> /tmp/pointer-failures.md + echo "::error::STALE MOVING POINTER: $1 - expected '$2', observed '$3'. $4" + fail=1 + } + declared() { # declared -- a channel with NO meaningful "latest" + echo "NOT APPLICABLE: $1 -- $2" + } + + # Same anonymous pull-token -> HEAD manifest -> Docker-Content-Digest flow the `verify` job + # uses. It is duplicated rather than shared because jobs cannot share a file without an + # artifact round-trip, and this is a pure function of its arguments with no repository + # state in it - the duplication that is dangerous is a duplicated FACT (a platform list), + # not a duplicated pure function. + reg_digest() { # reg_digest + local auth_host="$1" reg_host="$2" repo="$3" tag="$4" token_url token + if [ "$auth_host" = "auth.docker.io" ]; then + token_url="https://auth.docker.io/token?service=registry.docker.io&scope=repository:${repo}:pull" + else + token_url="https://${auth_host}/token?service=${auth_host}&scope=repository:${repo}:pull" + fi + token="$(curl -fsS --max-time 30 "$token_url" | jq -r '.token // .access_token' 2>/dev/null)" + [ -n "${token:-}" ] && [ "$token" != "null" ] || return 1 + curl -fsS --max-time 30 -I \ + -H "Authorization: Bearer $token" \ + -H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.oci.image.index.v1+json,application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.manifest.v1+json' \ + "https://${reg_host}/v2/${repo}/manifests/${tag}" \ + | tr -d '\r' | grep -i '^docker-content-digest:' | awk '{print $2}' + } + + # Newest published tag of a repo, from the RELEASES list - deliberately NOT from + # /releases/latest, which is itself one of the pointers under test. Asking the pointer what + # the newest release is and then checking the pointer against that answer is a check that + # can never fail. + newest_tag() { # newest_tag + local out + out="$(gh api --paginate "repos/$1/releases" \ + --jq '.[] | select(.draft==false and .prerelease==false) | .tag_name' 2>/dev/null \ + | sed 's/^v//' | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' | sort -V | tail -1)" + [ -n "$out" ] && { printf '%s\n' "$out"; return 0; } + # FALL BACK TO GIT TAGS, and this is not a nicety. The three SDK repos (busbar-python, + # busbar-js, busbar-go) publish to PyPI/npm/the Go proxy off a pushed TAG and create no + # GitHub Release at all. Reading releases only, this function returned empty for all + # three, and the caller then declared them "no releases yet, nothing to assert" -- three + # live, shipping, user-facing channels silently exempted from the sweep while PyPI, npm + # and proxy.golang.org were all serving 0.4.0. A pointer check that quietly excuses the + # channels it cannot read is worse than one that is absent, because it looks covered. + gh api --paginate "repos/$1/tags" --jq '.[].name' 2>/dev/null \ + | sed 's/^v//' | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' | sort -V | tail -1 + } + + # -- Which busbar release is under test ------------------------------------------------ + SUPPLIED="${SUPPLIED_VERSION#v}" + NEWEST="$(newest_tag GetBusbar/busbar)" + if [ -z "${NEWEST:-}" ]; then + echo "::error::could not enumerate GetBusbar/busbar's published releases; every pointer assertion below would be vacuous, so this fails rather than passes." + exit 1 + fi + if [ -n "$SUPPLIED" ] && [ "$SUPPLIED" != "$NEWEST" ]; then + # Not a failure by itself (a re-verify of an older version is legitimate), but the + # pointers are only ever asserted against the newest release, so say which one won. + echo "note: supplied version ${SUPPLIED} is not the newest published release (${NEWEST}); pointers are asserted against ${NEWEST}." + fi + V="$NEWEST" + TAG="v$V" + # Written BEFORE any assertion runs, on purpose: the `alert` job needs the version to + # title the issue, and it only ever reads this output when this step has FAILED. + echo "version=$V" >> "$GITHUB_OUTPUT" + echo "Newest published busbar release: ${TAG}. Every default pointer below must resolve to it." + + # -- P1/P2: container registries. THE 1.5.3 BUG, EXACTLY. ------------------------------ + # Retried because on release day docker.yml and this can race; a stale pointer is + # permanent and survives the retries, a race resolves inside them. + dh_ver="" dh_latest="" ghcr_ver="" ghcr_latest="" + for i in $(seq 1 10); do + dh_ver="$(reg_digest auth.docker.io registry-1.docker.io "$DOCKERHUB_IMAGE" "$V" || true)" + dh_latest="$(reg_digest auth.docker.io registry-1.docker.io "$DOCKERHUB_IMAGE" latest || true)" + [ -n "$dh_ver" ] && [ "$dh_ver" = "$dh_latest" ] && break + echo " ...docker.io :latest not yet == :${V} (attempt $i/10), retrying in 15s" + sleep 15 + done + if [ -z "$dh_ver" ]; then + record "docker.io ${DOCKERHUB_IMAGE}:${V}" "a resolvable manifest digest" "" \ + "The version-pinned image was never pushed. Fix: re-run docker.yml for ${TAG}." + elif [ "$dh_latest" = "$dh_ver" ]; then + echo "PASS: docker.io ${DOCKERHUB_IMAGE}:latest == :${V} (${dh_latest})" + else + record "docker.io ${DOCKERHUB_IMAGE}:latest" "$dh_ver (the :${V} digest)" "${dh_latest:-}" \ + "\`docker pull ${DOCKERHUB_IMAGE}\` -- the command in the README, the docs and on the site - is serving a DIFFERENT image than ${TAG}. This is the exact 1.5.3 defect: docker/metadata-action does not imply \`latest\` from \`type=semver,pattern={{version}}\`, so \`latest\` froze wherever a human last set it. Fix: confirm docker.yml's \`tags:\` block still emits an explicit \`type=raw,value=latest\` (gated on a real release), then re-run it for ${TAG}." + fi + for i in $(seq 1 10); do + ghcr_ver="$(reg_digest ghcr.io ghcr.io getbusbar/busbar "$V" || true)" + ghcr_latest="$(reg_digest ghcr.io ghcr.io getbusbar/busbar latest || true)" + [ -n "$ghcr_ver" ] && [ "$ghcr_ver" = "$ghcr_latest" ] && break + echo " ...ghcr.io :latest not yet == :${V} (attempt $i/10), retrying in 15s" + sleep 15 + done + if [ -z "$ghcr_ver" ]; then + record "ghcr.io/getbusbar/busbar:${V}" "a resolvable manifest digest" "" \ + "Fix: re-run docker.yml's ghcr push for ${TAG}." + elif [ "$ghcr_latest" != "$ghcr_ver" ]; then + record "ghcr.io/getbusbar/busbar:latest" "$ghcr_ver (the :${V} digest)" "${ghcr_latest:-}" \ + "Users pulling from GHCR without a tag get a different image than ${TAG}. Same fix as the Docker Hub case." + else + echo "PASS: ghcr.io/getbusbar/busbar:latest == :${V} (${ghcr_latest})" + fi + # And the two registries must agree, or "latest" means two different things depending on + # which registry you happened to pull from. + if [ -n "$dh_ver" ] && [ -n "$ghcr_ver" ] && [ "$dh_ver" != "$ghcr_ver" ]; then + record "ghcr.io vs docker.io for :${V}" "$dh_ver" "$ghcr_ver" \ + "The same tag resolves to DIFFERENT images on the two registries, so which bytes a user runs depends on which registry they pulled from. Fix: docker.yml copies the manifest cross-registry; re-run it for ${TAG}." + fi + + # -- P3: the GitHub /releases/latest redirect, and its assets -------------------------- + # Every download button on getbusbar.com, and install.sh, follow this. It 404s outright if + # the newest Release is not flagged 'latest', and silently serves the PREVIOUS release's + # bytes if the newest one never published. + loc="$(curl -fsS --max-time 30 -o /dev/null -w '%{redirect_url}' \ + "https://github.com/GetBusbar/busbar/releases/latest" || true)" + if [ "${loc##*/releases/tag/}" = "$TAG" ]; then + echo "PASS: github.com/GetBusbar/busbar/releases/latest -> ${TAG}" + else + record "github.com/GetBusbar/busbar/releases/latest" ".../releases/tag/${TAG}" "${loc:-}" \ + "install.sh and every download button on getbusbar.com resolve through this redirect, so all of them are handing users the wrong release. Fix: mark Release ${TAG} as 'latest' (it is probably still a draft or flagged prerelease)." + fi + # A redirect that lands in the right place still proves nothing if the assets behind it are + # not there. THIS IS THE 1.5.3 FIVE-OF-SEVEN CASE: the Release existed, was flagged latest, + # and `curl install.sh | sh` still 404'd on Apple Silicon because that platform's tarball + # was never uploaded. The expected names come from the release's OWN manifest at the tag, + # never from a list typed here. + # The manifest is fetched at the TAG first so the expectation tracks the release that + # produced the assets. Tags cut before the manifest existed do not carry it, and it has not + # reached `main` yet either, so the chain degrades tag -> main -> dev. Reading it from a + # branch is weaker (the platform set could have moved since the tag) but it is the same + # trade check (c) already makes, and the platform set changes rarely - whereas HARDCODING + # a list here would be the exact defect the manifest was introduced to remove. + got_manifest=0 + for ref in "${TAG}" main dev; do + if curl -fsS --max-time 30 \ + "https://raw.githubusercontent.com/GetBusbar/busbar/${ref}/.github/release-targets.json" \ + -o /tmp/pt-targets.json 2>/dev/null; then + [ "$ref" = "$TAG" ] || echo "note: ${TAG} does not carry .github/release-targets.json; using ${ref}'s copy." + got_manifest=1 + break + fi + done + if [ "$got_manifest" = 1 ]; then + mapfile -t want < <(python3 - "$TAG" <<'PY' + import json, sys + spec = json.load(open("/tmp/pt-targets.json")) + for t in spec["targets"]: + print("busbar-%s.%s" % (t["target"], t["archive"])) + PY + ) + if [ "${#want[@]}" -lt 5 ]; then + record "release-targets manifest at ${TAG}" ">= 5 platform archives" "${#want[@]}" \ + "Refusing to 'verify' the latest-download path against an empty expectation list: that passes for a release that published nothing." + fi + for a in "${want[@]}"; do + u="https://github.com/GetBusbar/busbar/releases/latest/download/${a}" + code="$(curl -sSL --max-time 60 --range 0-0 -o /dev/null -w '%{http_code}' "$u" || echo 000)" + case "$code" in + 200|206) echo "PASS: /releases/latest/download/${a} -> ${code}" ;; + *) record "/releases/latest/download/${a}" "HTTP 200/206" "HTTP ${code}" \ + "This is the platform-specific 404 that broke \`curl -fsSL https://getbusbar.com/install.sh | sh\` on Apple Silicon for the whole 1.5.3 release. A user on that platform gets nothing. Fix: find that target's leg in release.yml's \`upload-assets\` matrix, fix it, and re-upload the asset to ${TAG}." ;; + esac + done + else + record ".github/release-targets.json (tried ${TAG}, main, dev)" "fetchable over raw.githubusercontent" "" \ + "Without it this check cannot know which platforms ${TAG} owed, and guessing is how a missing platform got waved through in the first place. Fix: restore .github/release-targets.json - release.yml's own \`targets\` job reads the same file, so if it is really gone the release matrix is broken too." + fi + + # -- P4: Homebrew tap ------------------------------------------------------------------ + fver="$(curl -fsSL --max-time 30 \ + "https://raw.githubusercontent.com/GetBusbar/homebrew-busbar/main/Formula/busbar.rb" 2>/dev/null \ + | grep -m1 -E '^ *version "' | sed -E 's/.*version "([^"]+)".*/\1/' || true)" + if [ "$fver" = "$V" ]; then + echo "PASS: Homebrew tap formula version == ${V}" + else + record "Homebrew tap Formula/busbar.rb version" "$V" "${fver:-}" \ + "\`brew install getbusbar/busbar/busbar\` -- the documented command - installs an old binary. Fix: run/repair the tap's bump.yml workflow." + fi + + # -- P5: published Helm chart appVersion ----------------------------------------------- + appver="$(curl -fsS --max-time 60 "https://getbusbar.github.io/helm-charts/index.yaml" 2>/dev/null \ + | awk '/^ busbar:/{f=1} f && /appVersion:/{print $2; exit}' | tr -d '"' || true)" + if [ "$appver" = "$V" ]; then + echo "PASS: helm-charts busbar appVersion == ${V}" + else + record "GetBusbar/helm-charts busbar chart appVersion" "$V" "${appver:-}" \ + "\`helm install busbar getbusbar/busbar\` deploys an old gateway. Fix: run/repair helm-charts' release workflow." + fi + + # -- P6: what the site presents as current --------------------------------------------- + # Anchored: an unanchored substring match once passed v1.5.2 against a page advertising + # v1.5.20. + # NEVER `curl ... | grep -q` UNDER pipefail. `grep -q` exits the instant it matches, which + # closes the pipe, which kills curl with SIGPIPE (exit 23), which `pipefail` then reports + # as a failed pipeline - so the check goes RED EXACTLY WHEN THE ASSERTION HOLDS and green + # only when the page is missing the version. That inversion is live in check (g) below and + # is fixed there too. Fetch to a file, then grep the file. + # "COULD NOT READ THE PAGE" AND "THE PAGE IS STALE" ARE DIFFERENT INCIDENTS and must not + # share a message. The first run of this check on a GitHub runner reported "v1.5.3 not + # present on the page" for a page that plainly shows v1.5.3 from a laptop: the runner's + # datacenter IP gets a different response (bot challenge / edge block) than a browser. A + # check that says "the site is stale" when it means "I was blocked" sends whoever reads + # the issue to redeploy a site that is fine, and the second time it does that, everyone + # stops believing it. So: send the User-Agent a real visitor sends, and report the HTTP + # code and body size when the page cannot be read as a page, distinct from staleness. + dl_code="$(curl -sSL --max-time 30 \ + -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36' \ + -H 'Accept: text/html,application/xhtml+xml' \ + -o /tmp/pt-download.html -w '%{http_code}' "https://getbusbar.com/download/" 2>/dev/null || echo 000)" + dl_size="$(wc -c < /tmp/pt-download.html 2>/dev/null || echo 0)" + # 403/429 IS NOT THE SAME INCIDENT AS 404/5xx, and conflating them is why this check has + # been red on main every day. A 403 is aimed at US: getbusbar.com sits behind an edge that + # challenges datacenter IPs, so a GitHub runner is refused while a browser is served + # normally. That means the check COULD NOT RUN, not that the assertion failed - and + # reporting "the site is stale" for a site that is fine is how a gate gets ignored. It is a + # VISIBLE SKIP, announced three ways (log, annotation, step summary), never a silent one. + # Anything else non-200 is broken for real users too, and stays a hard failure. + case "${dl_code}:$([ "$dl_size" -lt 500 ] && echo small || echo ok)" in + 200:ok) dl_state=readable ;; + 403:*|429:*) dl_state=blocked ;; + *) dl_state=broken ;; + esac + if [ "$dl_state" = "blocked" ]; then + echo "VISIBLE SKIP: getbusbar.com/download/ returned HTTP ${dl_code} to this runner, so the advertised-version assertion could NOT be evaluated. This is an edge/bot block against GitHub's datacenter IPs, not a stale page: a browser is served normally." + echo "::warning::(pointers) PARTIAL: getbusbar.com/download/ returns HTTP ${dl_code} to GitHub Actions runners, so the 'site advertises the current version' assertion did not run. Every other pointer WAS checked and is authoritative. Fix is MARKETING-SIDE: allow GitHub Actions egress through the Cloudflare bot rules for /download/ (or expose a small unchallenged JSON route carrying the current version), so this assertion can be made from CI at all." + { echo "### Moving pointers: PARTIAL - getbusbar.com/download/ returns HTTP ${dl_code} to this runner, so the advertised-version check did not run"; } >> "$GITHUB_STEP_SUMMARY" + elif [ "$dl_state" = "broken" ]; then + record "getbusbar.com/download/ availability" "HTTP 200 and a real HTML page" "HTTP ${dl_code}, ${dl_size} bytes" \ + "The download page is not being served at all. Unlike a 403, this is broken for every visitor, not just for CI. Fix: redeploy the marketing site." + elif grep -qE "v${V}([^0-9]|\$)" /tmp/pt-download.html; then + echo "PASS: getbusbar.com/download/ advertises v${V}" + else + record "getbusbar.com/download/ advertised version" "v${V}" \ + "" \ + "The site tells visitors the current release is something other than ${TAG}. Fix: redeploy the marketing site." + fi + + # -- P7-P10: INDEPENDENT-SEMVER channels. The invariant is registry-latest == that repo's + # own newest published tag, NOT == busbar's version. ---------------------------------- + check_independent() { # check_independent