diff --git a/.github/workflows/esp-idf.yml b/.github/workflows/esp-idf.yml index 32d97bd..222daf3 100644 --- a/.github/workflows/esp-idf.yml +++ b/.github/workflows/esp-idf.yml @@ -2,28 +2,35 @@ # Builds ESP-IDF development image with pytest, QEMU, and testing tools # # Build Strategy: -# - Main repo master: Build + Push to GHCR -# - Main repo dev/PR: Build only (validation) -# - workflow_dispatch: Build on any branch of the main repo, push only on master +# - Tags (latest, sha-, -) are written on master only, by +# the manifest jobs. +# - Images themselves are pushed by digest on EVERY run, master or not, carrying no +# tag and reachable by nothing but that digest. esp-matter-build consumes them, +# which is what lets it validate on a pull request. The cost is untagged blobs +# accumulating in GHCR. +# - Dependabot is the exception: its GITHUB_TOKEN is read-only, so those runs build +# without pushing and esp-matter-build sits them out. # - Forks: nothing runs - the runner pools are not reachable there; use # ./scripts/build.sh instead # - Multi-platform: Platform matrix builds linux/amd64 and linux/arm64 in parallel, # each on a runner of its own architecture (matrix.runner) - no QEMU # -# ESP-Matter exception: esp-matter-build needs esp-idf-manifest, which runs for -# jethome-iot on master only, so the ESP-Matter jobs are skipped (not -# build-validated) on dev, on pull requests, on workflow_dispatch outside master, -# and in forks entirely. The image is built FROM the esp-idf digest that manifest -# job publishes and hands over in the same run. -# Consequence: a change touching only images/esp-matter/** still matches the paths -# filter below, so esp-idf-build runs on its unchanged context and this workflow -# reports success - a green check here does not mean ESP-Matter compiles. -# Validate it with ./scripts/build.sh esp-matter, or on master. +# ESP-Matter builds on the esp-idf image this same run produced, on every branch: +# esp-idf-build pushes by digest always, and esp-matter-build takes the digest for +# its own platform from that job's artifact. So a green check here does mean +# ESP-Matter compiles - including on a branch that bumps both versions at once, +# where the new base exists under no tag yet. +# Only the manifest jobs write tags, and they stay master-only. A pull request +# leaves untagged blobs in GHCR; they need an occasional cleanup. name: 🐳 ESP-IDF Docker Image on: push: branches: [master, dev] + # Negations come last and are order-sensitive; `paths:` and `paths-ignore:` + # cannot both be used for one event. Documentation-only commits used to + # trigger a full rebuild and republish every tag - the docs-only merge that + # started this series set off a 5.5-hour ESP-Matter leg. paths: - '.github/workflows/esp-idf.yml' - 'images/esp-idf/**' @@ -31,6 +38,7 @@ on: - 'images/versions.json' - 'scripts/versions-matrix.sh' - 'scripts/check-versions.sh' + - '!images/**/*.md' pull_request: branches: [master, dev] paths: @@ -40,6 +48,7 @@ on: - 'images/versions.json' - 'scripts/versions-matrix.sh' - 'scripts/check-versions.sh' + - '!images/**/*.md' workflow_dispatch: env: @@ -75,7 +84,7 @@ jobs: name: 🧮 Resolve matrices runs-on: ubuntu-latest timeout-minutes: 5 - if: github.repository_owner == 'jethome-iot' + if: github.repository_owner == 'jethome-iot' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) permissions: contents: read outputs: @@ -118,7 +127,11 @@ jobs: # to jethome-iot, a fork cannot resolve their labels, and an unresolvable # `runs-on` queues for 24 hours rather than failing. A fork that wants to build # these images runs ./scripts/build.sh. - if: github.repository_owner == 'jethome-iot' + # Two conditions, not one. The owner check keeps this out of forks that run the + # workflow themselves. The head-repo check keeps it out of pull requests *from* + # a fork, where github.repository_owner is still jethome-iot - so without it a + # fork-controlled Dockerfile would execute on this org's paid runner pools. + if: github.repository_owner == 'jethome-iot' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) permissions: contents: read packages: write @@ -133,8 +146,9 @@ jobs: - name: 🔧 Set up Docker Buildx uses: docker/setup-buildx-action@v4 + # Unconditional: the build below pushes by digest on every run, so every run + # needs credentials - not just master. - name: 🔐 Log in to GitHub Container Registry - if: github.ref_name == 'master' uses: docker/login-action@v4 with: registry: ${{ env.REGISTRY }} @@ -145,27 +159,41 @@ jobs: # name that the manifest job below has to look up again later, and between # those two moments another run can overwrite it. The digest is the only # thing handed forward. + # + # This pushes on pull requests too, not just master. Nothing published this + # way is reachable by name - only the manifest job creates tags, and it stays + # master-only - so a PR leaves untagged blobs behind and no moving tag. What + # it buys is that esp-matter-build can build on the ESP-IDF this very run + # produced, including on a branch that bumps both versions at once, where the + # new base does not exist under any tag yet. + # + # Except for Dependabot: GitHub hands its runs a read-only GITHUB_TOKEN + # whatever the permissions block asks for, so the push would be rejected. Its + # pull requests bump action versions, which this job still exercises by + # building; only the upload is skipped, and esp-matter-build sits that run out + # (see its `if`). - name: 🐳 Build and push by digest id: build uses: docker/build-push-action@v7 with: context: images/esp-idf platforms: ${{ matrix.platform }} - outputs: type=image,name=${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ env.ESP_IDF_IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=${{ github.ref_name == 'master' }} + outputs: type=image,name=${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ env.ESP_IDF_IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=${{ github.actor != 'dependabot[bot]' }} build-args: ${{ matrix.build_args }} # Matrix legs cannot each set a job output - they would overwrite one # another - so the digests travel as one empty file per leg, named after the - # digest itself. + # digest itself. Uploaded on every run: esp-matter-build consumes this on + # pull requests as well. - name: 📤 Export digest - if: github.ref_name == 'master' + if: github.actor != 'dependabot[bot]' run: | mkdir -p /tmp/digests digest="${{ steps.build.outputs.digest }}" touch "/tmp/digests/${digest#sha256:}" - name: ⬆️ Upload digest - if: github.ref_name == 'master' + if: github.actor != 'dependabot[bot]' uses: actions/upload-artifact@v7 with: name: digest-${{ env.ESP_IDF_IMAGE_NAME }}-${{ matrix.tag }}-${{ matrix.platform_tag }} @@ -244,40 +272,36 @@ jobs: tags+=(-t "${IMAGE}:${TAG}") docker buildx imagetools create "${tags[@]}" "${refs[@]}" - # Hand the manifest's own digest to esp-matter-build, so it builds FROM - # the exact image this run published rather than whatever the tag points - # at by the time it starts. It travels as an artifact rather than a job - # output because this job has a matrix: a scalar output would be - # overwritten by whichever leg finished last, silently pinning - # esp-matter to an arbitrary ESP-IDF version once there is more than one. + # Reported, not handed on: esp-matter-build takes the per-platform + # digests from esp-idf-build directly, so nothing downstream waits on + # this job any more. digest=$(docker buildx imagetools inspect "${IMAGE}:${TAG}" \ --format '{{json .Manifest}}' | jq -er '.digest') - case "${digest}" in - sha256:*) ;; - *) echo "::error::manifest digest not resolved: '${digest}'"; exit 1 ;; - esac - mkdir -p /tmp/manifest - printf '%s' "${digest}" > /tmp/manifest/digest echo "published ${IMAGE}:${TAG} as ${digest}" - - name: ⬆️ Upload base image digest - uses: actions/upload-artifact@v7 - with: - name: manifest-digest-${{ env.ESP_IDF_IMAGE_NAME }}-${{ matrix.tag }} - path: /tmp/manifest/digest - if-no-files-found: error - retention-days: 1 - esp-matter-build: name: esp-matter-build (${{ matrix.tag }}, ${{ matrix.platform }}) - needs: [prepare, esp-idf-manifest] + needs: [prepare, esp-idf-build] runs-on: ${{ matrix.runner }} # From images/versions.json. A cap on a wedged job, not a target: killing this # leg skips the manifest job with it and leaves the published tags on the # previous build while the other platform's image sits in GHCR unreferenced. timeout-minutes: ${{ matrix.timeout_minutes }} - # Owner-only, as esp-idf-build; see the note there. - if: github.repository_owner == 'jethome-iot' + # Depends on esp-idf-build, not esp-idf-manifest. The manifest job publishes + # tags and is master-only, so hanging off it is what kept ESP-Matter + # unvalidated on every pull request while a green "🐳 ESP-IDF Docker Image" + # check implied otherwise. What this job actually needs is the ESP-IDF image + # for its own platform, which the build leg pushes by digest on every run - so + # the dependency is plain, both branches behave identically, and no status + # expression is needed to work around a skipped job. + # + # Dependabot is excluded: its token cannot push, so esp-idf-build produces no + # digest to hand over. Its pull requests change action versions, which + # esp-idf-build still exercises. + if: >- + ${{ github.repository_owner == 'jethome-iot' + && github.actor != 'dependabot[bot]' + && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} permissions: contents: read packages: write @@ -296,34 +320,42 @@ jobs: - name: 🔧 Set up Docker Buildx uses: docker/setup-buildx-action@v4 + # Unconditional, unlike the other build jobs: this one pulls its base from + # GHCR on every run, including pull requests, and an anonymous pull is + # subject to rate limits the authenticated one is not. - name: 🔐 Log in to GitHub Container Registry - if: github.ref_name == 'master' uses: docker/login-action@v4 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - # Named after the ESP-IDF version this image declares in its own tag, so the - # base it builds on and the version it advertises cannot drift apart: bump - # esp-idf without bumping the blocks below and this download fails outright - # instead of publishing an idf-v tag built on v. - - name: ⬇️ Resolve base image digest + # The ESP-IDF image for this platform, pushed by digest by esp-idf-build in + # this same run. Not the multi-arch manifest: that is published only on + # master, and this leg builds one platform anyway. Naming the artifact after + # the ESP-IDF version this image declares in its own tag keeps the base it + # stands on and the version it advertises from drifting apart - a half-done + # bump fails here rather than publishing an idf-v tag built on v. + - name: ⬇️ Download base image digest uses: actions/download-artifact@v8 with: - name: manifest-digest-${{ env.ESP_IDF_IMAGE_NAME }}-${{ matrix.base_tag }} + name: digest-${{ env.ESP_IDF_IMAGE_NAME }}-${{ matrix.base_tag }}-${{ matrix.platform_tag }} path: /tmp/base - - name: 🏷️ Read base image digest + - name: 🏷️ Resolve base image reference id: base + env: + IMAGE: ${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ env.ESP_IDF_IMAGE_NAME }} run: | set -euo pipefail - digest=$(cat /tmp/base/digest) + # One file, named after the digest it stands for. + digest=$(find /tmp/base -type f -printf '%f\n' | head -1) case "${digest}" in - sha256:*) ;; - *) echo "::error::base digest not usable: '${digest}'"; exit 1 ;; + [0-9a-f]*) ;; + *) echo "::error::no usable base digest in the artifact"; exit 1 ;; esac - echo "digest=${digest}" >> "$GITHUB_OUTPUT" + echo "ref=${IMAGE}@sha256:${digest}" >> "$GITHUB_OUTPUT" + echo "base image: ${IMAGE}@sha256:${digest}" - name: 🐳 Build and push by digest id: build @@ -333,7 +365,7 @@ jobs: platforms: ${{ matrix.platform }} outputs: type=image,name=${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ env.ESP_MATTER_IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=${{ github.ref_name == 'master' }} build-args: | - ${{ matrix.base_arg }}=${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ env.ESP_IDF_IMAGE_NAME }}@${{ steps.base.outputs.digest }} + ${{ matrix.base_arg }}=${{ steps.base.outputs.ref }} ${{ matrix.build_args }} - name: 📤 Export digest diff --git a/.github/workflows/platformio.yml b/.github/workflows/platformio.yml index 3663100..2714187 100644 --- a/.github/workflows/platformio.yml +++ b/.github/workflows/platformio.yml @@ -2,7 +2,8 @@ # # Build Strategy: # - Main repo master: Build + Push to GHCR -# - Main repo dev/PR: Build only (validation) +# - Main repo dev/PR: Build only (validation) - nothing is uploaded. Unlike +# esp-idf.yml, no other job consumes this image, so there is nothing to push for. # - workflow_dispatch: Build on any branch of the main repo, push only on master # - Forks: nothing runs - the runner pools are not reachable there; use # ./scripts/build.sh instead @@ -13,12 +14,17 @@ name: 🐳 PlatformIO Docker Image on: push: branches: [master, dev] + # Negations come last and are order-sensitive; `paths:` and `paths-ignore:` + # cannot both be used for one event. Documentation-only commits used to + # trigger a full rebuild and republish every tag - the docs-only merge that + # started this series set off a 5.5-hour ESP-Matter leg. paths: - '.github/workflows/platformio.yml' - 'images/platformio/**' - 'images/versions.json' - 'scripts/versions-matrix.sh' - 'scripts/check-versions.sh' + - '!images/**/*.md' pull_request: branches: [master, dev] paths: @@ -27,6 +33,7 @@ on: - 'images/versions.json' - 'scripts/versions-matrix.sh' - 'scripts/check-versions.sh' + - '!images/**/*.md' workflow_dispatch: env: @@ -60,7 +67,7 @@ jobs: name: 🧮 Resolve matrices runs-on: ubuntu-latest timeout-minutes: 5 - if: github.repository_owner == 'jethome-iot' + if: github.repository_owner == 'jethome-iot' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) permissions: contents: read outputs: @@ -96,7 +103,11 @@ jobs: # to jethome-iot, a fork cannot resolve their labels, and an unresolvable # `runs-on` queues for 24 hours rather than failing. A fork that wants to build # this image runs ./scripts/build.sh. - if: github.repository_owner == 'jethome-iot' + # Two conditions, not one. The owner check keeps this out of forks that run the + # workflow themselves. The head-repo check keeps it out of pull requests *from* + # a fork, where github.repository_owner is still jethome-iot - so without it a + # fork-controlled Dockerfile would execute on this org's paid runner pools. + if: github.repository_owner == 'jethome-iot' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) permissions: contents: read packages: write diff --git a/CLAUDE.md b/CLAUDE.md index 2168890..9c48039 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -59,14 +59,13 @@ The authoritative files are `.github/workflows/esp-idf.yml` and There are no `-linux-` tags in GHCR any more. - The version tag is listed **last** in `imagetools create` on purpose: GHCR shows the last tag as the package's primary tag. -- `esp-idf-manifest` publishes the manifest's own digest as an artifact named - `manifest-digest--`, and `esp-matter-build` downloads the one - matching the ESP-IDF version in its own tag, then builds `FROM` it via a single - `BASE_IMAGE` build-arg. An artifact rather than a job output because that job has - a matrix: a scalar output would be overwritten by whichever leg finished last. - One argument rather than repo + tag because a digest needs `@`, not `:` — and it - makes the base repository overridable, so a fork or a local build can point at - its own esp-idf instead of this one. +- `esp-matter-build` depends on `esp-idf-build`, **not** on the manifest job, and + takes the per-platform digest for its own architecture from that build's + artifact. The artifact is named after the ESP-IDF version this image declares in + its own tag, so base and advertised version cannot drift apart. It reaches the + Dockerfile through a single `BASE_IMAGE` build-arg — one argument rather than + repo + tag, because a digest needs `@` where a tag needs `:`, and because it makes + the base repository overridable for a fork or a local build. - **`images/versions.json` is the single source of truth for what CI builds.** A `prepare` job runs `scripts/check-versions.sh`, then `scripts/versions-matrix.sh build|manifest`, and every other job takes its matrix from @@ -123,16 +122,30 @@ The authoritative files are `.github/workflows/esp-idf.yml` and built and thrown away, never uploaded. Manifest jobs ask both in their own job `if`, since they have no build step to gate. The org literal is hardcoded. -- `esp-matter-build` has `needs: esp-idf-manifest`, which requires both - `jethome-iot` **and** `master`, so both ESP-Matter jobs are skipped entirely on - `dev`, on PRs and on non-master dispatch. The dependency is real: - `images/esp-matter/Dockerfile` is `FROM ${BASE_IMAGE}`, and the digest CI passes - there is published by that manifest job in the same run. -- Worse than "no validation": a PR touching only `images/esp-matter/**` still - matches the workflow's `paths:` filter, so `esp-idf-build` runs on its unchanged - context and the check goes green — a passing "🐳 ESP-IDF Docker Image" on such a - PR says nothing about ESP-Matter. Validate it with `./scripts/build.sh esp-matter` - or on `master`. +- **Every image is build-validated on pull requests**, ESP-Matter included, and on + the ESP-IDF this very run produced — so a branch bumping both versions at once is + checked as the pair it will become, not against whatever is published today. +- That works because `esp-idf-build` pushes by digest on **every** run, not only on + master — so its GHCR login is unconditional too, unlike the other build jobs'. + Nothing published that way is reachable by name: only the manifest jobs create + tags, and they stay master-only. A pull request therefore leaves untagged blobs in + GHCR and moves no tag. They accumulate; clean them up periodically. +- **Dependabot is the exception.** GitHub gives its runs a read-only + `GITHUB_TOKEN` regardless of the job's `permissions:`, so the digest push would + be rejected. Its pull requests skip the push and the digest upload, and + `esp-matter-build` sits them out — the action bump they carry is still exercised + by esp-idf-build compiling the image. +- Publishing is still master-only in the sense that matters — no tag, `latest` + included, is ever written outside master. +- **Every build job also checks the PR's head repository**, not just the owner. On + a pull request *from* a fork `github.repository_owner` is still `jethome-iot`, so + the owner check alone would let a fork-controlled Dockerfile run on this org's + paid pools. The condition is + `github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository`. +- Both workflows exclude `!images/**/*.md` from their `paths:`. Negations come last + and are order-sensitive, and `paths:` cannot be mixed with `paths-ignore:` for one + event. Without this a documentation-only commit rebuilds and republishes every + image. - Every build job runs on a larger-runner pool, chosen by platform through a `runner` key in the matrix `include:` and read as `runs-on: ${{ matrix.runner }}` — `ubuntu-latest-8core` for `linux/amd64`, `ubuntu-latest-8core-arm` for diff --git a/README.md b/README.md index 7023a38..4fb30c9 100644 --- a/README.md +++ b/README.md @@ -134,19 +134,18 @@ for your host architecture only and with the Dockerfile's own `ARG` defaults instead of the versions CI passes in, so a green local build is not a green CI run. -**Note:** an image built `FROM` another image of this repo is not build-validated on -`dev`. Its build job needs the base image's manifest job, which runs only for -`jethome-iot` on `master` — that job publishes the multi-arch tag the derived -Dockerfile pulls. Today that is ESP-Matter: `esp-matter-build` needs -`esp-idf-manifest`, so both ESP-Matter jobs are skipped on `dev`, on pull requests, -on `workflow_dispatch` outside `master`, and in forks entirely. Every other image -builds from its own context and is validated on `dev`. - -A change touching only `images/esp-matter/**` still matches the workflow's path -filter, so `esp-idf-build` runs on its unchanged context and the check reports -success: a green "🐳 ESP-IDF Docker Image" on such a PR does **not** mean the -ESP-Matter image compiles. Validate it locally with -`./scripts/build.sh esp-matter` (~50GB of disk, several hours) or on `master`. +**Note:** every image is build-validated on pull requests and on `dev`, including +one built `FROM` another image of this repo, and on the base that same run +produced — so a change touching both images is checked as the pair it will become. +This works because the **ESP-IDF** build job pushes by digest on every run — it is +the only image another one is built from, so it is the only one that needs to. +Those pushes carry no tag, and only `master` ever writes `latest` or a version tag. +The untagged blobs a pull request leaves in GHCR accumulate and want an occasional +cleanup. PlatformIO, which nothing builds on, still uploads nothing outside +`master`. + +Documentation-only changes trigger nothing: `!images/**/*.md` is excluded from both +workflows' path filters. ### Manual Building