feat: s390x and ppc64le support

.github/workflows/binaries.yaml

@@ -1,8 +1,8 @@
 name: Binaries
 
-# Reusable. Cross-compiles chisel + spread (via Canonical's
-# ubuntu/go:1.25-26.04_edge builder) and fetches the upstream docker
-# static CLI for both arches. Uploads cache/binaries/ + the matching
+# Reusable. Cross-compiles chisel + spread + the docker CLI (via
+# Canonical's ubuntu/go:1.25-26.04_edge builder) for all target
+# arches. Uploads cache/binaries/ + the matching
 # .stamp/binaries so downstream jobs can short-circuit any rebuild.
 
 on:

.github/workflows/build-oci.yaml

@@ -1,15 +1,29 @@
 name: Build OCI image tarballs
 
-# Reusable. Per-arch native runner builds bread + bread-chisel-releases
+# Reusable. Per-arch runner builds bread + bread-chisel-releases
 # images into the local daemon and `docker save`s each to a
 # docker-archive tarball. Tarballs upload as artefacts
-# (images-amd64, images-arm64) so downstream jobs can stitch a multiarch
+# (images-<arch>) so downstream jobs can stitch a multiarch
 # manifest or attach to releases without rebuilding.
 #
+# amd64 / arm64 build on native runners; s390x / ppc64le have no hosted
+# runners and build under qemu on amd64 (slow, publish-untested).
+#
 # bread-test is intentionally not built here (test-infra-only).
 
 on:
   workflow_call:
+    inputs:
+      # JSON matrix-include list; callers can narrow to a subset of the
+      # default (e.g. ci builds only the qemu arches to avoid rebuilding
+      # what build-and-test already builds natively).
+      matrix:
+        type: string
+        default: >-
+          [{"runner": "ubuntu-24.04",     "arch": "amd64"},
+           {"runner": "ubuntu-24.04-arm", "arch": "arm64"},
+           {"runner": "ubuntu-24.04",     "arch": "s390x",   "qemu": true},
+           {"runner": "ubuntu-24.04",     "arch": "ppc64le", "qemu": true}]
 
 permissions:
   contents: read
@@ -19,18 +33,19 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - runner: ubuntu-24.04
-            arch: amd64
-          - runner: ubuntu-24.04-arm
-            arch: arm64
+        include: ${{ fromJSON(inputs.matrix) }}
     runs-on: ${{ matrix.runner }}
-    timeout-minutes: 30
+    # NOTE: 90 min covers the qemu lanes; native lanes finish well under 30.
+    timeout-minutes: 90
 
     steps:
       - name: Checkout repository
         uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
 
+      - name: Set up qemu (emulated arches only)
+        if: matrix.qemu
+        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0
+
       - name: Restore cached binaries + stamp
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:

.github/workflows/ci.yaml

@@ -25,3 +25,14 @@ jobs:
   build-and-test:
     needs: binaries
     uses: ./.github/workflows/build-and-test.yaml
+
+  # Also build the qemu-arch images (s390x / ppc64le) so image breakage
+  # surfaces at PR time, not at release. Native arches are excluded --
+  # build-and-test already builds those.
+  build-oci:
+    needs: binaries
+    uses: ./.github/workflows/build-oci.yaml
+    with:
+      matrix: >-
+        [{"runner": "ubuntu-24.04", "arch": "s390x",   "qemu": true},
+         {"runner": "ubuntu-24.04", "arch": "ppc64le", "qemu": true}]

.github/workflows/push-ghcr.yaml

@@ -4,7 +4,7 @@ name: Push to GHCR
 # stitches a multiarch manifest list per (flavour, ver) via buildah, and
 # pushes each to ghcr.io/<repo>/{bread,bread-chisel-releases}:<ver>.
 #
-# Expects images-amd64 + images-arm64 artefacts to already exist (caller
+# Expects images-<arch> artefacts (one per arch) to already exist (caller
 # must have run build-oci.yaml in the same workflow run).
 
 on:
@@ -23,17 +23,12 @@ jobs:
       id-token: write
 
     steps:
-      - name: Download amd64 image tarballs
+      # Each images-<arch> artefact lands in its own <arch>/ dir.
+      - name: Download per-arch image tarballs
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
-          name: images-amd64
-          path: amd64
-
-      - name: Download arm64 image tarballs
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          name: images-arm64
-          path: arm64
+          pattern: images-*
+          merge-multiple: false
 
       - name: Install buildah
         run: |
@@ -53,10 +48,10 @@ jobs:
             for ver in 24.04 25.10 26.04 26.10; do
               m="$flavour-$ver-manifest"
               buildah manifest create "$m"
-              buildah manifest add --arch amd64 "$m" \
-                "docker-archive:amd64/$flavour-$ver-amd64.tar"
-              buildah manifest add --arch arm64 "$m" \
-                "docker-archive:arm64/$flavour-$ver-arm64.tar"
+              for arch in amd64 arm64 s390x ppc64le; do
+                buildah manifest add --arch "$arch" "$m" \
+                  "docker-archive:images-$arch/$flavour-$ver-$arch.tar"
+              done
               buildah manifest push --format oci --all "$m" \
                 "docker://$REGISTRY/${{ github.repository }}/$flavour:$ver"
               buildah manifest rm "$m"

.github/workflows/release.yaml

@@ -65,15 +65,13 @@ jobs:
           # inlined yamls verbatim
           cp inlined/*.yaml release-out/
 
-          # spread binaries: rename to canonical go-release-style names
-          cp cache/binaries/spread-amd64 release-out/spread-linux-amd64
-          cp cache/binaries/spread-arm64 release-out/spread-linux-arm64
-          chmod +x release-out/spread-linux-*
-
-          # chisel-hacked binaries
-          cp cache/binaries/chisel-hacked-amd64 release-out/chisel-hacked-linux-amd64
-          cp cache/binaries/chisel-hacked-arm64 release-out/chisel-hacked-linux-arm64
-          chmod +x release-out/chisel-hacked-linux-*
+          # spread + chisel-hacked binaries: rename to canonical
+          # go-release-style names
+          for arch in amd64 arm64 s390x ppc64le; do
+            cp "cache/binaries/spread-$arch" "release-out/spread-linux-$arch"
+            cp "cache/binaries/chisel-hacked-$arch" "release-out/chisel-hacked-linux-$arch"
+          done
+          chmod +x release-out/spread-linux-* release-out/chisel-hacked-linux-*
 
           # sha256 sidecars (standard sha256sum format)
           cd release-out
@@ -88,7 +86,7 @@ jobs:
         run: |
           set -e
           cd release-out
-          for f in spread-linux-amd64 spread-linux-arm64 chisel-hacked-linux-amd64 chisel-hacked-linux-arm64; do
+          for f in spread-linux-{amd64,arm64,s390x,ppc64le} chisel-hacked-linux-{amd64,arm64,s390x,ppc64le}; do
             cosign sign-blob --yes \
               --bundle "$f.cosign.bundle" \
               "$f"
@@ -132,7 +130,8 @@ jobs:
             ```
 
             **precompiled spread binaries** -- statically-linked spread CLI
-            for linux amd64 / arm64, cross-compiled with go 1.25.x. each
+            for linux amd64 / arm64 / s390x / ppc64le, cross-compiled with
+            go 1.25.x. each
             binary ships with a `.sha256` checksum sidecar and a
             `.cosign.bundle` keyless signature bundle.
 

README.md

@@ -33,10 +33,10 @@ anyway, here is this project's pet bread:
 
 ## flavours
 
-two image flavours, each for ubuntu 24.04, 25.10, 26.04, 26.10 x amd64 / arm64, published as multiarch tags at `ghcr.io/lczyk/spread-bread/<flavour>:<ver>`:
+two image flavours, each for ubuntu 24.04, 25.10, 26.04, 26.10 x amd64 / arm64 / s390x / ppc64le, published as multiarch tags at `ghcr.io/lczyk/spread-bread/<flavour>:<ver>`. heads-up: s390x + ppc64le images build under qemu and ship untested (no native runners); amd64 + arm64 are tested on every release:
 
 - **`bread`** -- base: ubuntu + sshd. general-purpose spread system; the test suite installs whatever else it needs.
-- **`bread-chisel-releases`** -- bread + `chisel` + the shell + container tooling typically needed by [chisel-releases](https://github.com/canonical/chisel-releases) spread tests (curl, wget, git, jq, file, sudo, tree, docker, skopeo). `chisel` and `docker` are built from source (canonical/chisel pinned by SHA, docker fetched from docker.com static) so the bundled binaries are go 1.25+ and survive qemu emulation.
+- **`bread-chisel-releases`** -- bread + `chisel` + the shell + container tooling typically needed by [chisel-releases](https://github.com/canonical/chisel-releases) spread tests (curl, wget, git, jq, file, sudo, tree, docker, skopeo). `chisel` and `docker` are built from source (canonical/chisel pinned by SHA, docker/cli pinned by version tag) so the bundled binaries are go 1.25+ and survive qemu emulation.
 
 ## using (the common case)
 
@@ -56,7 +56,7 @@ available yamls in the release:
 
 ## install spread
 
-prefer a precompiled spread CLI over `go install`? same release ships statically-linked binaries for linux amd64 / arm64:
+prefer a precompiled spread CLI over `go install`? same release ships statically-linked binaries for linux amd64 / arm64 / s390x / ppc64le:
 
 ```
 curl -fsSL https://github.com/lczyk/spread-bread/releases/latest/download/spread-linux-amd64 -o /usr/local/bin/spread
@@ -87,7 +87,7 @@ the cert-identity regex above is approximate; consult the actual issued cert on
 spread-bread/
   makefile                       # build images + generate inlined yamls + run contract tests
   hack/
-    build_binaries.sh            # cross-compile chisel + spread, fetch docker static, in one ubuntu/go builder
+    build_binaries.sh            # cross-compile chisel + spread + docker cli in one ubuntu/go builder
     build_image.sh               # per-image build w/ hash-stamp short-circuit
     hash_inputs.sh               # per-image input hash (drives stamp invalidation)
     check_base.sh                # detect upstream ubuntu base digest drift; rewrite @sha256 pins
@@ -150,8 +150,8 @@ the outer spreads a `bread-test` container; that container runs an inner spread
 
 `release.yaml` triggers on push of an `r[0-9]+` tag. on success it:
 
-- builds the binary cache + all images on per-arch native runners.
-- pushes 6 multiarch manifests to `ghcr.io/lczyk/spread-bread/{bread,bread-chisel-releases}:<ver>`.
+- builds the binary cache + all images -- amd64 / arm64 on per-arch native runners, s390x / ppc64le under qemu.
+- pushes 8 multiarch manifests to `ghcr.io/lczyk/spread-bread/{bread,bread-chisel-releases}:<ver>`.
 - attaches `inlined/*.yaml` to a rolling github release called `yamls`. older `r*` releases are deleted (release objects only; the underlying tags stay).
 
 ci (`ci.yaml`) is build + test on every PR + push to main; no publish.

hack/build_binaries.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
-# Cross-compile chisel + spread for both target arches inside a single
-# Canonical ubuntu/go:1.25-26.04_edge container. Output binaries land in
-# ./cache/binaries/{chisel,chisel-hacked,spread}-{amd64,arm64}.
+# Cross-compile chisel + spread + docker cli for all target arches inside a
+# single Canonical ubuntu/go:1.25-26.04_edge container. Output binaries land
+# in ./cache/binaries/{chisel,chisel-hacked,spread,docker}-<arch>.
 #
 # Required env vars (set by makefile):
 #   CHISEL_REF        git ref (tag, branch, or SHA) for canonical/chisel
@@ -32,8 +32,9 @@ docker run --rm \
     -e HGID="$HGID" \
     "$GO_BUILDER_IMAGE" -ceuo pipefail '
 # Builder runs natively on host arch and cross-compiles via GOARCH for
-# the other arch. Both binaries are pure Go (no CGO), so cross-compile
+# the other arches. Both binaries are pure Go (no CGO), so cross-compile
 # is clean.
+TARGET_ARCHES="amd64 arm64 s390x ppc64le"
 
 # Canonical ubuntu/go image has /usr/bin/go as a broken symlink in some
 # revisions; pick the actual go binary out of /usr/lib/go-*/bin.
@@ -58,7 +59,7 @@ build() {
     git clone "$repo" "/src/$name"
     cd "/src/$name"
     git checkout "$ref"
-    for arch in amd64 arm64; do
+    for arch in $TARGET_ARCHES; do
         echo "==> building $name for $arch"
         CGO_ENABLED=0 GOOS=linux GOARCH="$arch" \
             go build -trimpath -ldflags "-s -w $extra_ldflags" \
@@ -97,7 +98,7 @@ for p in "${patches[@]}"; do
 done
 CHISEL_HACKED_VERSION_STRING="${CHISEL_VERSION_STRING}-hacked"
 echo "==> chisel-hacked version string: $CHISEL_HACKED_VERSION_STRING"
-for arch in amd64 arm64; do
+for arch in $TARGET_ARCHES; do
     echo "==> building chisel-hacked for $arch"
     CGO_ENABLED=0 GOOS=linux GOARCH="$arch" \
         go build -trimpath -ldflags "-s -w -X github.com/canonical/chisel/cmd.Version=$CHISEL_HACKED_VERSION_STRING" \
@@ -107,29 +108,25 @@ cd /
 
 build spread https://github.com/canonical/spread  "$SPREAD_REF" ./cmd/spread
 
-# Ensure curl + tar present for the docker download step.
-if ! command -v curl >/dev/null; then
-    apt-get update
-    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends curl
-fi
-
-# Fetch the upstream docker static tarball for each target arch and extract
-# only the docker CLI. Upstream binaries are built with a recent go toolchain
-# (1.26+) and so dodge the qemu emulation bug that hits Ubuntu apt'\''s docker.io
-# (built with go 1.24). docker.com uses uname-style arch names: aarch64,
-# x86_64.
-for arch in amd64 arm64; do
-    case "$arch" in
-        amd64) docker_arch=x86_64 ;;
-        arm64) docker_arch=aarch64 ;;
-    esac
-    url="https://download.docker.com/linux/static/stable/${docker_arch}/docker-${DOCKER_VERSION}.tgz"
-    echo "==> downloading docker $DOCKER_VERSION for $arch from $url"
-    tmpdir=$(mktemp -d)
-    curl -fsSL "$url" | tar -xz -C "$tmpdir" docker/docker
-    install -m 0755 "$tmpdir/docker/docker" "/out/docker-$arch"
-    rm -rf "$tmpdir"
+# Build the docker CLI from source for each target arch. docker.com'\''s
+# static tarballs only cover x86_64 / aarch64 at current versions (s390x /
+# ppc64le stopped at 18.06), and Ubuntu apt'\''s docker.io is built with go
+# 1.24 and crashes under qemu emulation -- building from source with the
+# go 1.25 toolchain covers all arches uniformly. docker/cli has no go.mod
+# (vendor.mod + vendor/ instead); the symlink trick puts it in module mode.
+git clone https://github.com/docker/cli /src/docker-cli
+cd /src/docker-cli
+git checkout "v$DOCKER_VERSION"
+ln -s vendor.mod go.mod
+ln -s vendor.sum go.sum
+for arch in $TARGET_ARCHES; do
+    echo "==> building docker cli for $arch"
+    CGO_ENABLED=0 GOOS=linux GOARCH="$arch" \
+        go build -mod=vendor -trimpath \
+        -ldflags "-s -w -X github.com/docker/cli/cli/version.Version=$DOCKER_VERSION" \
+        -o "/out/docker-$arch" ./cmd/docker
 done
+cd /
 
 # Hand ownership back to the invoking user so make can read/replace later.
 chown -R "$HUID:$HGID" /out

inlined/bread-24.04.yaml

@@ -86,6 +86,12 @@ backends:
       - ubuntu-24.04-arm64:
           password: bread
           workers: 2
+      - ubuntu-24.04-s390x:
+          password: bread
+          workers: 2
+      - ubuntu-24.04-ppc64le:
+          password: bread
+          workers: 2
 
 suites:
   tests/:

inlined/bread-25.10.yaml

@@ -86,6 +86,12 @@ backends:
       - ubuntu-25.10-arm64:
           password: bread
           workers: 2
+      - ubuntu-25.10-s390x:
+          password: bread
+          workers: 2
+      - ubuntu-25.10-ppc64le:
+          password: bread
+          workers: 2
 
 suites:
   tests/:

inlined/bread-26.04.yaml

@@ -86,6 +86,12 @@ backends:
       - ubuntu-26.04-arm64:
           password: bread
           workers: 2
+      - ubuntu-26.04-s390x:
+          password: bread
+          workers: 2
+      - ubuntu-26.04-ppc64le:
+          password: bread
+          workers: 2
 
 suites:
   tests/:

inlined/bread-26.10.yaml

@@ -86,6 +86,12 @@ backends:
       - ubuntu-26.10-arm64:
           password: bread
           workers: 2
+      - ubuntu-26.10-s390x:
+          password: bread
+          workers: 2
+      - ubuntu-26.10-ppc64le:
+          password: bread
+          workers: 2
 
 suites:
   tests/:

inlined/bread-chisel-releases-24.04.yaml

@@ -91,6 +91,12 @@ backends:
       - ubuntu-24.04-arm64:
           password: bread
           workers: 4
+      - ubuntu-24.04-s390x:
+          password: bread
+          workers: 4
+      - ubuntu-24.04-ppc64le:
+          password: bread
+          workers: 4
 
 suites:
   tests/spread/integration/:

inlined/bread-chisel-releases-25.10.yaml

@@ -91,6 +91,12 @@ backends:
       - ubuntu-25.10-arm64:
           password: bread
           workers: 4
+      - ubuntu-25.10-s390x:
+          password: bread
+          workers: 4
+      - ubuntu-25.10-ppc64le:
+          password: bread
+          workers: 4
 
 suites:
   tests/spread/integration/:

inlined/bread-chisel-releases-26.04.yaml

@@ -91,6 +91,12 @@ backends:
       - ubuntu-26.04-arm64:
           password: bread
           workers: 4
+      - ubuntu-26.04-s390x:
+          password: bread
+          workers: 4
+      - ubuntu-26.04-ppc64le:
+          password: bread
+          workers: 4
 
 suites:
   tests/spread/integration/: