feat: chisel v1.5.0 + chisel-releases rootfs sweep

.github/workflows/release.yaml

@@ -140,7 +140,7 @@ jobs:
             chmod +x /usr/local/bin/spread
             ```
 
-            **precompiled chisel-hacked binaries** -- chisel v1.4.2 with
+            **precompiled chisel-hacked binaries** -- chisel v1.5.0 with
             CHISEL_HACKS patches applied. when `CHISEL_HACKS=1` is set,
             skips release maintenance checks and release validation -- used
             by chisel-releases tests against in-development releases.

README.md

@@ -23,7 +23,7 @@ prebuilt docker images + ready-to-use spread yamls so you can drop Canonical [`s
 two image flavours, each for ubuntu 22.04, 24.04, 25.10, 26.04, 26.10 x amd64 / arm64 / s390x / ppc64le, published as multiarch tags at `ghcr.io/lczyk/spread-bread/<flavour>:<ver>`. heads-up: s390x + ppc64le images build under qemu and ship untested (no native runners); amd64 + arm64 are tested on every release:
 
 - **`bread`** -- base: ubuntu + sshd. general-purpose spread system; the test suite installs whatever else it needs.
-- **`bread-chisel-releases`** -- bread + `chisel` + the shell + container tooling typically needed by [chisel-releases](https://github.com/canonical/chisel-releases) spread tests (curl, wget, git, jq, file, sudo, tree, docker, skopeo). `chisel` and `docker` are built from source (canonical/chisel pinned by SHA, docker/cli pinned by version tag) so the bundled binaries are go 1.25+ and survive qemu emulation.
+- **`bread-chisel-releases`** -- bread + `chisel` + the shell + container tooling typically needed by [chisel-releases](https://github.com/canonical/chisel-releases) spread tests (curl, wget, git, jq, file, sudo, tree, docker, skopeo). `chisel` and `docker` are built from source (canonical/chisel + docker/cli pinned by version tag) so the bundled binaries are go 1.25+ and survive qemu emulation.
 
 ## using (the common case)
 
@@ -114,6 +114,7 @@ spread-bread/
     Dockerfile.bread-test-26.04  # test-host image: bread:26.04 + docker + spread (not published)
     contract-{bread,bread-chisel-releases}/run/task.yaml
     _inner-{bread,bread-chisel-releases}/contract/task.yaml
+    lib/clean-rootfs             # stand-in for the chisel-releases lib the inlined yamls' restore-each calls
   .github/workflows/             # ci (build + test) on PR / push to main; release on r* tag; daily base-refresh
 ```
 

hack/build_binaries.sh

@@ -49,8 +49,12 @@ go version
 
 # Ensure git is available (the ubuntu/go image may not ship it).
 if ! command -v git >/dev/null; then
+    # Docker Desktop mounts the host bundle over this path read-only, which
+    # the ca-certificates postinst cannot replace.
+    pkgs="git"
+    [ -r /etc/ssl/certs/ca-certificates.crt ] || pkgs="$pkgs ca-certificates"
     apt-get update
-    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends git ca-certificates
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends $pkgs
 fi
 
 # Build helper. extra_ldflags is appended to the standard "-s -w".

inlined/bread-chisel-releases-22.04.yaml

@@ -5,6 +5,8 @@ path: /chisel-releases
 environment:
   PROJECT_PATH: /chisel-releases
   SHARED_LIBRARIES: $PROJECT_PATH/tests/spread/lib
+  # Rootfs'es share one directory so that clean-rootfs can sweep them between tasks.
+  ROOTFS_DIR: /tmp/chisel-rootfs
   PATH: /snap/bin:$PATH:$SHARED_LIBRARIES
   # NOTE: empty by default; opt in per-run with `CHISEL_HACKS=1 spread ...`.
   # $(HOST:...) forwards the host env var into the remote (spread sends only
@@ -194,6 +196,11 @@ backends:
           password: bread
           workers: 4
 
+# One instance serves every task of a run, so rootfs'es have to go as they finish.
+restore-each: clean-rootfs
+
 suites:
   tests/spread/integration/:
     summary: Tests common scenarios
+
+kill-timeout: 1.5h

inlined/bread-chisel-releases-24.04.yaml

@@ -5,6 +5,8 @@ path: /chisel-releases
 environment:
   PROJECT_PATH: /chisel-releases
   SHARED_LIBRARIES: $PROJECT_PATH/tests/spread/lib
+  # Rootfs'es share one directory so that clean-rootfs can sweep them between tasks.
+  ROOTFS_DIR: /tmp/chisel-rootfs
   PATH: /snap/bin:$PATH:$SHARED_LIBRARIES
   # NOTE: empty by default; opt in per-run with `CHISEL_HACKS=1 spread ...`.
   # $(HOST:...) forwards the host env var into the remote (spread sends only
@@ -194,6 +196,11 @@ backends:
           password: bread
           workers: 4
 
+# One instance serves every task of a run, so rootfs'es have to go as they finish.
+restore-each: clean-rootfs
+
 suites:
   tests/spread/integration/:
     summary: Tests common scenarios
+
+kill-timeout: 1.5h

inlined/bread-chisel-releases-25.10.yaml

@@ -197,3 +197,5 @@ backends:
 suites:
   tests/spread/integration/:
     summary: Tests common scenarios
+
+kill-timeout: 1.5h

inlined/bread-chisel-releases-26.04.yaml

@@ -5,6 +5,8 @@ path: /chisel-releases
 environment:
   PROJECT_PATH: /chisel-releases
   SHARED_LIBRARIES: $PROJECT_PATH/tests/spread/lib
+  # Rootfs'es share one directory so that clean-rootfs can sweep them between tasks.
+  ROOTFS_DIR: /tmp/chisel-rootfs
   PATH: /snap/bin:$PATH:$SHARED_LIBRARIES
   # NOTE: empty by default; opt in per-run with `CHISEL_HACKS=1 spread ...`.
   # $(HOST:...) forwards the host env var into the remote (spread sends only
@@ -194,6 +196,11 @@ backends:
           password: bread
           workers: 4
 
+# One instance serves every task of a run, so rootfs'es have to go as they finish.
+restore-each: clean-rootfs
+
 suites:
   tests/spread/integration/:
     summary: Tests common scenarios
+
+kill-timeout: 1.5h

inlined/bread-chisel-releases-26.10.yaml

@@ -5,6 +5,8 @@ path: /chisel-releases
 environment:
   PROJECT_PATH: /chisel-releases
   SHARED_LIBRARIES: $PROJECT_PATH/tests/spread/lib
+  # Rootfs'es share one directory so that clean-rootfs can sweep them between tasks.
+  ROOTFS_DIR: /tmp/chisel-rootfs
   PATH: /snap/bin:$PATH:$SHARED_LIBRARIES
   # NOTE: empty by default; opt in per-run with `CHISEL_HACKS=1 spread ...`.
   # $(HOST:...) forwards the host env var into the remote (spread sends only
@@ -194,6 +196,11 @@ backends:
           password: bread
           workers: 4
 
+# One instance serves every task of a run, so rootfs'es have to go as they finish.
+restore-each: clean-rootfs
+
 suites:
   tests/spread/integration/:
     summary: Tests common scenarios
+
+kill-timeout: 1.5h

makefile

@@ -5,13 +5,13 @@ DOCKER ?= docker
 
 # Cross-compiled go binaries baked into bread-chisel-releases + bread-test.
 # Pinned to specific upstream commits for reproducibility.
-CHISEL_REF := v1.4.2
+CHISEL_REF := v1.5.0
 SPREAD_REF := 9fdce848027b944a50d25ed2271f17c213b44bd5
 GO_BUILDER_IMAGE := ubuntu/go:1.25-26.04_edge
 # Docker CLI built from docker/cli source at tag v$(DOCKER_VERSION); ubuntu
 # apt's docker.io is built with go 1.24 and crashes under qemu emulation, and
 # docker.com's static tarballs don't cover s390x / ppc64le.
-DOCKER_VERSION := 29.5.2
+DOCKER_VERSION := 29.8.0
 
 # Full matrix.
 VERSIONS := 22.04 24.04 25.10 26.04 26.10

templates/bread-chisel-releases-22.04.yaml.in

@@ -5,6 +5,8 @@ path: /chisel-releases
 environment:
   PROJECT_PATH: /chisel-releases
   SHARED_LIBRARIES: $PROJECT_PATH/tests/spread/lib
+  # Rootfs'es share one directory so that clean-rootfs can sweep them between tasks.
+  ROOTFS_DIR: /tmp/chisel-rootfs
   PATH: /snap/bin:$PATH:$SHARED_LIBRARIES
   # NOTE: empty by default; opt in per-run with `CHISEL_HACKS=1 spread ...`.
   # $(HOST:...) forwards the host env var into the remote (spread sends only
@@ -39,6 +41,11 @@ backends:
           password: bread
           workers: 4
 
+# One instance serves every task of a run, so rootfs'es have to go as they finish.
+restore-each: clean-rootfs
+
 suites:
   tests/spread/integration/:
     summary: Tests common scenarios
+
+kill-timeout: 1.5h

templates/bread-chisel-releases-24.04.yaml.in

@@ -5,6 +5,8 @@ path: /chisel-releases
 environment:
   PROJECT_PATH: /chisel-releases
   SHARED_LIBRARIES: $PROJECT_PATH/tests/spread/lib
+  # Rootfs'es share one directory so that clean-rootfs can sweep them between tasks.
+  ROOTFS_DIR: /tmp/chisel-rootfs
   PATH: /snap/bin:$PATH:$SHARED_LIBRARIES
   # NOTE: empty by default; opt in per-run with `CHISEL_HACKS=1 spread ...`.
   # $(HOST:...) forwards the host env var into the remote (spread sends only
@@ -39,6 +41,11 @@ backends:
           password: bread
           workers: 4
 
+# One instance serves every task of a run, so rootfs'es have to go as they finish.
+restore-each: clean-rootfs
+
 suites:
   tests/spread/integration/:
     summary: Tests common scenarios
+
+kill-timeout: 1.5h

templates/bread-chisel-releases-25.10.yaml.in

@@ -42,3 +42,5 @@ backends:
 suites:
   tests/spread/integration/:
     summary: Tests common scenarios
+
+kill-timeout: 1.5h

templates/bread-chisel-releases-26.04.yaml.in

@@ -5,6 +5,8 @@ path: /chisel-releases
 environment:
   PROJECT_PATH: /chisel-releases
   SHARED_LIBRARIES: $PROJECT_PATH/tests/spread/lib
+  # Rootfs'es share one directory so that clean-rootfs can sweep them between tasks.
+  ROOTFS_DIR: /tmp/chisel-rootfs
   PATH: /snap/bin:$PATH:$SHARED_LIBRARIES
   # NOTE: empty by default; opt in per-run with `CHISEL_HACKS=1 spread ...`.
   # $(HOST:...) forwards the host env var into the remote (spread sends only
@@ -39,6 +41,11 @@ backends:
           password: bread
           workers: 4
 
+# One instance serves every task of a run, so rootfs'es have to go as they finish.
+restore-each: clean-rootfs
+
 suites:
   tests/spread/integration/:
     summary: Tests common scenarios
+
+kill-timeout: 1.5h

templates/bread-chisel-releases-26.10.yaml.in

@@ -5,6 +5,8 @@ path: /chisel-releases
 environment:
   PROJECT_PATH: /chisel-releases
   SHARED_LIBRARIES: $PROJECT_PATH/tests/spread/lib
+  # Rootfs'es share one directory so that clean-rootfs can sweep them between tasks.
+  ROOTFS_DIR: /tmp/chisel-rootfs
   PATH: /snap/bin:$PATH:$SHARED_LIBRARIES
   # NOTE: empty by default; opt in per-run with `CHISEL_HACKS=1 spread ...`.
   # $(HOST:...) forwards the host env var into the remote (spread sends only
@@ -39,6 +41,11 @@ backends:
           password: bread
           workers: 4
 
+# One instance serves every task of a run, so rootfs'es have to go as they finish.
+restore-each: clean-rootfs
+
 suites:
   tests/spread/integration/:
     summary: Tests common scenarios
+
+kill-timeout: 1.5h

tests/_inner-bread-chisel-releases/contract/task.yaml

@@ -23,20 +23,28 @@ execute: |
   fi
 
   # only the hacked chisel is shipped; `chisel` is a symlink to chisel-hacked,
-  # so `chisel --version` also reports the -hacked suffix (pinned v1.4.2).
-  chisel --version | grep -q "v1\.4\.2-hacked" || {
+  # so `chisel --version` also reports the -hacked suffix (pinned v1.5.0).
+  chisel --version | grep -q "v1\.5\.0-hacked" || {
       echo "chisel binary missing or wrong version"
       chisel --version || true
       exit 1
   }
 
   # chisel-hacked: same base version, -hacked suffix in version string.
-  chisel-hacked --version | grep -q "v1\.4\.2-hacked" || {
+  chisel-hacked --version | grep -q "v1\.5\.0-hacked" || {
       echo "chisel-hacked binary missing or wrong version"
       chisel-hacked --version || true
       exit 1
   }
 
+  # chisel-releases' install-slices refuses to run without ROOTFS_DIR; the
+  # 25.10 branch predates it. The restore-each hook that sweeps it is checked
+  # by spread itself, which fails the task when clean-rootfs does not resolve.
+  if [ "$expected_ver" != 25.10 ] && [ "${ROOTFS_DIR:-}" != /tmp/chisel-rootfs ]; then
+      echo "ROOTFS_DIR: expected=/tmp/chisel-rootfs got=${ROOTFS_DIR:-unset}"
+      exit 1
+  fi
+
   for cmd in curl wget git jq file sudo tree docker skopeo; do
       command -v "$cmd" >/dev/null || {
           echo "missing command: $cmd"

tests/contract-bread-chisel-releases/run/task.yaml

@@ -23,6 +23,10 @@ execute: |
   mkdir -p "$work/tests/spread/integration"
   cp -r "$REPO/tests/_inner-bread-chisel-releases/." "$work/tests/spread/integration"
 
+  # The yaml's restore-each hook calls clean-rootfs from tests/spread/lib.
+  mkdir -p "$work/tests/spread/lib"
+  cp "$REPO/tests/lib/clean-rootfs" "$work/tests/spread/lib/"
+
   cd "$work"
 
   # Filter inner spread to outer's arch -- ci per-arch jobs only build

tests/lib/clean-rootfs

@@ -0,0 +1,7 @@
+#!/bin/bash -e
+# Stand-in for chisel-releases' tests/spread/lib/clean-rootfs, so the
+# restore-each hook in the inlined bread-chisel-releases yamls resolves
+# inside the contract suite.
+
+: "${ROOTFS_DIR:?ROOTFS_DIR is not set}"
+rm -rf -- "${ROOTFS_DIR:?}"/*