update to latest buildkit

.claude/settings.local.json

@@ -13,7 +13,19 @@
       "Bash(ls:*)",
       "Bash(git rev-parse:*)",
       "WebFetch(domain:earthly.dev)",
-      "Bash(gh issue view:*)"
+      "Bash(gh issue view:*)",
+      "Bash(go get:*)",
+      "Bash(go mod:*)",
+      "Bash(go build:*)",
+      "Bash(go doc:*)",
+      "Bash(earth +build)",
+      "Bash(gh api:*)",
+      "Bash(cd:*)",
+      "Read(//private/tmp/**)",
+      "Bash(unzip -o ci-logs-435.zip -d ci-logs-435)",
+      "Bash(gh pr:*)",
+      "Bash(awk -F'\\\\t' '$2==\"fail\"')",
+      "Bash(earthly --version)"
     ]
   }
 }

.github/actions/failure-diagnostics/action.yml

@@ -0,0 +1,69 @@
+name: Failure diagnostics
+description: Dump useful host and buildkit state after a CI job failure.
+inputs:
+  BINARY:
+    description: "Container engine to inspect, usually docker or podman."
+    required: true
+  SUDO:
+    description: "Optional sudo prefix."
+    required: false
+    default: ""
+runs:
+  using: composite
+  steps:
+    - shell: bash
+      run: |
+        set +e
+
+        engine="${{ inputs.BINARY }}"
+        sudo_prefix="${{ inputs.SUDO }}"
+
+        run_cmd() {
+          echo "+ $*"
+          "$@"
+        }
+
+        run_shell() {
+          echo "+ $*"
+          bash -lc "$*"
+        }
+
+        echo "::group::host memory, disk and pressure"
+        date -u
+        uname -a
+        run_cmd free -h
+        run_cmd df -h
+        run_cmd uptime
+        run_shell 'test -r /proc/pressure/memory && cat /proc/pressure/memory || true'
+        run_shell 'test -r /proc/pressure/io && cat /proc/pressure/io || true'
+        echo "::endgroup::"
+
+        echo "::group::top processes"
+        run_shell 'ps -eo pid,ppid,stat,pcpu,pmem,rss,vsz,comm,args --sort=-rss | head -80'
+        run_shell 'for pid in $(ps -eo pid= --sort=-rss | head -80); do printf "%s oom_score=%s oom_score_adj=%s\n" "$pid" "$(cat /proc/$pid/oom_score 2>/dev/null || echo unknown)" "$(cat /proc/$pid/oom_score_adj 2>/dev/null || echo unknown)"; done'
+        echo "::endgroup::"
+
+        echo "::group::kernel oom and cgroup messages"
+        $sudo_prefix dmesg -T 2>/dev/null | grep -Ei 'out of memory|oom|killed process|memory cgroup|cgroup' | tail -200 || true
+        echo "::endgroup::"
+
+        echo "::group::container engine state"
+        run_shell "$sudo_prefix $engine version || true"
+        run_shell "$sudo_prefix $engine info || true"
+        run_shell "$sudo_prefix $engine ps -a || true"
+        run_shell "$sudo_prefix $engine stats --no-stream --all || true"
+        echo "::endgroup::"
+
+        echo "::group::buildkit containers"
+        for name in earthly-buildkitd earthly-dev-buildkitd earthly-integration-buildkitd earthly-test-buildkitd; do
+          echo "--- $name inspect ---"
+          $sudo_prefix "$engine" inspect "$name" 2>/dev/null || true
+          echo "--- $name logs tail ---"
+          $sudo_prefix "$engine" logs --tail 300 "$name" 2>&1 || true
+        done
+        echo "::endgroup::"
+
+        echo "::group::earthly directories"
+        run_shell 'du -sh ~/.earthly ~/.earthly-dev 2>/dev/null || true'
+        run_shell 'find ~/.earthly ~/.earthly-dev -maxdepth 2 -type d 2>/dev/null | sort | head -100 || true'
+        echo "::endgroup::"

.github/actions/stage2-setup/action.yml

@@ -39,10 +39,34 @@ inputs:
   DOCKERHUB_MIRROR_PASSWORD:
     description: "Docker Hub mirror password (legacy - GCR mirror is public)"
     required: false
+  MORE_MEMORY:
+    description: "Add 8GB swap to increase available memory (useful for Podman)"
+    required: false
+    default: 'false'
 
 runs:
   using: "composite"
   steps:
+    - name: Add swap for extra memory headroom
+      # build-earthly always adds 12G swap; stage2-setup previously only did
+      # so for podman / MORE_MEMORY. Tests on docker runners fail with
+      # earthly "Canceled" mid-build and runc "file already closed" errors,
+      # consistent with silent OOM-killed buildkit-runc children on the
+      # 16 GiB runner. Mirror build-earthly and add swap unconditionally so
+      # the test path has the same headroom as the build path.
+      shell: bash
+      run: |
+        # The 12G swapfile alone nearly fills the runner's ~14G free root
+        # disk, causing "No space left on device" build failures. Reclaim
+        # ~25G of preinstalled toolchains we never use before allocating.
+        sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \
+          /usr/local/.ghcup /opt/hostedtoolcache/CodeQL
+        df -h / | tail -1
+        sudo fallocate -l 12G /extra-swapfile
+        sudo chmod 600 /extra-swapfile
+        sudo mkswap /extra-swapfile
+        sudo swapon /extra-swapfile
+        echo "Swap added: $(swapon --show)"
     - name: Unset CI
       shell: bash
       run: |
@@ -58,14 +82,16 @@ runs:
           echo "is_fork=false" >> $GITHUB_OUTPUT
           echo "Running on main repo - will use GHCR"
         fi
-    - name: Download artifacts (fork)
-      if: steps.fork-check.outputs.is_fork == 'true'
+    - name: Download build-earthly artifacts
+      # Always download — the artifact contains at minimum the
+      # buildkitd-image tarball (every run) and on fork PRs also the
+      # earthly binary + tag-suffix.txt. Loading the local tarball
+      # avoids a per-job ghcr pull of the ~300 MB staging image.
       uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8
       with:
         name: earthly-build-ubuntu-latest-${{ inputs.BINARY }}${{ inputs.USE_NEXT == 'true' && '-ticktock' || '' }}
         path: ./fork-artifacts
-    - name: Load buildkitd image from artifact (fork)
-      if: steps.fork-check.outputs.is_fork == 'true'
+    - name: Load buildkitd image from artifact
       shell: bash
       run: |
         echo "Loading buildkitd image from artifact..."
@@ -115,6 +141,10 @@ runs:
         password: ${{ env.GITHUB_TOKEN }}
     - name: Login to Docker Hub
       if: inputs.BINARY == 'docker' && inputs.DOCKERHUB_PASSWORD != '' && steps.fork-check.outputs.is_fork != 'true'
+      # Hub auth only raises pull rate limits (the GCR mirror serves most
+      # pulls); a transient registry-1.docker.io timeout must not fail the
+      # whole job.
+      continue-on-error: true
       uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
       with:
         username: ${{ inputs.DOCKERHUB_USERNAME }}
@@ -159,11 +189,19 @@ runs:
       # qemu-user-static needed for cross-compilation (--platform) targets
       run: ${{inputs.SUDO}} apt-get update && ${{inputs.SUDO}} apt-get install -y qemu-user-static
       shell: bash
-    - name: Set fixed buildkitd image for Docker 29+ (non-fork)
+    - name: Point outer buildkit at the PR's own staging image (non-fork)
+      # Previously hardcoded to buildkitd-v0.8.17-fix.5 (the stable pinned
+      # image). That meant CI was testing the PR's earthly CLI against the
+      # *old* buildkit on the outer layer, and only exercising the PR's
+      # buildkit inside earthly-in-earthly tests. Use the staging tag that
+      # build-earthly pushed for this SHA so the outer layer covers the
+      # PR's buildkit too — matches what the fork path already does above.
       if: steps.fork-check.outputs.is_fork != 'true'
       shell: bash
       run: |
-        echo "EARTHLY_BUILDKIT_IMAGE=ghcr.io/earthbuild/earthbuild:buildkitd-v0.8.17-fix.1" >> $GITHUB_ENV
+        TAG_SUFFIX="ubuntu-latest-${{inputs.BINARY}}"
+        if [ "${{inputs.USE_NEXT}}" = "true" ]; then TAG_SUFFIX="$TAG_SUFFIX-ticktock"; fi
+        echo "EARTHLY_BUILDKIT_IMAGE=ghcr.io/earthbuild/earthbuild:buildkitd-staging-${{ github.sha }}-${TAG_SUFFIX}" >> $GITHUB_ENV
     - name: Get binary from GHCR (non-fork)
       if: steps.fork-check.outputs.is_fork != 'true'
       run: |-
@@ -190,6 +228,9 @@ runs:
         (strings ${{inputs.BUILT_EARTHLY_PATH}} | grep "$expected_buildkit_client_sha" ) || ( echo "expected to find $expected_buildkit_client_sha in earthly binary" && exit 1)
         echo "correctly found $expected_buildkit_client_sha in earthly binary; this confirms earthly-next was used"
       shell: bash
+    - name: Limit buildkit parallelism to avoid OOM on CI runners
+      run: ${{inputs.SUDO}} ${{inputs.BUILT_EARTHLY_PATH}} config global.buildkit_max_parallelism 1
+      shell: bash
     - if: ${{ inputs.BINARY == 'podman' }}
       run: ${{inputs.SUDO}} ${{inputs.BUILT_EARTHLY_PATH}} bootstrap
       shell: bash