From f9215daad768c868a880c5dc4d257e6174e3e187 Mon Sep 17 00:00:00 2001 From: Sergey Zelenov Date: Mon, 8 Jun 2026 10:26:20 +0200 Subject: [PATCH 1/6] ci: add retry logic and caching for zstd source download The zstd source tarball is downloaded from GitHub releases via an unauthenticated curl in etc/install-zstd.sh. When many CI jobs run in parallel they hit GitHub's rate limit, receiving a ~92-byte error response that causes tar to fail with "not in gzip format". - Add --retry 5 / --retry-delay 5 / --retry-all-errors / --fail to the curl invocation so rate-limited requests are retried rather than silently producing a corrupt archive - Download to a temp file before extracting so curl and tar failures are decoupled and clearly attributed - Make the script idempotent: skip download and build if deps/zstd/out already exists (enables cache restoration) - Add actions/cache@v5 for host tests, keyed by OS/arch and hashes of package.json + package-lock.json; all parallel matrix jobs share the cache so only the first runner downloads - Add Docker Buildx GHA cache (--cache-from/--cache-to type=gha) to container tests with per arch+node scopes - Remove --no-cache from the musl buildx command which was explicitly defeating layer caching - Restructure both Dockerfiles to COPY package.json, package-lock.json and the install script before running npm run install-zstd, then COPY the rest; this pins the zstd download layer to those files only so it is not invalidated on every source commit --- .github/docker/Dockerfile.glibc | 8 +++++++- .github/docker/Dockerfile.musl | 12 ++++++++++-- .github/workflows/test.yml | 12 +++++++++++- etc/install-zstd.sh | 32 ++++++++++++++++++++++++-------- 4 files changed, 52 insertions(+), 12 deletions(-) diff --git a/.github/docker/Dockerfile.glibc b/.github/docker/Dockerfile.glibc index f5d9500..3775dc8 100644 --- a/.github/docker/Dockerfile.glibc +++ b/.github/docker/Dockerfile.glibc @@ -7,14 +7,20 @@ RUN mkdir -p /nodejs && tar -xzf /node-v${NODE_VERSION}-linux-${NODE_ARCH}.tar.g ENV PATH=$PATH:/nodejs/bin WORKDIR /zstd -COPY . . RUN yum install -y python39 git make curl cmake gcc-toolset-14 ENV PATH=/opt/rh/gcc-toolset-14/root/bin/:$PATH RUN python3 --version +# Copy only what's needed to download and build the zstd source, so this +# layer is only invalidated when the zstd version or install script changes, +# not on every source commit. +COPY package.json package-lock.json . +COPY etc/install-zstd.sh etc/ RUN npm run install-zstd + +COPY . . RUN npm install RUN npm run prebuild diff --git a/.github/docker/Dockerfile.musl b/.github/docker/Dockerfile.musl index 7a4edd0..58d08fe 100644 --- a/.github/docker/Dockerfile.musl +++ b/.github/docker/Dockerfile.musl @@ -5,10 +5,18 @@ ARG NODE_VERSION=20.19.0 FROM ${PLATFORM}/node:${NODE_VERSION}-alpine AS build WORKDIR /zstd -COPY . . RUN apk --no-cache add make g++ libc-dev curl bash python3 py3-pip cmake -RUN npm run install-zstd && npm i + +# Copy only what's needed to download and build the zstd source, so this +# layer is only invalidated when the zstd version or install script changes, +# not on every source commit. +COPY package.json package-lock.json . +COPY etc/install-zstd.sh etc/ +RUN npm run install-zstd + +COPY . . +RUN npm i RUN npm run prebuild ARG RUN_TEST diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 73538be..a9f200c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,6 +24,12 @@ jobs: cache: "npm" registry-url: "https://registry.npmjs.org" + - name: Cache zstd build + uses: actions/cache@v5 + with: + path: deps + key: zstd-deps-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('package.json', 'package-lock.json') }} + - name: Install zstd run: npm run install-zstd shell: bash @@ -66,6 +72,8 @@ jobs: run: | docker buildx create --name builder --bootstrap --use docker buildx build \ + --cache-from type=gha,scope=glibc-${{ matrix.linux_arch }}-${{ matrix.node }} \ + --cache-to type=gha,mode=max,scope=glibc-${{ matrix.linux_arch }}-${{ matrix.node }} \ --platform linux/${{ matrix.linux_arch }} \ --build-arg="NODE_ARCH=${{ matrix.linux_arch == 'amd64' && 'x64' || matrix.linux_arch }}" \ --build-arg="NODE_VERSION=${{ steps.get_nodejs_version.outputs.version }}" \ @@ -103,7 +111,9 @@ jobs: - name: Run Buildx run: | docker buildx create --name builder --bootstrap --use - docker --debug buildx build --progress=plain --no-cache \ + docker buildx build \ + --cache-from type=gha,scope=musl-${{ matrix.linux_arch }}-${{ matrix.node }} \ + --cache-to type=gha,mode=max,scope=musl-${{ matrix.linux_arch }}-${{ matrix.node }} \ --platform linux/${{ matrix.linux_arch }} \ --build-arg="PLATFORM=${{ matrix.linux_arch == 'arm64' && 'arm64v8' || matrix.linux_arch }}" \ --build-arg="NODE_VERSION=${{ steps.get_nodejs_version.outputs.version }}" \ diff --git a/etc/install-zstd.sh b/etc/install-zstd.sh index 1f3bae8..eb997f8 100644 --- a/etc/install-zstd.sh +++ b/etc/install-zstd.sh @@ -12,14 +12,25 @@ download_zstd() { # only unpack the source and build files needed to compile the project necessary_files="zstd-$ZSTD_VERSION/build zstd-$ZSTD_VERSION/lib zstd-$ZSTD_VERSION/programs" - + + TMPFILE=$(mktemp) # flags # -L follow redirects - # -C output directory - # - tar from stdin + # -o download to file (decouples curl errors from tar) + # --fail exit non-zero on HTTP 4xx/5xx so --retry applies + # --retry retry up to 5 times (GitHub releases rate-limits concurrent CI jobs) + # --retry-delay wait between retries + # --retry-all-errors retry on any error, not just connection failures (curl 7.71+) # --strip-components ignore the top-level directory when unpacking - curl -L "https://github.com/facebook/zstd/releases/download/v$ZSTD_VERSION/zstd-$ZSTD_VERSION.tar.gz" \ - | tar -zxf - -C deps/zstd --strip-components 1 $necessary_files + curl -L \ + --fail \ + --retry 5 \ + --retry-delay 5 \ + --retry-all-errors \ + -o "$TMPFILE" \ + "https://github.com/facebook/zstd/releases/download/v$ZSTD_VERSION/zstd-$ZSTD_VERSION.tar.gz" + tar -zxf "$TMPFILE" -C deps/zstd --strip-components 1 $necessary_files + rm -f "$TMPFILE" } build_zstd() { @@ -42,6 +53,11 @@ build_zstd() { cmake --build . --target libzstd_static --config Release } -clean_deps -download_zstd -build_zstd \ No newline at end of file +# If a previous build is restored from cache, skip everything. +if [ -d "deps/zstd/out" ]; then + echo "deps/zstd already built, skipping download and build" +else + clean_deps + download_zstd + build_zstd +fi \ No newline at end of file From df6f522fc31d7b58b52f78d1f6ef28945e25973b Mon Sep 17 00:00:00 2001 From: Sergey Zelenov Date: Mon, 8 Jun 2026 10:32:40 +0200 Subject: [PATCH 2/6] ci: use shell retry loop instead of --retry-all-errors for curl compatibility --- etc/install-zstd.sh | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/etc/install-zstd.sh b/etc/install-zstd.sh index eb997f8..6e678c4 100644 --- a/etc/install-zstd.sh +++ b/etc/install-zstd.sh @@ -13,22 +13,24 @@ download_zstd() { # only unpack the source and build files needed to compile the project necessary_files="zstd-$ZSTD_VERSION/build zstd-$ZSTD_VERSION/lib zstd-$ZSTD_VERSION/programs" + # Download to a file before extracting so curl and tar failures are decoupled. + # Retry loop is used instead of --retry-all-errors because UBI8 ships curl 7.61 + # which predates that flag (added in 7.71). --fail makes curl exit non-zero on + # HTTP 4xx/5xx (e.g. GitHub rate limiting concurrent CI jobs). TMPFILE=$(mktemp) - # flags - # -L follow redirects - # -o download to file (decouples curl errors from tar) - # --fail exit non-zero on HTTP 4xx/5xx so --retry applies - # --retry retry up to 5 times (GitHub releases rate-limits concurrent CI jobs) - # --retry-delay wait between retries - # --retry-all-errors retry on any error, not just connection failures (curl 7.71+) - # --strip-components ignore the top-level directory when unpacking - curl -L \ - --fail \ - --retry 5 \ - --retry-delay 5 \ - --retry-all-errors \ - -o "$TMPFILE" \ + ATTEMPTS=0 + until curl -L --fail -o "$TMPFILE" \ "https://github.com/facebook/zstd/releases/download/v$ZSTD_VERSION/zstd-$ZSTD_VERSION.tar.gz" + do + ATTEMPTS=$((ATTEMPTS + 1)) + if [ "$ATTEMPTS" -ge 5 ]; then + echo "Failed to download zstd after $ATTEMPTS attempts, giving up" + rm -f "$TMPFILE" + exit 1 + fi + echo "Download failed, retrying in 5s (attempt $ATTEMPTS/5)..." + sleep 5 + done tar -zxf "$TMPFILE" -C deps/zstd --strip-components 1 $necessary_files rm -f "$TMPFILE" } From f8cef2cdc5bdab2def7d059f4cdbd71c04645397 Mon Sep 17 00:00:00 2001 From: Sergey Zelenov Date: Mon, 8 Jun 2026 10:34:11 +0200 Subject: [PATCH 3/6] ci: include install script in cache key --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a9f200c..3508edc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,7 +28,7 @@ jobs: uses: actions/cache@v5 with: path: deps - key: zstd-deps-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('package.json', 'package-lock.json') }} + key: zstd-deps-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('package.json', 'package-lock.json', 'etc/install-zstd.sh') }} - name: Install zstd run: npm run install-zstd From 2a00b3d3fd60e3787168478f2dcbfc9f19fe24b4 Mon Sep 17 00:00:00 2001 From: Sergey Zelenov Date: Mon, 8 Jun 2026 11:31:36 +0200 Subject: [PATCH 4/6] ci: add 30m timeout to container test jobs to fail fast on QEMU hangs --- .github/workflows/test.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3508edc..e070d77 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -44,6 +44,9 @@ jobs: container_tests_glibc: runs-on: ubuntu-latest + # QEMU-emulated builds occasionally hang; fail fast rather than burning the + # 6-hour default timeout. Successful runs complete in ~15 min. + timeout-minutes: 30 strategy: matrix: linux_arch: [s390x, arm64, amd64] @@ -84,6 +87,9 @@ jobs: container_tests_musl: runs-on: ubuntu-latest + # QEMU-emulated builds occasionally hang; fail fast rather than burning the + # 6-hour default timeout. Successful runs complete in ~15 min. + timeout-minutes: 30 strategy: matrix: linux_arch: [amd64, arm64] From 4cd65b9a09fcf80fc817c00bf4a187089c6d91f3 Mon Sep 17 00:00:00 2001 From: Sergey Zelenov Date: Mon, 8 Jun 2026 12:24:51 +0200 Subject: [PATCH 5/6] ci: drop buildx gha cache for container jobs The type=gha cache never functioned: raw `docker buildx build` does not receive the Actions cache-service token env vars (only build-push-action or third-party token-export actions provide them), so import/export silently no-op'd and every container build ran from scratch anyway. Rather than pull in a third-party action to expose the runtime token, drop container-layer caching entirely. The download flakiness that originally broke CI is fixed by the retry loop in install-zstd.sh, and QEMU hangs by timeout-minutes; layer caching was only a speed optimization. Revert the Dockerfile COPY reordering since its sole purpose was cache granularity. Host-job actions/cache (which does work) and the container timeout are retained. --- .github/docker/Dockerfile.glibc | 8 +------- .github/docker/Dockerfile.musl | 12 ++---------- .github/workflows/test.yml | 6 +----- 3 files changed, 4 insertions(+), 22 deletions(-) diff --git a/.github/docker/Dockerfile.glibc b/.github/docker/Dockerfile.glibc index 3775dc8..f5d9500 100644 --- a/.github/docker/Dockerfile.glibc +++ b/.github/docker/Dockerfile.glibc @@ -7,20 +7,14 @@ RUN mkdir -p /nodejs && tar -xzf /node-v${NODE_VERSION}-linux-${NODE_ARCH}.tar.g ENV PATH=$PATH:/nodejs/bin WORKDIR /zstd +COPY . . RUN yum install -y python39 git make curl cmake gcc-toolset-14 ENV PATH=/opt/rh/gcc-toolset-14/root/bin/:$PATH RUN python3 --version -# Copy only what's needed to download and build the zstd source, so this -# layer is only invalidated when the zstd version or install script changes, -# not on every source commit. -COPY package.json package-lock.json . -COPY etc/install-zstd.sh etc/ RUN npm run install-zstd - -COPY . . RUN npm install RUN npm run prebuild diff --git a/.github/docker/Dockerfile.musl b/.github/docker/Dockerfile.musl index 58d08fe..7a4edd0 100644 --- a/.github/docker/Dockerfile.musl +++ b/.github/docker/Dockerfile.musl @@ -5,18 +5,10 @@ ARG NODE_VERSION=20.19.0 FROM ${PLATFORM}/node:${NODE_VERSION}-alpine AS build WORKDIR /zstd +COPY . . RUN apk --no-cache add make g++ libc-dev curl bash python3 py3-pip cmake - -# Copy only what's needed to download and build the zstd source, so this -# layer is only invalidated when the zstd version or install script changes, -# not on every source commit. -COPY package.json package-lock.json . -COPY etc/install-zstd.sh etc/ -RUN npm run install-zstd - -COPY . . -RUN npm i +RUN npm run install-zstd && npm i RUN npm run prebuild ARG RUN_TEST diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e070d77..7698e11 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -75,8 +75,6 @@ jobs: run: | docker buildx create --name builder --bootstrap --use docker buildx build \ - --cache-from type=gha,scope=glibc-${{ matrix.linux_arch }}-${{ matrix.node }} \ - --cache-to type=gha,mode=max,scope=glibc-${{ matrix.linux_arch }}-${{ matrix.node }} \ --platform linux/${{ matrix.linux_arch }} \ --build-arg="NODE_ARCH=${{ matrix.linux_arch == 'amd64' && 'x64' || matrix.linux_arch }}" \ --build-arg="NODE_VERSION=${{ steps.get_nodejs_version.outputs.version }}" \ @@ -117,9 +115,7 @@ jobs: - name: Run Buildx run: | docker buildx create --name builder --bootstrap --use - docker buildx build \ - --cache-from type=gha,scope=musl-${{ matrix.linux_arch }}-${{ matrix.node }} \ - --cache-to type=gha,mode=max,scope=musl-${{ matrix.linux_arch }}-${{ matrix.node }} \ + docker --debug buildx build --progress=plain --no-cache \ --platform linux/${{ matrix.linux_arch }} \ --build-arg="PLATFORM=${{ matrix.linux_arch == 'arm64' && 'arm64v8' || matrix.linux_arch }}" \ --build-arg="NODE_VERSION=${{ steps.get_nodejs_version.outputs.version }}" \ From c21afbb170b520cb032975d548dd30755e7dc996 Mon Sep 17 00:00:00 2001 From: Sergey Zelenov Date: Wed, 17 Jun 2026 18:16:19 +0200 Subject: [PATCH 6/6] ci: use EXIT trap to clean up tmpfile on curl/tar failure --- etc/install-zstd.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/etc/install-zstd.sh b/etc/install-zstd.sh index 6e678c4..112addd 100644 --- a/etc/install-zstd.sh +++ b/etc/install-zstd.sh @@ -18,6 +18,7 @@ download_zstd() { # which predates that flag (added in 7.71). --fail makes curl exit non-zero on # HTTP 4xx/5xx (e.g. GitHub rate limiting concurrent CI jobs). TMPFILE=$(mktemp) + trap 'rm -f "$TMPFILE"' EXIT ATTEMPTS=0 until curl -L --fail -o "$TMPFILE" \ "https://github.com/facebook/zstd/releases/download/v$ZSTD_VERSION/zstd-$ZSTD_VERSION.tar.gz" @@ -25,14 +26,12 @@ download_zstd() { ATTEMPTS=$((ATTEMPTS + 1)) if [ "$ATTEMPTS" -ge 5 ]; then echo "Failed to download zstd after $ATTEMPTS attempts, giving up" - rm -f "$TMPFILE" exit 1 fi echo "Download failed, retrying in 5s (attempt $ATTEMPTS/5)..." sleep 5 done tar -zxf "$TMPFILE" -C deps/zstd --strip-components 1 $necessary_files - rm -f "$TMPFILE" } build_zstd() {