From b4e16b963ffec22398a4c32d8aea82f51a1ac428 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Tue, 1 Sep 2026 08:55:35 +0200 Subject: [PATCH 1/5] fix(ci): stop the e2e teardown from failing a green suite (#11816) * fix(ci): remove the e2e container before removing its image `docker stop` returns as soon as the container exits, but the daemon reaps a `--rm` container asynchronously after that. The `docker rmi localai-tests` that follows teardown-e2e then loses the race against the reaper and fails with "conflict: ... is using its referenced image", so make exits 1 and the job goes red after every spec has passed. This is why the E2E Backend Tests job fails at random across pull requests. Runs 33435319093, 33435332991, 33412165884 and 33444669207 all report "SUCCESS! -- 235 Passed | 0 Failed" and then die in teardown. `docker rm -f` is synchronous, so the image reference is gone before teardown-e2e returns. It also covers the case where no container is running, which `docker stop` could not because it rejects an empty argument list. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude:claude-opus-5 [Claude Code] * fix(ci): open a tmate session only when a PR asks for one The tmate step runs on every failure and then holds the runner until GitHub cancels the job at the 6 hour limit. A one second cleanup race in the e2e teardown therefore costs a whole ubuntu-latest slot. The recent run list is full of 6h, 7h and 12h cancelled runs for that reason. The step now needs the `ci-debug` label on the pull request, so a session opens when somebody wants to debug and never otherwise. The 30 minute step timeout caps the cost when the label is left behind. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude:claude-opus-5 [Claude Code] --------- Signed-off-by: Ettore Di Giacinto Co-authored-by: Ettore Di Giacinto --- .github/workflows/test.yml | 14 ++++++++++++-- .github/workflows/tests-aio.yml | 7 ++++++- .github/workflows/tests-e2e.yml | 7 ++++++- .github/workflows/tests-pii-ner-e2e.yml | 7 ++++++- .github/workflows/tests-ui-e2e.yml | 7 ++++++- Makefile | 10 +++++++++- 6 files changed, 45 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7929f6b652a9..74d57567fdcd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -80,8 +80,13 @@ jobs: coverage/coverage.out coverage/coverage.html if-no-files-found: ignore + # tmate keeps the runner busy until the 6 hour job limit, so a single + # failure costs a whole runner slot. Only open a session when someone + # asked for one by labelling the pull request `ci-debug`, and cap the + # session so a forgotten label cannot idle a runner either. - name: Setup tmate session if tests fail - if: ${{ failure() }} + if: ${{ failure() && contains(github.event.pull_request.labels.*.name, 'ci-debug') }} + timeout-minutes: 30 uses: mxschmitt/action-tmate@v3.23 with: detached: true @@ -125,8 +130,13 @@ jobs: export PATH="/opt/homebrew/opt/make/libexec/gnubin:$PATH" PATH="$PATH:$HOME/go/bin" make protogen-go PATH="$PATH:$HOME/go/bin" BUILD_TYPE="GITHUB_CI_HAS_BROKEN_METAL" CMAKE_ARGS="-DGGML_F16C=OFF -DGGML_AVX512=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF" make --jobs 4 --output-sync=target test + # tmate keeps the runner busy until the 6 hour job limit, so a single + # failure costs a whole runner slot. Only open a session when someone + # asked for one by labelling the pull request `ci-debug`, and cap the + # session so a forgotten label cannot idle a runner either. - name: Setup tmate session if tests fail - if: ${{ failure() }} + if: ${{ failure() && contains(github.event.pull_request.labels.*.name, 'ci-debug') }} + timeout-minutes: 30 uses: mxschmitt/action-tmate@v3.23 with: detached: true diff --git a/.github/workflows/tests-aio.yml b/.github/workflows/tests-aio.yml index f8d3d34f077c..738f96daccc9 100644 --- a/.github/workflows/tests-aio.yml +++ b/.github/workflows/tests-aio.yml @@ -77,8 +77,13 @@ jobs: - name: Test run: | PATH="$PATH:$HOME/go/bin" make backends/local-store backends/silero-vad backends/llama-cpp backends/whisper backends/piper backends/stablediffusion-ggml docker-build-e2e e2e-aio + # tmate keeps the runner busy until the 6 hour job limit, so a single + # failure costs a whole runner slot. Only open a session when someone + # asked for one by labelling the pull request `ci-debug`, and cap the + # session so a forgotten label cannot idle a runner either. - name: Setup tmate session if tests fail - if: ${{ failure() }} + if: ${{ failure() && contains(github.event.pull_request.labels.*.name, 'ci-debug') }} + timeout-minutes: 30 uses: mxschmitt/action-tmate@v3.23 with: detached: true diff --git a/.github/workflows/tests-e2e.yml b/.github/workflows/tests-e2e.yml index 3c1cb711c79b..b5c84d2a85bc 100644 --- a/.github/workflows/tests-e2e.yml +++ b/.github/workflows/tests-e2e.yml @@ -63,8 +63,13 @@ jobs: - name: Test Backend E2E run: | PATH="$PATH:$HOME/go/bin" make build-mock-backend test-e2e + # tmate keeps the runner busy until the 6 hour job limit, so a single + # failure costs a whole runner slot. Only open a session when someone + # asked for one by labelling the pull request `ci-debug`, and cap the + # session so a forgotten label cannot idle a runner either. - name: Setup tmate session if tests fail - if: ${{ failure() }} + if: ${{ failure() && contains(github.event.pull_request.labels.*.name, 'ci-debug') }} + timeout-minutes: 30 uses: mxschmitt/action-tmate@v3.23 with: detached: true diff --git a/.github/workflows/tests-pii-ner-e2e.yml b/.github/workflows/tests-pii-ner-e2e.yml index 800f67190b9f..c9ab182f260b 100644 --- a/.github/workflows/tests-pii-ner-e2e.yml +++ b/.github/workflows/tests-pii-ner-e2e.yml @@ -88,8 +88,13 @@ jobs: # CPU and runs the token_classify capability spec (byte-offset contract). - name: Run live PII NER backend E2E run: PATH="$PATH:$HOME/go/bin" make test-extra-backend-privacy-filter + # tmate keeps the runner busy until the 6 hour job limit, so a single + # failure costs a whole runner slot. Only open a session when someone + # asked for one by labelling the pull request `ci-debug`, and cap the + # session so a forgotten label cannot idle a runner either. - name: Setup tmate session if tests fail - if: ${{ failure() }} + if: ${{ failure() && contains(github.event.pull_request.labels.*.name, 'ci-debug') }} + timeout-minutes: 30 uses: mxschmitt/action-tmate@v3.23 with: detached: true diff --git a/.github/workflows/tests-ui-e2e.yml b/.github/workflows/tests-ui-e2e.yml index 1a170fcf365e..6d45c8e9ff9d 100644 --- a/.github/workflows/tests-ui-e2e.yml +++ b/.github/workflows/tests-ui-e2e.yml @@ -75,8 +75,13 @@ jobs: path: core/http/react-ui/coverage/ if-no-files-found: ignore retention-days: 7 + # tmate keeps the runner busy until the 6 hour job limit, so a single + # failure costs a whole runner slot. Only open a session when someone + # asked for one by labelling the pull request `ci-debug`, and cap the + # session so a forgotten label cannot idle a runner either. - name: Setup tmate session if tests fail - if: ${{ failure() }} + if: ${{ failure() && contains(github.event.pull_request.labels.*.name, 'ci-debug') }} + timeout-minutes: 30 uses: mxschmitt/action-tmate@v3.23 with: detached: true diff --git a/Makefile b/Makefile index ebedb2c98248..cf3248c3be56 100644 --- a/Makefile +++ b/Makefile @@ -393,9 +393,17 @@ test-e2e: build-mock-backend build-cloud-proxy-backend prepare-e2e run-e2e-image $(MAKE) teardown-e2e docker rmi localai-tests +# `docker stop` returns as soon as the container exits, but Docker reaps a +# `--rm` container asynchronously after that. The `docker rmi localai-tests` in +# test-e2e then loses the race against the reaper and fails on a still +# referenced image, turning a green suite red. Removing the container ourselves +# is synchronous, so the image reference is gone before we return. It also +# covers the case where nothing is running, which `docker stop` could not +# because it rejects an empty argument list. teardown-e2e: rm -rf $(TEST_DIR) || true - docker stop $$(docker ps -q --filter ancestor=localai-tests) + @CONTAINERS=$$(docker ps -aq --filter ancestor=localai-tests 2>/dev/null); \ + if [ -n "$$CONTAINERS" ]; then docker rm -f $$CONTAINERS || true; fi ######################################################## ## Integration and unit tests From 06ebc28ae68731b6652fa62180c011fc6aeb9583 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Tue, 1 Sep 2026 12:37:10 +0200 Subject: [PATCH 2/5] chore: :arrow_up: Update mudler/depth-anything.cpp to `02ba082274e001a63e50de5a1eb0ccc50c6af4b1` (#11810) :arrow_up: Update mudler/depth-anything.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/go/depth-anything-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go/depth-anything-cpp/Makefile b/backend/go/depth-anything-cpp/Makefile index 14fa58ec9c06..2679556185db 100644 --- a/backend/go/depth-anything-cpp/Makefile +++ b/backend/go/depth-anything-cpp/Makefile @@ -14,7 +14,7 @@ JOBS?=$(shell nproc --ignore=1) # It is kept alive by the upstream tag da2-support (survives a squash-merge); # repoint to the master merge commit once mudler/depth-anything.cpp PR #1 lands. DEPTHANYTHING_REPO?=https://github.com/mudler/depth-anything.cpp.git -DEPTHANYTHING_VERSION?=739992d10bf9472c46dcd4622b14d2b20766c58d +DEPTHANYTHING_VERSION?=02ba082274e001a63e50de5a1eb0ccc50c6af4b1 ifeq ($(NATIVE),false) CMAKE_ARGS+=-DGGML_NATIVE=OFF From 34e9ad0cec35b904027e6d508c5834bd5fb45089 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Tue, 1 Sep 2026 12:38:24 +0200 Subject: [PATCH 3/5] chore: :arrow_up: Update CrispStrobe/CrispASR to `78c545eb80409b91291642ddb23b3a6dc044fd34` (#11811) :arrow_up: Update CrispStrobe/CrispASR Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/go/crispasr/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go/crispasr/Makefile b/backend/go/crispasr/Makefile index 0772d44318d3..da4d19480eb1 100644 --- a/backend/go/crispasr/Makefile +++ b/backend/go/crispasr/Makefile @@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1) # CrispASR version (release tag) CRISPASR_REPO?=https://github.com/CrispStrobe/CrispASR -CRISPASR_VERSION?=18b3e3f8456748a6380dc4c13817df244b695d39 +CRISPASR_VERSION?=78c545eb80409b91291642ddb23b3a6dc044fd34 SO_TARGET?=libgocrispasr.so CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF From 1da0dd7de2072114ec108f30ca4c1850815b1abe Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Tue, 1 Sep 2026 12:43:04 +0200 Subject: [PATCH 4/5] chore: :arrow_up: Update 0xShug0/audio.cpp to `3497b7cc44753e2c141d8fe60ac42cec433e3281` (#11818) :arrow_up: Update 0xShug0/audio.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/cpp/audio-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cpp/audio-cpp/Makefile b/backend/cpp/audio-cpp/Makefile index 7cfba527c795..231ab85a10fe 100644 --- a/backend/cpp/audio-cpp/Makefile +++ b/backend/cpp/audio-cpp/Makefile @@ -9,7 +9,7 @@ # recipe is a make target (not a prepare.sh) so 'make purge && make' is a clean # rebuild and so the bump bot can see the pin. -AUDIO_CPP_VERSION?=bf3315fe4aaa16dc1125f580c29aff90a8900b36 +AUDIO_CPP_VERSION?=3497b7cc44753e2c141d8fe60ac42cec433e3281 AUDIO_CPP_REPO?=https://github.com/0xShug0/audio.cpp CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) From aa1f0f8599068da06ad4997df7ef020924107ce7 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Tue, 1 Sep 2026 13:00:02 +0200 Subject: [PATCH 5/5] chore: :arrow_up: Update mudler/vllm.cpp to `839ea1ceddb787778b6bd86a38a917a1aab74d8f` (#11817) :arrow_up: Update mudler/vllm.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/go/vllm-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go/vllm-cpp/Makefile b/backend/go/vllm-cpp/Makefile index e7888e402518..e73976023648 100644 --- a/backend/go/vllm-cpp/Makefile +++ b/backend/go/vllm-cpp/Makefile @@ -11,7 +11,7 @@ JOBS?=$(shell nproc --ignore=1 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || e # vllm.cpp version VLLM_CPP_REPO?=https://github.com/mudler/vllm.cpp -VLLM_CPP_VERSION?=6a544bdb89eb5a3512ac922241439e45f24d74d4 +VLLM_CPP_VERSION?=839ea1ceddb787778b6bd86a38a917a1aab74d8f # MLX GEMM provider (darwin/metal only; see the metal branch below for why). # Consumed as the prebuilt pip wheel: building MLX from source needs `xcrun