diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7929f6b652a9..74d57567fdcd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -80,8 +80,13 @@ jobs: coverage/coverage.out coverage/coverage.html if-no-files-found: ignore + # tmate keeps the runner busy until the 6 hour job limit, so a single + # failure costs a whole runner slot. Only open a session when someone + # asked for one by labelling the pull request `ci-debug`, and cap the + # session so a forgotten label cannot idle a runner either. - name: Setup tmate session if tests fail - if: ${{ failure() }} + if: ${{ failure() && contains(github.event.pull_request.labels.*.name, 'ci-debug') }} + timeout-minutes: 30 uses: mxschmitt/action-tmate@v3.23 with: detached: true @@ -125,8 +130,13 @@ jobs: export PATH="/opt/homebrew/opt/make/libexec/gnubin:$PATH" PATH="$PATH:$HOME/go/bin" make protogen-go PATH="$PATH:$HOME/go/bin" BUILD_TYPE="GITHUB_CI_HAS_BROKEN_METAL" CMAKE_ARGS="-DGGML_F16C=OFF -DGGML_AVX512=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF" make --jobs 4 --output-sync=target test + # tmate keeps the runner busy until the 6 hour job limit, so a single + # failure costs a whole runner slot. Only open a session when someone + # asked for one by labelling the pull request `ci-debug`, and cap the + # session so a forgotten label cannot idle a runner either. - name: Setup tmate session if tests fail - if: ${{ failure() }} + if: ${{ failure() && contains(github.event.pull_request.labels.*.name, 'ci-debug') }} + timeout-minutes: 30 uses: mxschmitt/action-tmate@v3.23 with: detached: true diff --git a/.github/workflows/tests-aio.yml b/.github/workflows/tests-aio.yml index f8d3d34f077c..738f96daccc9 100644 --- a/.github/workflows/tests-aio.yml +++ b/.github/workflows/tests-aio.yml @@ -77,8 +77,13 @@ jobs: - name: Test run: | PATH="$PATH:$HOME/go/bin" make backends/local-store backends/silero-vad backends/llama-cpp backends/whisper backends/piper backends/stablediffusion-ggml docker-build-e2e e2e-aio + # tmate keeps the runner busy until the 6 hour job limit, so a single + # failure costs a whole runner slot. Only open a session when someone + # asked for one by labelling the pull request `ci-debug`, and cap the + # session so a forgotten label cannot idle a runner either. - name: Setup tmate session if tests fail - if: ${{ failure() }} + if: ${{ failure() && contains(github.event.pull_request.labels.*.name, 'ci-debug') }} + timeout-minutes: 30 uses: mxschmitt/action-tmate@v3.23 with: detached: true diff --git a/.github/workflows/tests-e2e.yml b/.github/workflows/tests-e2e.yml index 3c1cb711c79b..b5c84d2a85bc 100644 --- a/.github/workflows/tests-e2e.yml +++ b/.github/workflows/tests-e2e.yml @@ -63,8 +63,13 @@ jobs: - name: Test Backend E2E run: | PATH="$PATH:$HOME/go/bin" make build-mock-backend test-e2e + # tmate keeps the runner busy until the 6 hour job limit, so a single + # failure costs a whole runner slot. Only open a session when someone + # asked for one by labelling the pull request `ci-debug`, and cap the + # session so a forgotten label cannot idle a runner either. - name: Setup tmate session if tests fail - if: ${{ failure() }} + if: ${{ failure() && contains(github.event.pull_request.labels.*.name, 'ci-debug') }} + timeout-minutes: 30 uses: mxschmitt/action-tmate@v3.23 with: detached: true diff --git a/.github/workflows/tests-pii-ner-e2e.yml b/.github/workflows/tests-pii-ner-e2e.yml index 800f67190b9f..c9ab182f260b 100644 --- a/.github/workflows/tests-pii-ner-e2e.yml +++ b/.github/workflows/tests-pii-ner-e2e.yml @@ -88,8 +88,13 @@ jobs: # CPU and runs the token_classify capability spec (byte-offset contract). - name: Run live PII NER backend E2E run: PATH="$PATH:$HOME/go/bin" make test-extra-backend-privacy-filter + # tmate keeps the runner busy until the 6 hour job limit, so a single + # failure costs a whole runner slot. Only open a session when someone + # asked for one by labelling the pull request `ci-debug`, and cap the + # session so a forgotten label cannot idle a runner either. - name: Setup tmate session if tests fail - if: ${{ failure() }} + if: ${{ failure() && contains(github.event.pull_request.labels.*.name, 'ci-debug') }} + timeout-minutes: 30 uses: mxschmitt/action-tmate@v3.23 with: detached: true diff --git a/.github/workflows/tests-ui-e2e.yml b/.github/workflows/tests-ui-e2e.yml index 1a170fcf365e..6d45c8e9ff9d 100644 --- a/.github/workflows/tests-ui-e2e.yml +++ b/.github/workflows/tests-ui-e2e.yml @@ -75,8 +75,13 @@ jobs: path: core/http/react-ui/coverage/ if-no-files-found: ignore retention-days: 7 + # tmate keeps the runner busy until the 6 hour job limit, so a single + # failure costs a whole runner slot. Only open a session when someone + # asked for one by labelling the pull request `ci-debug`, and cap the + # session so a forgotten label cannot idle a runner either. - name: Setup tmate session if tests fail - if: ${{ failure() }} + if: ${{ failure() && contains(github.event.pull_request.labels.*.name, 'ci-debug') }} + timeout-minutes: 30 uses: mxschmitt/action-tmate@v3.23 with: detached: true diff --git a/Makefile b/Makefile index ebedb2c98248..cf3248c3be56 100644 --- a/Makefile +++ b/Makefile @@ -393,9 +393,17 @@ test-e2e: build-mock-backend build-cloud-proxy-backend prepare-e2e run-e2e-image $(MAKE) teardown-e2e docker rmi localai-tests +# `docker stop` returns as soon as the container exits, but Docker reaps a +# `--rm` container asynchronously after that. The `docker rmi localai-tests` in +# test-e2e then loses the race against the reaper and fails on a still +# referenced image, turning a green suite red. Removing the container ourselves +# is synchronous, so the image reference is gone before we return. It also +# covers the case where nothing is running, which `docker stop` could not +# because it rejects an empty argument list. teardown-e2e: rm -rf $(TEST_DIR) || true - docker stop $$(docker ps -q --filter ancestor=localai-tests) + @CONTAINERS=$$(docker ps -aq --filter ancestor=localai-tests 2>/dev/null); \ + if [ -n "$$CONTAINERS" ]; then docker rm -f $$CONTAINERS || true; fi ######################################################## ## Integration and unit tests diff --git a/backend/cpp/audio-cpp/Makefile b/backend/cpp/audio-cpp/Makefile index 7cfba527c795..231ab85a10fe 100644 --- a/backend/cpp/audio-cpp/Makefile +++ b/backend/cpp/audio-cpp/Makefile @@ -9,7 +9,7 @@ # recipe is a make target (not a prepare.sh) so 'make purge && make' is a clean # rebuild and so the bump bot can see the pin. -AUDIO_CPP_VERSION?=bf3315fe4aaa16dc1125f580c29aff90a8900b36 +AUDIO_CPP_VERSION?=3497b7cc44753e2c141d8fe60ac42cec433e3281 AUDIO_CPP_REPO?=https://github.com/0xShug0/audio.cpp CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) diff --git a/backend/go/crispasr/Makefile b/backend/go/crispasr/Makefile index 0772d44318d3..da4d19480eb1 100644 --- a/backend/go/crispasr/Makefile +++ b/backend/go/crispasr/Makefile @@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1) # CrispASR version (release tag) CRISPASR_REPO?=https://github.com/CrispStrobe/CrispASR -CRISPASR_VERSION?=18b3e3f8456748a6380dc4c13817df244b695d39 +CRISPASR_VERSION?=78c545eb80409b91291642ddb23b3a6dc044fd34 SO_TARGET?=libgocrispasr.so CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF diff --git a/backend/go/depth-anything-cpp/Makefile b/backend/go/depth-anything-cpp/Makefile index 14fa58ec9c06..2679556185db 100644 --- a/backend/go/depth-anything-cpp/Makefile +++ b/backend/go/depth-anything-cpp/Makefile @@ -14,7 +14,7 @@ JOBS?=$(shell nproc --ignore=1) # It is kept alive by the upstream tag da2-support (survives a squash-merge); # repoint to the master merge commit once mudler/depth-anything.cpp PR #1 lands. DEPTHANYTHING_REPO?=https://github.com/mudler/depth-anything.cpp.git -DEPTHANYTHING_VERSION?=739992d10bf9472c46dcd4622b14d2b20766c58d +DEPTHANYTHING_VERSION?=02ba082274e001a63e50de5a1eb0ccc50c6af4b1 ifeq ($(NATIVE),false) CMAKE_ARGS+=-DGGML_NATIVE=OFF diff --git a/backend/go/vllm-cpp/Makefile b/backend/go/vllm-cpp/Makefile index e7888e402518..e73976023648 100644 --- a/backend/go/vllm-cpp/Makefile +++ b/backend/go/vllm-cpp/Makefile @@ -11,7 +11,7 @@ JOBS?=$(shell nproc --ignore=1 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || e # vllm.cpp version VLLM_CPP_REPO?=https://github.com/mudler/vllm.cpp -VLLM_CPP_VERSION?=6a544bdb89eb5a3512ac922241439e45f24d74d4 +VLLM_CPP_VERSION?=839ea1ceddb787778b6bd86a38a917a1aab74d8f # MLX GEMM provider (darwin/metal only; see the metal branch below for why). # Consumed as the prebuilt pip wheel: building MLX from source needs `xcrun