From 1313a4a5a2c3c7ad0f87ac6c55f1251e9e3585a8 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Sun, 23 Aug 2026 21:37:28 +0200 Subject: [PATCH 01/13] feat(gallery): add LFM2.5 DSpark variants (#11676) LiquidAI now publishes official DSpark draft sidecars for its 2.6B target. Pair Q4 and Q8 targets with matching draft choices so LocalAI can use speculative decoding across different memory budgets. Assisted-by: Codex:gpt-5 [systematic-debugging] Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- gallery/index.yaml | 101 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/gallery/index.yaml b/gallery/index.yaml index 4f0fb21920c9..0ffa08d5c136 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -4865,7 +4865,9 @@ - gpu icon: https://cdn-uploads.huggingface.co/production/uploads/61b8e2ba285851687028d395/2b08LKpev0DNEk6DlnWkY.png variants: + - model: lfm2.5-2.6b-dspark - model: lfm2.5-2.6b-q8 + - model: lfm2.5-2.6b-q8-dspark overrides: backend: llama-cpp context_size: 131072 @@ -4921,6 +4923,105 @@ - filename: llama-cpp/models/LFM2.5-2.6B-GGUF/LFM2.5-2.6B-Q8_0.gguf uri: https://huggingface.co/LiquidAI/LFM2.5-2.6B-GGUF/resolve/main/LFM2.5-2.6B-Q8_0.gguf sha256: 1e22128dfa128bdfb684da167e74e072d0a056baa7d06d9f280291e2839b0fc9 +- !!merge <<: *lfm2-5-2-6b + name: "lfm2.5-2.6b-dspark" + description: | + LFM2.5-2.6B with LiquidAI's DSpark speculative drafter. This build pairs + the Q4_K_M target with the compact Q4_K_M draft sidecar for lower-memory + hosts. DSpark proposes blocks of tokens that the target model verifies, + which preserves the target model's output while accelerating generation. + tags: + - llm + - gguf + - reasoning + - cpu + - gpu + - dspark + variants: null + urls: + - https://huggingface.co/LiquidAI/LFM2.5-2.6B + - https://huggingface.co/LiquidAI/LFM2.5-2.6B-GGUF + - https://huggingface.co/LiquidAI/LFM2.5-2.6B-DSpark-GGUF + overrides: + backend: llama-cpp + context_size: 131072 + draft_model: llama-cpp/models/LFM2.5-2.6B-DSpark-GGUF/LFM2.5-2.6B-DSpark-Q4_K_M.gguf + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - completion + options: + - use_jinja:true + - spec_type:draft-dspark + - spec_n_max:10 + - spec_n_min:0 + parameters: + model: llama-cpp/models/LFM2.5-2.6B-GGUF/LFM2.5-2.6B-Q4_K_M.gguf + repeat_penalty: 1.1 + temperature: 0.1 + top_k: 50 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/LFM2.5-2.6B-GGUF/LFM2.5-2.6B-Q4_K_M.gguf + uri: https://huggingface.co/LiquidAI/LFM2.5-2.6B-GGUF/resolve/main/LFM2.5-2.6B-Q4_K_M.gguf + sha256: 02a8b7e17487d326e46d68ce0ba24211e1b80a14c4cd0597fa73c1cd697f52ed + - filename: llama-cpp/models/LFM2.5-2.6B-DSpark-GGUF/LFM2.5-2.6B-DSpark-Q4_K_M.gguf + uri: https://huggingface.co/LiquidAI/LFM2.5-2.6B-DSpark-GGUF/resolve/main/LFM2.5-2.6B-DSpark-Q4_K_M.gguf + sha256: 63786b768e43562591a934625a66539eab39beb09e6e3d38e17509e34b79a3cd +- !!merge <<: *lfm2-5-2-6b + name: "lfm2.5-2.6b-q8-dspark" + description: | + LFM2.5-2.6B with LiquidAI's DSpark speculative drafter. This build pairs + the higher-quality Q8_0 target with the recommended F16 draft sidecar for + the best acceptance length. DSpark proposes blocks of tokens that the + target model verifies, which preserves the target model's output while + accelerating generation. + tags: + - llm + - gguf + - reasoning + - cpu + - gpu + - dspark + variants: null + urls: + - https://huggingface.co/LiquidAI/LFM2.5-2.6B + - https://huggingface.co/LiquidAI/LFM2.5-2.6B-GGUF + - https://huggingface.co/LiquidAI/LFM2.5-2.6B-DSpark-GGUF + overrides: + backend: llama-cpp + context_size: 131072 + draft_model: llama-cpp/models/LFM2.5-2.6B-DSpark-GGUF/LFM2.5-2.6B-DSpark-F16.gguf + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - completion + options: + - use_jinja:true + - spec_type:draft-dspark + - spec_n_max:10 + - spec_n_min:0 + parameters: + model: llama-cpp/models/LFM2.5-2.6B-GGUF/LFM2.5-2.6B-Q8_0.gguf + repeat_penalty: 1.1 + temperature: 0.1 + top_k: 50 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/LFM2.5-2.6B-GGUF/LFM2.5-2.6B-Q8_0.gguf + uri: https://huggingface.co/LiquidAI/LFM2.5-2.6B-GGUF/resolve/main/LFM2.5-2.6B-Q8_0.gguf + sha256: 1e22128dfa128bdfb684da167e74e072d0a056baa7d06d9f280291e2839b0fc9 + - filename: llama-cpp/models/LFM2.5-2.6B-DSpark-GGUF/LFM2.5-2.6B-DSpark-F16.gguf + uri: https://huggingface.co/LiquidAI/LFM2.5-2.6B-DSpark-GGUF/resolve/main/LFM2.5-2.6B-DSpark-F16.gguf + sha256: e198962c08903f3ba29f0ce6bf8e17f5e60bf85ec2f8673e1e2aab03508937e5 - &bigbang-v1 name: "bigbang-v1-q4-k-m" variants: From 031eb5fc7cd28f261f0ac24b65cf475e1d43423a Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Sun, 23 Aug 2026 21:37:45 +0200 Subject: [PATCH 02/13] feat(gallery): add Qwen3.8 small distills (#11675) Add the 2B and 4B distilled models alongside the 9B model. Their compact GGUF builds make the Qwen3.8 reasoning distill practical on smaller hosts. Assisted-by: Codex:gpt-5 Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- gallery/index.yaml | 158 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) diff --git a/gallery/index.yaml b/gallery/index.yaml index 0ffa08d5c136..2047a42f430e 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -1483,6 +1483,164 @@ - filename: llama-cpp/mmproj/qwen3.5-9b-defiant-fable/mmproj-BF16.gguf uri: huggingface://DavidAU/Qwen3.5-9B-The-Defiant-Fable-Uncensored-Heretic-NEO-IMATRIX-MAX-MTP-GGUF/mmproj-BF16.gguf sha256: 853698ce7aa6c7ba732478bad280240969ddf7b0fcbf93900046f63903a83383 +- &qwen3-8-2b-distill + name: "qwen3.8-2b-distill-q4" + variants: + - model: qwen3.8-2b-distill-q8 + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/empero-ai/Qwen3.8-2B-Distill + - https://huggingface.co/empero-ai/Qwen3.8-2B-Distill-GGUF + description: | + Qwen3.8 2B Distill is an Apache-2.0, text-only Qwen3.5 2B fine-tune + distilled from Qwen3.8 2.4T A95B reasoning traces. It targets compact + reasoning, coding, instruction following, and function calling with a 262K + native context window. This entry uses the balanced Q4_K_M GGUF + quantization; the Q8_0 variant offers higher fidelity. + license: apache-2.0 + icon: https://qianwen-res.oss-cn-beijing.aliyuncs.com/logo_qwen.jpg + tags: + - llm + - gguf + - cpu + - gpu + - qwen3.5 + - reasoning + - coding + - tool-use + last_checked: "2026-08-23" + overrides: + backend: llama-cpp + context_size: 32768 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + - reasoning_format:deepseek + parameters: + model: llama-cpp/models/qwen3.8-2b-distill/Qwen3.8-2B-Q4_K_M.gguf + temperature: 0.6 + top_k: 20 + top_p: 0.95 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/qwen3.8-2b-distill/Qwen3.8-2B-Q4_K_M.gguf + uri: huggingface://empero-ai/Qwen3.8-2B-Distill-GGUF/Qwen3.8-2B-Q4_K_M.gguf + sha256: 4aa0fb13c431514262f259d420ecc95a8714df58ac2a2384514e20b93983f0ff +- !!merge <<: *qwen3-8-2b-distill + name: "qwen3.8-2b-distill-q8" + variants: [] + description: | + Qwen3.8 2B Distill in the higher-fidelity Q8_0 GGUF format. This text-only + Qwen3.5 2B fine-tune targets reasoning, coding, instruction following, and + function calling with a 262K native context window. + overrides: + backend: llama-cpp + context_size: 32768 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + - reasoning_format:deepseek + parameters: + model: llama-cpp/models/qwen3.8-2b-distill/Qwen3.8-2B-Q8_0.gguf + temperature: 0.6 + top_k: 20 + top_p: 0.95 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/qwen3.8-2b-distill/Qwen3.8-2B-Q8_0.gguf + uri: huggingface://empero-ai/Qwen3.8-2B-Distill-GGUF/Qwen3.8-2B-Q8_0.gguf + sha256: 866773b0d68f09a1db9733555e92daff85b617f9a2e601773dff494c5ca2bbf2 +- &qwen3-8-4b-distill + name: "qwen3.8-4b-distill-q4" + variants: + - model: qwen3.8-4b-distill-q8 + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/empero-ai/Qwen3.8-4B-Distill + - https://huggingface.co/empero-ai/Qwen3.8-4B-Distill-GGUF + description: | + Qwen3.8 4B Distill is an Apache-2.0, text-only Qwen3.5 4B fine-tune + distilled from Qwen3.8 2.4T A95B reasoning traces. It targets reasoning, + coding, instruction following, and function calling with a 262K native + context window. This entry uses the balanced Q4_K_M GGUF quantization; the + Q8_0 variant offers higher fidelity. + license: apache-2.0 + icon: https://qianwen-res.oss-cn-beijing.aliyuncs.com/logo_qwen.jpg + tags: + - llm + - gguf + - cpu + - gpu + - qwen3.5 + - reasoning + - coding + - tool-use + last_checked: "2026-08-23" + overrides: + backend: llama-cpp + context_size: 32768 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + - reasoning_format:deepseek + parameters: + model: llama-cpp/models/qwen3.8-4b-distill/Qwen3.8-4B-Q4_K_M.gguf + temperature: 0.6 + top_k: 20 + top_p: 0.95 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/qwen3.8-4b-distill/Qwen3.8-4B-Q4_K_M.gguf + uri: huggingface://empero-ai/Qwen3.8-4B-Distill-GGUF/Qwen3.8-4B-Q4_K_M.gguf + sha256: dec96e8cf2e11b613bb46513dec485377f9ca5a351e71712ee0e244f287c6790 +- !!merge <<: *qwen3-8-4b-distill + name: "qwen3.8-4b-distill-q8" + variants: [] + description: | + Qwen3.8 4B Distill in the higher-fidelity Q8_0 GGUF format. This text-only + Qwen3.5 4B fine-tune targets reasoning, coding, instruction following, and + function calling with a 262K native context window. + overrides: + backend: llama-cpp + context_size: 32768 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + - reasoning_format:deepseek + parameters: + model: llama-cpp/models/qwen3.8-4b-distill/Qwen3.8-4B-Q8_0.gguf + temperature: 0.6 + top_k: 20 + top_p: 0.95 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/qwen3.8-4b-distill/Qwen3.8-4B-Q8_0.gguf + uri: huggingface://empero-ai/Qwen3.8-4B-Distill-GGUF/Qwen3.8-4B-Q8_0.gguf + sha256: 770b780d6754a4954d1caf395c9239eaeb394f15c7a7ea34039883377c93c9c3 - name: "btl-4-compact" url: "github:mudler/LocalAI/gallery/virtual.yaml@master" urls: From b9914b56fb055167ec39e317054886851b5f8ea2 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Sun, 23 Aug 2026 21:37:58 +0200 Subject: [PATCH 03/13] feat(gallery): add UI-Mate 27B variants (#11672) Add Q4_K_M and Q8_0 builds for Tencent's Qwen3.6-based computer-use model. Assisted-by: Codex:gpt-5.6 [Codex] Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- gallery/index.yaml | 88 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/gallery/index.yaml b/gallery/index.yaml index 2047a42f430e..3b967b3cd972 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -9066,6 +9066,94 @@ - filename: llama-cpp/mmproj/tencent_UI-Mate-9B-GGUF/mmproj-tencent_UI-Mate-9B-f16.gguf sha256: 5a8380c4637dddceed9dbc28fffcdfa8601909c0ece9fe218fbd6888ec5d2c16 uri: huggingface://bartowski/tencent_UI-Mate-9B-GGUF/mmproj-tencent_UI-Mate-9B-f16.gguf +- &ui-mate-27b + name: ui-mate-27b + url: github:mudler/LocalAI/gallery/virtual.yaml@master + variants: + - model: ui-mate-27b-q8 + urls: + - https://huggingface.co/tencent/UI-Mate-27B + - https://huggingface.co/bartowski/tencent_UI-Mate-27B-GGUF + description: | + UI-Mate-27B is Tencent's 27B-parameter multimodal computer-use agent, + fine-tuned from Qwen3.6-27B. It accepts task instructions, screenshots, + and interaction history, then emits reasoning and structured mouse and + keyboard actions for long-running desktop tasks. The model requires an + external runtime to execute its actions and should run with human + confirmation for sensitive operations. This entry uses the recommended + Q4_K_M GGUF quantization. + license: apache-2.0 + tags: + - ui-mate + - qwen + - qwen3.6 + - 27b + - llm + - gguf + - quantized + - chat + - vision + - multimodal + - agent + - computer-use + - gpu + - cpu + last_checked: "2026-08-22" + overrides: + backend: llama-cpp + function: + grammar: + disable: true + known_usecases: + - chat + - vision + mmproj: llama-cpp/mmproj/tencent_UI-Mate-27B-GGUF/mmproj-tencent_UI-Mate-27B-f16.gguf + options: + - use_jinja:true + parameters: + model: llama-cpp/models/tencent_UI-Mate-27B-GGUF/tencent_UI-Mate-27B-Q4_K_M.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/tencent_UI-Mate-27B-GGUF/tencent_UI-Mate-27B-Q4_K_M.gguf + sha256: 05ee4cca09f10de19e828e51497cc8b4c9eabffe2a177a54d1188ff869669f20 + uri: huggingface://bartowski/tencent_UI-Mate-27B-GGUF/tencent_UI-Mate-27B-Q4_K_M.gguf + - filename: llama-cpp/mmproj/tencent_UI-Mate-27B-GGUF/mmproj-tencent_UI-Mate-27B-f16.gguf + sha256: 991376d8e954dda92b454358306fb2b4bbd51d522b12977cdfeb3eebbb904fbb + uri: huggingface://bartowski/tencent_UI-Mate-27B-GGUF/mmproj-tencent_UI-Mate-27B-f16.gguf +- !!merge <<: *ui-mate-27b + name: ui-mate-27b-q8 + variants: [] + description: | + UI-Mate-27B is Tencent's 27B-parameter multimodal computer-use agent, + fine-tuned from Qwen3.6-27B. It accepts task instructions, screenshots, + and interaction history, then emits reasoning and structured mouse and + keyboard actions for long-running desktop tasks. The model requires an + external runtime to execute its actions and should run with human + confirmation for sensitive operations. This entry uses the higher-quality + Q8_0 GGUF quantization. + overrides: + backend: llama-cpp + function: + grammar: + disable: true + known_usecases: + - chat + - vision + mmproj: llama-cpp/mmproj/tencent_UI-Mate-27B-GGUF/mmproj-tencent_UI-Mate-27B-f16.gguf + options: + - use_jinja:true + parameters: + model: llama-cpp/models/tencent_UI-Mate-27B-GGUF/tencent_UI-Mate-27B-Q8_0.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/tencent_UI-Mate-27B-GGUF/tencent_UI-Mate-27B-Q8_0.gguf + sha256: 3626d336fb902e91f2e7d241d95c9410c74094d5f1739c76ef896140878bcd68 + uri: huggingface://bartowski/tencent_UI-Mate-27B-GGUF/tencent_UI-Mate-27B-Q8_0.gguf + - filename: llama-cpp/mmproj/tencent_UI-Mate-27B-GGUF/mmproj-tencent_UI-Mate-27B-f16.gguf + sha256: 991376d8e954dda92b454358306fb2b4bbd51d522b12977cdfeb3eebbb904fbb + uri: huggingface://bartowski/tencent_UI-Mate-27B-GGUF/mmproj-tencent_UI-Mate-27B-f16.gguf - &fara1-5-4b name: fara1.5-4b url: github:mudler/LocalAI/gallery/virtual.yaml@master From eafc7fda27122a8a223b885fa7d5db49365d1cef Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Sun, 23 Aug 2026 21:38:10 +0200 Subject: [PATCH 04/13] feat(gallery): add Homura 30B (#11680) Add the publisher\047s Q4_K_M build for the recent Muse Glimmer agent fine-tune. Assisted-by: Codex:gpt-5 Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- gallery/index.yaml | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/gallery/index.yaml b/gallery/index.yaml index 3b967b3cd972..31cb6a925c34 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -1386,6 +1386,47 @@ - filename: llama-cpp/models/muse-glimmer-30b/dflash-kquant.gguf uri: huggingface://meta-models/Muse-Glimmer-30B-GGUF/dflash-kquant.gguf sha256: 27d9a805fa29b943cfb6ad4843367cd4eaaaf06bd452d8cc3e00a2cd18a677bc +- name: "homura-30b-q4" + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/hyrelabs/Homura-30B-GGUF + - https://huggingface.co/darkc0de/Muse-Glimmer-30B-heretic + description: | + Homura 30B is an English, agent-focused fine-tune of Muse Glimmer 30B. + It targets autonomous tool use and direct instruction following. This + entry uses the publisher's 16.9 GB Q4_K_M GGUF and supports a 131K-token + context window. + license: "apache-2.0" + tags: + - llm + - gguf + - cpu + - gpu + - agent + - tools + - long-context + - uncensored + last_checked: "2026-08-23" + overrides: + backend: llama-cpp + context_size: 131072 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + parameters: + model: llama-cpp/models/homura-30b/Homura-30B-Q4_K_M.gguf + temperature: 0.2 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/homura-30b/Homura-30B-Q4_K_M.gguf + uri: huggingface://hyrelabs/Homura-30B-GGUF/Homura-30B-Q4_K_M.gguf + sha256: fd4dc394b193ac1cc0f4bb9fb44bff3d6eebd34df62246fc753104c0cc98cc51 - &qwen3-5-9b-defiant-fable name: "qwen3.5-9b-defiant-fable-mtp" variants: From 4bad644498a89d536abce80c1131795ff319aaa0 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 23 Aug 2026 19:53:28 +0000 Subject: [PATCH 05/13] fix(distributed): name both revisions in the stale error "stale model config revision" reported only that two hashes differed. It named neither, so an operator could not tell an edited configuration from a revision that is not reproducible for one unchanged file, and the failing value appears in no table. The error now carries the revision the request brought and the one the controller holds. It still wraps ErrStaleModelConfigRevision, so callers that classify the error keep working. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude Code:claude-opus-5 [golangci-lint] --- core/services/nodes/registry.go | 19 ++++++++++++++- .../nodes/revision_error_detail_test.go | 24 +++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 core/services/nodes/revision_error_detail_test.go diff --git a/core/services/nodes/registry.go b/core/services/nodes/registry.go index d6fd59662ff0..1e5d52db02f3 100644 --- a/core/services/nodes/registry.go +++ b/core/services/nodes/registry.go @@ -1159,11 +1159,28 @@ func requireCurrentRevision(tx *gorm.DB, modelName, revision string) error { return err } if state.ConfigRevision != revision { - return ErrStaleModelConfigRevision + // Name both sides. "stale model config revision" on its own says only + // that two hashes differ, which leaves an operator no way to tell an + // edited configuration from a revision that is not reproducible for one + // unchanged file. + return fmt.Errorf("%w (request carries %s, controller holds %s)", + ErrStaleModelConfigRevision, shortRevision(revision), shortRevision(state.ConfigRevision)) } return nil } +// shortRevision trims a revision for log and error output. The full value is a +// sha256 hex digest; the leading bytes identify it well enough to compare two. +func shortRevision(revision string) string { + if revision == "" { + return "(none)" + } + if len(revision) > 12 { + return revision[:12] + } + return revision +} + func validateRevisionWrite(modelName, revision string, revisionRequired bool) error { if modelName == "" { return fmt.Errorf("model name is required") diff --git a/core/services/nodes/revision_error_detail_test.go b/core/services/nodes/revision_error_detail_test.go new file mode 100644 index 000000000000..25be6866a5a0 --- /dev/null +++ b/core/services/nodes/revision_error_detail_test.go @@ -0,0 +1,24 @@ +package nodes + +import ( + "errors" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Stale revision error detail", func() { + It("keeps errors.Is matching so callers can still classify it", func() { + err := fmt.Errorf("%w (request carries %s, controller holds %s)", + ErrStaleModelConfigRevision, shortRevision("aaaabbbbccccdddd"), shortRevision("1111222233334444")) + Expect(errors.Is(err, ErrStaleModelConfigRevision)).To(BeTrue()) + Expect(err.Error()).To(ContainSubstring("stale model config revision")) + }) + + It("names both revisions so an operator can tell which side moved", func() { + Expect(shortRevision("aaaabbbbccccdddd")).To(Equal("aaaabbbbcccc")) + Expect(shortRevision("short")).To(Equal("short")) + Expect(shortRevision("")).To(Equal("(none)")) + }) +}) From cee87d1608a696dd5c6206d689da2e84e4a84d53 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 23 Aug 2026 20:20:02 +0000 Subject: [PATCH 06/13] fix(distributed): expire staged request files on the worker A request that carries a file stages it to the worker, which writes it under its staging directory. Nothing removed it afterwards. The frontend expires ephemeral keys from object storage, but that sweep never covered a worker's local disk, so every image, audio clip and video a worker ever served stayed on it. One worker had accumulated 175 request directories over three months. The volume reached 100 percent, and from that point every backend start failed because the process manager could not create a state directory. The worker now sweeps its ephemeral staging directory on a timer and once at startup, so files left by a crash are reclaimed too. Staged model files live beside that directory and are not touched. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude Code:claude-opus-5 [golangci-lint] --- core/services/worker/ephemeral_cleanup.go | 112 ++++++++++++++++++ .../services/worker/ephemeral_cleanup_test.go | 58 +++++++++ core/services/worker/worker.go | 4 + docs/content/features/distributed-mode.md | 6 + 4 files changed, 180 insertions(+) create mode 100644 core/services/worker/ephemeral_cleanup.go create mode 100644 core/services/worker/ephemeral_cleanup_test.go diff --git a/core/services/worker/ephemeral_cleanup.go b/core/services/worker/ephemeral_cleanup.go new file mode 100644 index 000000000000..4a8e92da9677 --- /dev/null +++ b/core/services/worker/ephemeral_cleanup.go @@ -0,0 +1,112 @@ +package worker + +import ( + "context" + "os" + "path/filepath" + "time" + + "github.com/mudler/xlog" +) + +const ( + // defaultEphemeralStagingTTL bounds how long a staged request input can + // outlive the request that needed it. Inference reads these files while the + // request runs, so the window has to cover a slow multimodal request; it + // does not have to cover anything longer. + defaultEphemeralStagingTTL = 6 * time.Hour + // defaultEphemeralStagingSweep is how often the worker sweeps. + defaultEphemeralStagingSweep = 30 * time.Minute +) + +// StartEphemeralStagingCleanup sweeps the worker's own staging directory for +// per-request input files left behind by finished requests. +// +// The frontend already expires ephemeral keys from object storage +// (services/storage.StartEphemeralCleanup), but a worker receives these files +// over the file-transfer server and writes them to its local disk, where +// nothing expired them. They accumulated for as long as the worker lived and +// eventually filled the volume, at which point every backend start failed +// because the process manager could no longer create a state directory. +func StartEphemeralStagingCleanup(ctx context.Context, stagingDir string, ttl, interval time.Duration) { + if stagingDir == "" { + return + } + if ttl <= 0 { + ttl = defaultEphemeralStagingTTL + } + if interval <= 0 { + interval = defaultEphemeralStagingSweep + } + + go func() { + ticker := time.NewTicker(interval) + defer ticker.Stop() + // Sweep once at startup: a worker that crashed with staged files leaves + // them behind, and waiting a full interval to reclaim that space is the + // case that hurts on a volume that is already close to full. + CleanEphemeralStaging(stagingDir, ttl) + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + CleanEphemeralStaging(stagingDir, ttl) + } + } + }() + + xlog.Info("Ephemeral staging cleanup started", "dir", stagingDir, "ttl", ttl, "interval", interval) +} + +// CleanEphemeralStaging removes staged per-request directories older than ttl. +// It only ever descends into /ephemeral, so staged model weights, +// which live alongside it and are not scratch, are never considered. +func CleanEphemeralStaging(stagingDir string, ttl time.Duration) { + root := filepath.Join(stagingDir, "ephemeral") + categories, err := os.ReadDir(root) + if err != nil { + // A worker that has never served a file-bearing request has no + // ephemeral directory at all. That is the normal case, not a fault. + if !os.IsNotExist(err) { + xlog.Warn("Ephemeral staging cleanup: cannot read staging root", "dir", root, "error", err) + } + return + } + + cutoff := time.Now().Add(-ttl) + removed := 0 + for _, category := range categories { + if !category.IsDir() { + continue + } + categoryDir := filepath.Join(root, category.Name()) + entries, err := os.ReadDir(categoryDir) + if err != nil { + xlog.Warn("Ephemeral staging cleanup: cannot read category", "dir", categoryDir, "error", err) + continue + } + for _, entry := range entries { + path := filepath.Join(categoryDir, entry.Name()) + info, err := entry.Info() + if err != nil { + xlog.Warn("Ephemeral staging cleanup: cannot stat entry", "path", path, "error", err) + continue + } + // A request rewrites nothing after staging, so the entry's own + // modification time is when its request was served. + if !info.ModTime().Before(cutoff) { + continue + } + if err := os.RemoveAll(path); err != nil { + xlog.Warn("Ephemeral staging cleanup: cannot remove", "path", path, "error", err) + continue + } + removed++ + } + } + + if removed > 0 { + xlog.Info("Ephemeral staging cleanup removed stale request files", "count", removed, "dir", root) + } +} diff --git a/core/services/worker/ephemeral_cleanup_test.go b/core/services/worker/ephemeral_cleanup_test.go new file mode 100644 index 000000000000..3542f1afcffa --- /dev/null +++ b/core/services/worker/ephemeral_cleanup_test.go @@ -0,0 +1,58 @@ +package worker + +import ( + "os" + "path/filepath" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Worker ephemeral staging cleanup", func() { + var stagingDir string + + // mkEphemeral creates one staged request directory holding a file, and + // backdates both so the sweeper sees it as `age` old. + mkEphemeral := func(id string, age time.Duration) string { + dir := filepath.Join(stagingDir, "ephemeral", "inputs", id) + Expect(os.MkdirAll(dir, 0o750)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(dir, "payload.bin"), []byte("x"), 0o600)).To(Succeed()) + stamp := time.Now().Add(-age) + Expect(os.Chtimes(filepath.Join(dir, "payload.bin"), stamp, stamp)).To(Succeed()) + Expect(os.Chtimes(dir, stamp, stamp)).To(Succeed()) + return dir + } + + BeforeEach(func() { stagingDir = GinkgoT().TempDir() }) + + It("removes staged request directories older than the TTL", func() { + old := mkEphemeral("aaaa1111", 48*time.Hour) + CleanEphemeralStaging(stagingDir, time.Hour) + Expect(old).ToNot(BeAnExistingFile()) + }) + + It("keeps directories a running request may still be reading", func() { + fresh := mkEphemeral("bbbb2222", 5*time.Minute) + CleanEphemeralStaging(stagingDir, time.Hour) + Expect(fresh).To(BeAnExistingFile()) + }) + + It("leaves staged models and everything outside ephemeral alone", func() { + modelDir := filepath.Join(stagingDir, "models", "some-model") + Expect(os.MkdirAll(modelDir, 0o750)).To(Succeed()) + weights := filepath.Join(modelDir, "weights.gguf") + Expect(os.WriteFile(weights, []byte("w"), 0o600)).To(Succeed()) + stamp := time.Now().Add(-90 * 24 * time.Hour) + Expect(os.Chtimes(weights, stamp, stamp)).To(Succeed()) + Expect(os.Chtimes(modelDir, stamp, stamp)).To(Succeed()) + + CleanEphemeralStaging(stagingDir, time.Hour) + + Expect(weights).To(BeAnExistingFile(), "a staged model is not ephemeral scratch") + }) + + It("does nothing when no ephemeral directory exists", func() { + Expect(func() { CleanEphemeralStaging(stagingDir, time.Hour) }).ToNot(Panic()) + }) +}) diff --git a/core/services/worker/worker.go b/core/services/worker/worker.go index 2c48c14ea6e5..6434c3cd6b69 100644 --- a/core/services/worker/worker.go +++ b/core/services/worker/worker.go @@ -159,6 +159,10 @@ func Run(ctx *cliContext.Context, cfg *Config) error { return fmt.Errorf("starting HTTP file transfer server: %w", err) } + // Per-request input files land in stagingDir over that server and nothing + // used to remove them, so a long-lived worker filled its own disk. + StartEphemeralStagingCleanup(shutdownCtx, stagingDir, 0, 0) + // Connect to NATS xlog.Info("Connecting to NATS", "url", sanitize.URL(cfg.NatsURL)) natsClient, err := connectNats() diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index 3d1e42a85a43..54c3abbab106 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -1020,6 +1020,12 @@ Notes: - Upgrade the worker when it does not support the exact model-stop request. - Stop and restart the stale backend only as an operational recovery action. LocalAI keeps it non-routable while durable cleanup is pending. +**A worker fills its own disk over time:** +- A request that carries a file (an image, an audio clip, a video) stages that file to the worker under `/../staging/ephemeral/`. The worker deletes these 6 hours after the request that needed them, and sweeps every 30 minutes plus once at startup, so a worker that crashed mid-request still reclaims the space. +- Releases before this sweep existed kept every staged input for the lifetime of the worker. Delete `/../staging/ephemeral/` on an affected worker once, as the user the worker runs as; the sweep keeps it bounded from then on. +- Staged **model** files are not touched by this. They live beside the ephemeral directory and are not per-request scratch. +- A worker whose volume is genuinely full reports `creating backend process state directory under ...: no space left on device` when a backend starts. + **Requests fail with `stale model config revision` although nobody edited the model:** - A model's stored revision must describe its persisted configuration. Releases before this fix also hashed the per-request prediction parameters, so the first request after a restart pinned the revision to its own `temperature`, `top_p`, `stop` and similar values. Every later request that sent different values was then rejected. - Upgrade the frontend replicas first. After the upgrade the revision is stamped when the configuration is loaded, so it no longer depends on the request body. From d40662cfd5785cce0c60ae215b1d15726dda4fa3 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 23 Aug 2026 20:20:02 +0000 Subject: [PATCH 07/13] fix(model): report why a backend state directory fails process.New applies its options and discards the error they return. When WithTemporaryStateDir could not create a directory, StateDir stayed empty and every later option went unapplied, so the failure surfaced from Run as "mkdir : no such file or directory" naming no path. That message cost a full day of diagnosis on a worker whose volume was full: the real errno was ENOSPC and nothing reported it. The loader now creates the directory itself and returns the underlying error with the path attached. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude Code:claude-opus-5 [golangci-lint] --- pkg/model/process.go | 23 +++++++++++++++++- pkg/model/process_statedir_test.go | 38 ++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 pkg/model/process_statedir_test.go diff --git a/pkg/model/process.go b/pkg/model/process.go index 4fd6041e4508..fb1ed004a1d1 100644 --- a/pkg/model/process.go +++ b/pkg/model/process.go @@ -231,6 +231,16 @@ func (ml *ModelLoader) StartProcess(grpcProcess, id string, serverAddress string return ml.startProcess(grpcProcess, id, serverAddress, args...) } +// newProcessStateDir creates the directory a backend process uses for its pid, +// state and log files, and reports why when it cannot. +func newProcessStateDir() (string, error) { + dir, err := os.MkdirTemp(os.TempDir(), "go-processmanager") + if err != nil { + return "", fmt.Errorf("creating backend process state directory under %s: %w", os.TempDir(), err) + } + return dir, nil +} + func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string, args ...string) (*process.Process, error) { // Make sure the process is executable // Check first if it has executable permissions @@ -261,8 +271,19 @@ func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string // and the GPU would silently fall back to CPU). No-op for other backends. env = append(env, vulkanICDEnv(workDir)...) + // Resolve the state directory here rather than through + // process.WithTemporaryStateDir(). process.New applies its options but + // discards the error they return, so a temp directory that cannot be + // created leaves StateDir empty and every later option unapplied. Run() + // then reported "mkdir : no such file or directory" with no path, hiding + // the real cause (a full volume, or a TMPDIR that no longer resolves). + stateDir, err := newProcessStateDir() + if err != nil { + return nil, err + } + grpcControlProcess := process.New( - process.WithTemporaryStateDir(), + process.WithStateDir(stateDir), process.WithName(filepath.Base(grpcProcess)), process.WithArgs(append(args, []string{"--addr", serverAddress}...)...), process.WithEnvironment(env...), diff --git a/pkg/model/process_statedir_test.go b/pkg/model/process_statedir_test.go new file mode 100644 index 000000000000..207c2991d991 --- /dev/null +++ b/pkg/model/process_statedir_test.go @@ -0,0 +1,38 @@ +package model + +import ( + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Backend process state directory", func() { + It("reports why the state directory could not be created", func() { + // A worker whose volume is full, or whose TMPDIR no longer resolves, + // cannot get a state directory. go-processmanager's New() drops the + // option error, leaving StateDir empty, and Run() then failed with + // "mkdir : no such file or directory" naming no path at all. Resolving + // the directory here keeps the real cause attached. + GinkgoT().Setenv("TMPDIR", filepath.Join(GinkgoT().TempDir(), "does-not-exist")) + + dir, err := newProcessStateDir() + Expect(err).To(HaveOccurred()) + Expect(dir).To(BeEmpty()) + Expect(err.Error()).To(ContainSubstring("backend process state directory")) + Expect(err.Error()).To(ContainSubstring("does-not-exist"), + "the error must name the directory it could not create") + }) + + It("returns a usable directory when the temp location works", func() { + GinkgoT().Setenv("TMPDIR", GinkgoT().TempDir()) + + dir, err := newProcessStateDir() + Expect(err).ToNot(HaveOccurred()) + Expect(dir).ToNot(BeEmpty()) + info, statErr := os.Stat(dir) + Expect(statErr).ToNot(HaveOccurred()) + Expect(info.IsDir()).To(BeTrue()) + }) +}) From ac9969ef4dd6cff3d39343e146be572193a9346e Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 23 Aug 2026 20:26:47 +0000 Subject: [PATCH 08/13] chore: bump go-processmanager to dfa0ed8 Picks up "surface option errors instead of discarding them". New records the error from applying its options and Run returns it, so a process whose state directory cannot be created reports the real cause instead of failing later inside os.MkdirAll("") with an empty path. LocalAI already resolves that directory itself, so this covers the other process.New call sites rather than changing behaviour on the backend start path. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude Code:claude-opus-5 --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 8e8084cec68a..5418b351de5d 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( github.com/modelcontextprotocol/go-sdk v1.5.0 github.com/mudler/cogito v0.11.1-0.20260721122412-6eece18a6bb6 github.com/mudler/edgevpn v0.34.0 - github.com/mudler/go-processmanager v0.1.2-0.20260720195933-3d64f5c974fc + github.com/mudler/go-processmanager v0.1.2-0.20260823202314-dfa0ed852db6 github.com/mudler/memory v0.0.0-20260406210934-424c1ecf2cf8 github.com/mudler/nib v0.6.0 github.com/mudler/xlog v0.0.6 diff --git a/go.sum b/go.sum index 7bcbec8e6fd3..6169840bd308 100644 --- a/go.sum +++ b/go.sum @@ -1027,6 +1027,8 @@ github.com/mudler/go-piper v0.0.0-20241023091659-2494246fd9fc h1:RxwneJl1VgvikiX github.com/mudler/go-piper v0.0.0-20241023091659-2494246fd9fc/go.mod h1:O7SwdSWMilAWhBZMK9N9Y/oBDyMMzshE3ju8Xkexwig= github.com/mudler/go-processmanager v0.1.2-0.20260720195933-3d64f5c974fc h1:NEFmd7+JoImN5dZI81/vcBRjtMg+GfEa7nEjQ029hd0= github.com/mudler/go-processmanager v0.1.2-0.20260720195933-3d64f5c974fc/go.mod h1:h6kmHUZeafr+k5hRYpGLMzJFH4hItHffgpRo2QIkP+o= +github.com/mudler/go-processmanager v0.1.2-0.20260823202314-dfa0ed852db6 h1:/nFm1Ttf8g1BnWtEth986JR34pCh9rzae5A2vKBZosc= +github.com/mudler/go-processmanager v0.1.2-0.20260823202314-dfa0ed852db6/go.mod h1:h6kmHUZeafr+k5hRYpGLMzJFH4hItHffgpRo2QIkP+o= github.com/mudler/localrecall v0.6.3 h1:uXOrP9JmetzxgVKzSrawviyBHZfAcvPBBIrvVUdZjDA= github.com/mudler/localrecall v0.6.3/go.mod h1:28k5n19raUrkuwXkacdNsBlj8yuSnGhpT16tu+2+4dU= github.com/mudler/memory v0.0.0-20260406210934-424c1ecf2cf8 h1:Ry8RiWy8fZ6Ff4E7dPmjRsBrnHOnPeOOj2LhCgyjQu0= From c541dbeef40786ac004d48bb993b79f13a945ea7 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 23 Aug 2026 20:44:43 +0000 Subject: [PATCH 09/13] fix(distributed): check a node answers before scheduling onto it A node's status comes from its HTTP heartbeat. Backend installs travel over NATS. The two are independent, so a worker that dies stops answering on the bus at once but stays healthy in the database until its heartbeat ages out. Inside that window the scheduler picked a node it could not reach, and the request failed with "no responders available" rather than moving to a node that was up. The scheduler now probes the node it selected and, when nothing answers, marks it unhealthy and selects again. The demotion is what makes the retry terminate: the next selection reads only healthy nodes. It also tells the other frontends what this one learned, so the cluster does not rediscover a dead worker one failed request at a time. Only nats.ErrNoResponders counts as absent. A worker that answers slowly stays eligible, because dropping it would cost capacity that is really there. The probe reuses the models.running subject: a new subject would go unanswered by workers that have not been upgraded, and every one of them would then look dead. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude Code:claude-opus-5 [golangci-lint] --- core/services/nodes/interfaces.go | 1 + core/services/nodes/model_router_test.go | 4 + core/services/nodes/router.go | 54 ++++---- core/services/nodes/router_liveness.go | 60 +++++++++ .../nodes/router_nats_liveness_test.go | 118 ++++++++++++++++++ core/services/nodes/router_test.go | 31 ++++- core/services/nodes/unloader.go | 25 ++++ docs/content/features/distributed-mode.md | 6 + 8 files changed, 276 insertions(+), 23 deletions(-) create mode 100644 core/services/nodes/router_liveness.go create mode 100644 core/services/nodes/router_nats_liveness_test.go diff --git a/core/services/nodes/interfaces.go b/core/services/nodes/interfaces.go index c204752de19b..93399bc145dc 100644 --- a/core/services/nodes/interfaces.go +++ b/core/services/nodes/interfaces.go @@ -60,6 +60,7 @@ type ModelRouter interface { GetNodeLabels(ctx context.Context, nodeID string) ([]NodeLabel, error) FindNodesWithModel(ctx context.Context, modelName string) ([]BackendNode, error) LoadedReplicaStats(ctx context.Context, modelName string, candidateNodeIDs []string) ([]ReplicaCandidate, error) + MarkUnhealthy(ctx context.Context, nodeID string) error LoadJobStore } diff --git a/core/services/nodes/model_router_test.go b/core/services/nodes/model_router_test.go index 43002006a1b0..13e36b12d40a 100644 --- a/core/services/nodes/model_router_test.go +++ b/core/services/nodes/model_router_test.go @@ -250,3 +250,7 @@ var _ = Describe("ModelRouterAdapter", func() { }) }) }) + +func (f *fakeModelRouterForSmartRouter) MarkUnhealthy(_ context.Context, _ string) error { + return nil +} diff --git a/core/services/nodes/router.go b/core/services/nodes/router.go index 465b2d44ec9a..14b19092e5f7 100644 --- a/core/services/nodes/router.go +++ b/core/services/nodes/router.go @@ -1089,34 +1089,44 @@ func (r *SmartRouter) scheduleNewModel(ctx context.Context, backendType, modelID // If freeSlotNodes is empty (everyone full), candidateNodeIDs is whatever // it was — we'll fall through to eviction below. - var node *BackendNode - - if estimatedVRAM > 0 { - if candidateNodeIDs != nil { - node, err = r.registry.FindNodeWithVRAMFromSet(ctx, estimatedVRAM, candidateNodeIDs) - } else { - node, err = r.registry.FindNodeWithVRAM(ctx, estimatedVRAM) - } - if err != nil { - xlog.Warn("No nodes with enough VRAM, falling back to standard scheduling", - "required_vram", vram.FormatBytes(estimatedVRAM), "error", err) + // Node choice is wrapped in a liveness check: a node's stored status comes + // from its HTTP heartbeat, which is a different channel from the bus that + // carries the install. A worker that has died stops answering on the bus at + // once but stays healthy in the database until its heartbeat ages out, so + // without this the scheduler could commit to a node it cannot reach. + selectNode := func() *BackendNode { + var candidate *BackendNode + var selErr error + if estimatedVRAM > 0 { + if candidateNodeIDs != nil { + candidate, selErr = r.registry.FindNodeWithVRAMFromSet(ctx, estimatedVRAM, candidateNodeIDs) + } else { + candidate, selErr = r.registry.FindNodeWithVRAM(ctx, estimatedVRAM) + } + if selErr != nil { + xlog.Warn("No nodes with enough VRAM, falling back to standard scheduling", + "required_vram", vram.FormatBytes(estimatedVRAM), "error", selErr) + } } - } - if node == nil { - if candidateNodeIDs != nil { - node, err = r.registry.FindIdleNodeFromSet(ctx, candidateNodeIDs) - if err != nil { - node, err = r.registry.FindLeastLoadedNodeFromSet(ctx, candidateNodeIDs) - } - } else { - node, err = r.registry.FindIdleNode(ctx) - if err != nil { - node, err = r.registry.FindLeastLoadedNode(ctx) + if candidate == nil { + if candidateNodeIDs != nil { + candidate, selErr = r.registry.FindIdleNodeFromSet(ctx, candidateNodeIDs) + if selErr != nil { + candidate, _ = r.registry.FindLeastLoadedNodeFromSet(ctx, candidateNodeIDs) + } + } else { + candidate, selErr = r.registry.FindIdleNode(ctx) + if selErr != nil { + candidate, _ = r.registry.FindLeastLoadedNode(ctx) + } } } + return candidate } + node := r.pickReachableNode(ctx, selectNode) + // 4. Preemptive eviction: if no suitable node found, evict the LRU model with zero in-flight if node == nil { evictedNode, evictErr := r.evictLRUAndFreeNode(ctx) diff --git a/core/services/nodes/router_liveness.go b/core/services/nodes/router_liveness.go new file mode 100644 index 000000000000..88646162fde2 --- /dev/null +++ b/core/services/nodes/router_liveness.go @@ -0,0 +1,60 @@ +package nodes + +import ( + "context" + "errors" + + "github.com/mudler/xlog" + "github.com/nats-io/nats.go" +) + +// maxNodeLivenessRetries bounds how many unreachable nodes a single scheduling +// attempt discards before giving up. Each discarded node is marked unhealthy, +// so the bound only has to cover one burst of dead workers rather than the +// whole fleet. +const maxNodeLivenessRetries = 3 + +// nodeAnswersOnBus reports whether a node still has a live subscription. +// +// Only nats.ErrNoResponders means "absent". Any other outcome, a timeout or a +// transport hiccup, leaves the node eligible: wrongly excluding a node that is +// merely slow costs real capacity, while the install that follows already +// reports its own failure. When no command sender is configured there is no bus +// to consult and every node is treated as reachable, which preserves the +// behaviour of deployments that do not run one. +func (r *SmartRouter) nodeAnswersOnBus(node *BackendNode) bool { + if r.unloader == nil || node == nil { + return true + } + err := r.unloader.PingNode(node.ID) + return !errors.Is(err, nats.ErrNoResponders) +} + +// pickReachableNode calls selectNode until it yields a node that still answers +// on the bus, and returns nil when it cannot find one. +// +// A node that does not answer is marked unhealthy before the next attempt. That +// both removes it from the next selection, which queries only healthy nodes, +// and tells every other scheduler in the cluster what this one just learned, so +// the discovery is not repeated one failed request at a time. +func (r *SmartRouter) pickReachableNode(ctx context.Context, selectNode func() *BackendNode) *BackendNode { + for range maxNodeLivenessRetries { + node := selectNode() + if node == nil { + return nil + } + if r.nodeAnswersOnBus(node) { + return node + } + xlog.Warn("Scheduled node is not answering on the bus, marking unhealthy and re-scheduling", + "node", node.Name, "nodeID", node.ID) + if err := r.registry.MarkUnhealthy(ctx, node.ID); err != nil { + // Without the demotion the next selection would hand back the same + // node, so stop rather than spin. + xlog.Warn("Failed to mark unreachable node unhealthy", + "node", node.Name, "nodeID", node.ID, "error", err) + return nil + } + } + return nil +} diff --git a/core/services/nodes/router_nats_liveness_test.go b/core/services/nodes/router_nats_liveness_test.go new file mode 100644 index 000000000000..ec4820c9f0a8 --- /dev/null +++ b/core/services/nodes/router_nats_liveness_test.go @@ -0,0 +1,118 @@ +package nodes + +import ( + "context" + "errors" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// A node's stored status comes from its HTTP heartbeat, but work is dispatched +// over NATS. A worker that dies stops answering on the bus immediately and +// keeps its healthy status until the heartbeat ages out, so the scheduler could +// commit to a node it could not reach. The request then failed outright with +// "no responders available" rather than moving to a node that was actually up. +var _ = Describe("Scheduling past a node that left the bus", func() { + var ( + reg *fakeModelRouter + fake *fakeUnloader + router *SmartRouter + ) + + newNode := func(id string) *BackendNode { + return &BackendNode{ID: id, Name: id, Address: id + ":50051"} + } + + // selectorReturning hands back each node in turn, mimicking a scheduler + // that re-picks after the previous choice was demoted. + selectorReturning := func(nodes ...*BackendNode) func() *BackendNode { + i := 0 + return func() *BackendNode { + if i >= len(nodes) { + return nil + } + n := nodes[i] + i++ + return n + } + } + + BeforeEach(func() { + reg = &fakeModelRouter{} + fake = &fakeUnloader{deadNodes: map[string]bool{}} + router = NewSmartRouter(reg, SmartRouterOptions{Unloader: fake}) + }) + + It("passes over a node that no longer answers and takes one that does", func() { + dead, alive := newNode("dead-node"), newNode("alive-node") + fake.deadNodes["dead-node"] = true + + picked := router.pickReachableNode(context.Background(), selectorReturning(dead, alive)) + + Expect(picked).ToNot(BeNil()) + Expect(picked.ID).To(Equal("alive-node")) + Expect(fake.pingCalls).To(Equal([]string{"dead-node", "alive-node"})) + }) + + It("demotes the absent node so other schedulers stop choosing it", func() { + dead, alive := newNode("dead-node"), newNode("alive-node") + fake.deadNodes["dead-node"] = true + + router.pickReachableNode(context.Background(), selectorReturning(dead, alive)) + + Expect(reg.markedUnhealthy).To(Equal([]string{"dead-node"})) + }) + + It("takes the first node when it answers, without probing further", func() { + first, second := newNode("first"), newNode("second") + + picked := router.pickReachableNode(context.Background(), selectorReturning(first, second)) + + Expect(picked.ID).To(Equal("first")) + Expect(fake.pingCalls).To(Equal([]string{"first"})) + }) + + It("gives up rather than spinning when every node is gone", func() { + a, b, c, d := newNode("a"), newNode("b"), newNode("c"), newNode("d") + for _, id := range []string{"a", "b", "c", "d"} { + fake.deadNodes[id] = true + } + + picked := router.pickReachableNode(context.Background(), selectorReturning(a, b, c, d)) + + Expect(picked).To(BeNil()) + Expect(len(fake.pingCalls)).To(BeNumerically("<=", maxNodeLivenessRetries)) + }) + + It("stops when the demotion itself fails, so it cannot loop on one node", func() { + dead := newNode("dead-node") + fake.deadNodes["dead-node"] = true + reg.markUnhealthyErr = errors.New("database is down") + + picked := router.pickReachableNode(context.Background(), selectorReturning(dead, dead, dead)) + + Expect(picked).To(BeNil()) + Expect(fake.pingCalls).To(Equal([]string{"dead-node"})) + }) + + // Only a no-responders answer proves absence. Excluding a node that is + // merely slow would cost real capacity. + It("keeps a node that answers slowly or errors for another reason", func() { + slow := newNode("slow-node") + fake.pingErr = errors.New("timeout waiting for reply") + + picked := router.pickReachableNode(context.Background(), selectorReturning(slow)) + + Expect(picked).ToNot(BeNil()) + Expect(picked.ID).To(Equal("slow-node")) + Expect(reg.markedUnhealthy).To(BeEmpty()) + }) + + It("treats every node as reachable when no command sender is configured", func() { + plain := NewSmartRouter(reg, SmartRouterOptions{}) + node := newNode("only-node") + + Expect(plain.pickReachableNode(context.Background(), selectorReturning(node))).To(Equal(node)) + }) +}) diff --git a/core/services/nodes/router_test.go b/core/services/nodes/router_test.go index 96db9b93fcb9..0f0938335778 100644 --- a/core/services/nodes/router_test.go +++ b/core/services/nodes/router_test.go @@ -17,6 +17,7 @@ import ( "github.com/mudler/LocalAI/pkg/distributedhdr" grpc "github.com/mudler/LocalAI/pkg/grpc" pb "github.com/mudler/LocalAI/pkg/grpc/proto" + "github.com/nats-io/nats.go" ggrpc "google.golang.org/grpc" "google.golang.org/protobuf/proto" "gorm.io/gorm" @@ -61,6 +62,10 @@ func (f *fakeFileStager) ListRemoteDir(_ context.Context, _, _ string) ([]string // fakeModelRouter implements ModelRouter with configurable return values. type fakeModelRouter struct { + // markedUnhealthy records nodes demoted by the scheduler's liveness check. + markedUnhealthy []string + markUnhealthyErr error + fakeLoadJobStore // FindAndLockNodeWithModel returns @@ -474,7 +479,15 @@ type fakeUnloader struct { stopCalls []string // "nodeID:model" stopErr error unloadCalls []string - unloadErr error + + // deadNodes names the nodes PingNode reports as absent from the bus, and + // pingCalls records every node it was asked about, in order. + deadNodes map[string]bool + pingCalls []string + // pingErr is returned for nodes not in deadNodes, so a spec can model a + // node that is reachable but answering badly. + pingErr error + unloadErr error } // installCall captures the args we care about when asserting that the @@ -532,6 +545,22 @@ func (f *fakeUnloader) UnloadModelOnNode(nodeID, modelName string) error { return f.unloadErr } +func (f *fakeModelRouter) MarkUnhealthy(_ context.Context, nodeID string) error { + f.markedUnhealthy = append(f.markedUnhealthy, nodeID) + return f.markUnhealthyErr +} + +func (f *fakeUnloader) PingNode(nodeID string) error { + f.mu.Lock() + f.pingCalls = append(f.pingCalls, nodeID) + dead := f.deadNodes[nodeID] + f.mu.Unlock() + if dead { + return nats.ErrNoResponders + } + return f.pingErr +} + // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- diff --git a/core/services/nodes/unloader.go b/core/services/nodes/unloader.go index 8d47d71a622d..3b1cd15b8955 100644 --- a/core/services/nodes/unloader.go +++ b/core/services/nodes/unloader.go @@ -36,6 +36,10 @@ type NodeCommandSender interface { ListBackends(nodeID string) (*messaging.BackendListReply, error) StopBackend(nodeID, backend string) error UnloadModelOnNode(nodeID, modelName string) error + // PingNode reports whether the node is still subscribed on the bus. It + // returns nats.ErrNoResponders when nothing answers for the node, which is + // the only condition callers may read as "this node cannot be given work". + PingNode(nodeID string) error } // RemoteUnloaderAdapter implements NodeCommandSender and model.RemoteModelUnloader @@ -360,6 +364,27 @@ func (a *RemoteUnloaderAdapter) ListBackends(nodeID string) (*messaging.BackendL return messaging.RequestJSON[messaging.BackendListRequest, messaging.BackendListReply](a.nats, subject, messaging.BackendListRequest{}, 30*time.Second) } +// PingNode checks that a worker still has a live subscription on the bus. +// +// A node's status in the database comes from its HTTP heartbeat, which is a +// separate channel from NATS. A worker that has died stops answering on NATS +// at once but keeps its healthy status until the heartbeat ages out, so the +// scheduler could pick a node that could not be given work and the request +// failed with "no responders available". +// +// It reuses the models.running subject rather than a dedicated ping subject on +// purpose: a new subject would go unanswered by any worker that has not been +// upgraded yet, and this check would then report every one of them as dead. +// The worker answers out of its in-memory process table, so a live node +// replies immediately, and NATS reports no-responders without waiting out the +// timeout. +func (a *RemoteUnloaderAdapter) PingNode(nodeID string) error { + subject := messaging.SubjectNodeModelsRunning(nodeID) + _, err := messaging.RequestJSON[messaging.ModelsRunningRequest, messaging.ModelsRunningReply]( + a.nats, subject, messaging.ModelsRunningRequest{}, 5*time.Second) + return err +} + // ListRunningModels asks a worker node which model backend processes it // currently has running, via NATS request-reply. // diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index 54c3abbab106..667408fbbe30 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -1020,6 +1020,12 @@ Notes: - Upgrade the worker when it does not support the exact model-stop request. - Stop and restart the stale backend only as an operational recovery action. LocalAI keeps it non-routable while durable cleanup is pending. +**A request fails with `nats: no responders available for request`:** +- The chosen worker was not subscribed on the bus when the frontend tried to install the backend on it. A node's status comes from its HTTP heartbeat, which is a separate channel: a worker that stops stays `healthy` until that heartbeat ages out. +- The scheduler now checks that a node still answers on the bus before it commits to it, marks one that does not as unhealthy, and picks another. A request should therefore see this only when no reachable node is left. +- Only a no-responders answer counts as absent. A worker that answers slowly stays eligible, because excluding it would cost capacity that is really there. +- Check the worker process is running and its NATS connection is up. `Scheduled node is not answering on the bus` in the frontend log names each node demoted this way. + **A worker fills its own disk over time:** - A request that carries a file (an image, an audio clip, a video) stages that file to the worker under `/../staging/ephemeral/`. The worker deletes these 6 hours after the request that needed them, and sweeps every 30 minutes plus once at startup, so a worker that crashed mid-request still reclaims the space. - Releases before this sweep existed kept every staged input for the lifetime of the worker. Delete `/../staging/ephemeral/` on an affected worker once, as the user the worker runs as; the sweep keeps it bounded from then on. From e6269e3cdd09f902bf22671bf9593eeaf3bbb30e Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 23 Aug 2026 21:07:29 +0000 Subject: [PATCH 10/13] fix(distributed): reclaim replica slots held by abandoned loads A replica row in staging or loading holds its slot, because slot allocation counts every state except unloading. Nothing ever reclaimed such a row: every reconciler pass and the router's eviction query filter state = "loaded", and the per-model probe skips rows without an address, which is exactly what a row that never finished loading has. So a worker that dropped out mid-transfer left a row that pinned the only replica slot for that model on that node. Scheduling then found no free slot and eviction found nothing it was allowed to evict, and the request failed with "no replica slot on and eviction failed: all models busy". The state persisted until an operator intervened. The reconciler now reclaims a row stuck before serving when no load job is driving it. Ownership is decided by the job's LastProgress heartbeat, not by elapsed time: staging a large checkpoint legitimately runs for a long while without touching the replica row, so a deadline would either be a model-size cliff or reclaim a healthy transfer. That heartbeat is the same signal job takeover already trusts. Any error reading the job leaves the slot held, because holding one for another pass costs a scheduling opportunity while a wrong reclaim restarts a multi-gigabyte transfer. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude Code:claude-opus-5 [golangci-lint] --- core/services/nodes/reconciler.go | 8 +- .../nodes/reconciler_abandoned_load.go | 98 +++++++++++++ .../nodes/reconciler_abandoned_load_test.go | 133 ++++++++++++++++++ docs/content/features/distributed-mode.md | 6 + 4 files changed, 243 insertions(+), 2 deletions(-) create mode 100644 core/services/nodes/reconciler_abandoned_load.go create mode 100644 core/services/nodes/reconciler_abandoned_load_test.go diff --git a/core/services/nodes/reconciler.go b/core/services/nodes/reconciler.go index f7fefc8356c8..88ca28bbe2fb 100644 --- a/core/services/nodes/reconciler.go +++ b/core/services/nodes/reconciler.go @@ -278,8 +278,9 @@ func (rc *ReplicaReconciler) reconcileOnce(ctx context.Context) { // reconcileState runs the state-reconciliation passes: drain pending backend // ops for freshly-healthy nodes, reconcile registry rows against what workers -// report they are running, then port-probe whatever is left. All passes are -// best-effort: a failure on one node doesn't stop the rest. +// report they are running, port-probe whatever is left, then reclaim replica +// slots held by loads nobody is driving. All passes are best-effort: a failure +// on one node doesn't stop the rest. // // Order matters. The worker pass runs first and refreshes updated_at for every // model a worker vouches for, which takes those rows out of the port prober's @@ -292,6 +293,9 @@ func (rc *ReplicaReconciler) reconcileState(ctx context.Context) { rc.reconcileNodeProcesses(ctx) rc.probeLoadedModels(ctx) rc.sweepLeakedInFlight(ctx) + // Runs last: the passes above can move a row into a serving state, and a + // row that just became loaded is no longer this sweeper's business. + rc.reclaimAbandonedLoads(ctx) } // drainPendingBackendOps retries queued backend ops whose next_retry_at has diff --git a/core/services/nodes/reconciler_abandoned_load.go b/core/services/nodes/reconciler_abandoned_load.go new file mode 100644 index 000000000000..9b6b7b65430b --- /dev/null +++ b/core/services/nodes/reconciler_abandoned_load.go @@ -0,0 +1,98 @@ +package nodes + +import ( + "context" + "errors" + "time" + + "github.com/mudler/xlog" + "gorm.io/gorm" +) + +const ( + // abandonedLoadGrace is how long a replica row may sit in a pre-serving + // state before the sweeper will consider it at all. + // + // It exists to cover the window between creating the replica row and + // writing the load job that vouches for it. Without it a load could be + // reclaimed in the moment before its own job row exists. It is not the + // thing that protects a long transfer: the job heartbeat does that. + abandonedLoadGrace = 5 * time.Minute +) + +// preServingStates are the replica states that hold a slot without being able +// to serve a request. NextFreeReplicaIndex counts every state except +// "unloading", so a row parked in one of these occupies capacity while +// answering nothing. +var preServingStates = []string{"loading", "staging"} + +// reclaimAbandonedLoads removes replica rows whose load will never finish. +// +// The other reconciler passes and the router's eviction query all filter +// state = "loaded", and the per-model probe skips rows without an address, so +// nothing reclaimed a row that never got that far. On a node with one replica +// slot per model, a single interrupted transfer made the model unschedulable +// there until an operator intervened: scheduling saw no free slot, and eviction +// found nothing it was allowed to evict. +// +// A row is abandoned when no live load job vouches for it. Ownership is decided +// by the job's LastProgress heartbeat rather than elapsed time, because staging +// a large checkpoint legitimately runs for a long while without touching the +// replica row. That is the same signal job takeover already trusts, so a +// transfer this sweeper reclaims is one no replica is still driving. +func (rc *ReplicaReconciler) reclaimAbandonedLoads(ctx context.Context) { + if rc.db == nil { + return + } + + cutoff := time.Now().Add(-abandonedLoadGrace) + var stuck []NodeModel + if err := rc.db.WithContext(ctx). + Where("state IN ? AND updated_at < ?", preServingStates, cutoff). + Find(&stuck).Error; err != nil { + xlog.Warn("Reconciler: failed to list replicas stuck before serving", "error", err) + return + } + + now := time.Now() + for _, row := range stuck { + if rc.loadStillRunning(ctx, row.ModelName, now) { + continue + } + if err := rc.registry.RemoveNodeModel(ctx, row.NodeID, row.ModelName, row.ReplicaIndex); err != nil { + xlog.Warn("Reconciler: failed to reclaim abandoned load", + "node", row.NodeID, "model", row.ModelName, "replica", row.ReplicaIndex, + "state", row.State, "error", err) + continue + } + xlog.Warn("Reconciler: reclaimed a replica slot held by a load nobody is driving", + "node", row.NodeID, "model", row.ModelName, "replica", row.ReplicaIndex, "state", row.State) + } +} + +// loadStillRunning reports whether a load job is actively driving this model. +// +// A missing job means nobody is loading it. A failed job has already given up. +// An orphaned job stopped heartbeating, which is the condition another replica +// uses to take it over, so the transfer behind it is not progressing either. +// Any error reading the job is treated as "still running": leaving a slot held +// for one more pass costs a scheduling opportunity, while removing a row out +// from under a live transfer would restart a multi-gigabyte load. +func (rc *ReplicaReconciler) loadStillRunning(ctx context.Context, modelName string, now time.Time) bool { + job, err := rc.registry.GetLoadJob(ctx, modelName) + if errors.Is(err, gorm.ErrRecordNotFound) { + return false + } + if err != nil { + xlog.Warn("Reconciler: cannot read load job, leaving the replica slot held", + "model", modelName, "error", err) + return true + } + if job == nil { + return false + } + if job.State == LoadJobStateFailed { + return false + } + return !job.IsOrphaned(now) +} diff --git a/core/services/nodes/reconciler_abandoned_load_test.go b/core/services/nodes/reconciler_abandoned_load_test.go new file mode 100644 index 000000000000..6740b13aafa7 --- /dev/null +++ b/core/services/nodes/reconciler_abandoned_load_test.go @@ -0,0 +1,133 @@ +package nodes + +import ( + "context" + "runtime" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/gorm" + + "github.com/mudler/LocalAI/core/services/testutil" +) + +// A replica row in loading or staging holds its slot: NextFreeReplicaIndex +// counts every state except unloading. Nothing reclaimed such a row. Every +// reconciler sweep and the router's eviction query filter state = "loaded", and +// the per-model health probe skips rows with no address, which is exactly what a +// row that never finished loading has. So a worker that dropped out mid-transfer +// left a row that pinned the only replica slot on that node for that model, and +// the next request failed with "no replica slot ... all models busy". +// +// Elapsed time alone cannot decide this: staging a large checkpoint legitimately +// runs for tens of minutes. The load job's LastProgress heartbeat is the +// discriminator, the same signal job takeover already trusts. +var _ = Describe("ReplicaReconciler — abandoned load sweeper", func() { + var ( + db *gorm.DB + registry *NodeRegistry + node *BackendNode + rc *ReplicaReconciler + ) + + BeforeEach(func() { + if runtime.GOOS == "darwin" { + Skip("testcontainers requires Docker, not available on macOS CI") + } + db = testutil.SetupTestDB() + var err error + registry, err = NewNodeRegistry(db) + Expect(err).ToNot(HaveOccurred()) + node = &BackendNode{Name: "n1", NodeType: NodeTypeBackend, Address: "10.0.0.1:50051"} + Expect(registry.Register(context.Background(), node, true)).To(Succeed()) + rc = NewReplicaReconciler(ReplicaReconcilerOptions{Registry: registry, DB: db}) + }) + + // seedReplica creates a replica row in the given state, aged so it is past + // the sweeper's grace period unless stated otherwise. + seedReplica := func(model, state string, age time.Duration) { + Expect(db.Create(&NodeModel{ + ID: model + "-row", + NodeID: node.ID, + ModelName: model, + State: state, + UpdatedAt: time.Now().Add(-age), + }).Error).To(Succeed()) + } + + seedJob := func(model, state string, sinceProgress time.Duration) { + Expect(db.Create(&ModelLoadJob{ + TrackingKey: model, + State: state, + OwnerReplica: "someone", + LastProgress: time.Now().Add(-sinceProgress), + CreatedAt: time.Now().Add(-sinceProgress), + UpdatedAt: time.Now().Add(-sinceProgress), + }).Error).To(Succeed()) + } + + rowExists := func(model string) bool { + var count int64 + Expect(db.Model(&NodeModel{}).Where("model_name = ?", model).Count(&count).Error).To(Succeed()) + return count > 0 + } + + It("reclaims a staging row whose load job has stopped heartbeating", func() { + seedReplica("abandoned", "staging", time.Hour) + seedJob("abandoned", LoadJobStateStaging, 30*time.Minute) + + rc.reclaimAbandonedLoads(context.Background()) + + Expect(rowExists("abandoned")).To(BeFalse()) + }) + + It("reclaims a loading row that has no load job at all", func() { + seedReplica("orphan", "loading", time.Hour) + + rc.reclaimAbandonedLoads(context.Background()) + + Expect(rowExists("orphan")).To(BeFalse()) + }) + + It("keeps a long transfer whose job is still heartbeating", func() { + // The row itself is old, because staging does not touch it. Only the + // job proves the transfer is alive. + seedReplica("big-model", "staging", time.Hour) + seedJob("big-model", LoadJobStateStaging, time.Second) + + rc.reclaimAbandonedLoads(context.Background()) + + Expect(rowExists("big-model")).To(BeTrue(), "a live transfer must never be reclaimed") + }) + + It("leaves a freshly created row alone while its job row is still being written", func() { + seedReplica("just-started", "loading", time.Second) + + rc.reclaimAbandonedLoads(context.Background()) + + Expect(rowExists("just-started")).To(BeTrue()) + }) + + It("does not touch loaded replicas, which the other sweeps own", func() { + seedReplica("serving", "loaded", time.Hour) + + rc.reclaimAbandonedLoads(context.Background()) + + Expect(rowExists("serving")).To(BeTrue()) + }) + + It("frees the slot so the model can be scheduled on that node again", func() { + seedReplica("wedged", "staging", time.Hour) + seedJob("wedged", LoadJobStateFailed, time.Minute) + + _, err := registry.NextFreeReplicaIndex(context.Background(), node.ID, "wedged", 1) + Expect(err).To(MatchError(ErrNoFreeSlot), "precondition: the stuck row holds the only slot") + + rc.reclaimAbandonedLoads(context.Background()) + + idx, err := registry.NextFreeReplicaIndex(context.Background(), node.ID, "wedged", 1) + Expect(err).ToNot(HaveOccurred()) + Expect(idx).To(Equal(0)) + }) +}) diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index 667408fbbe30..d82616024928 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -1020,6 +1020,12 @@ Notes: - Upgrade the worker when it does not support the exact model-stop request. - Stop and restart the stale backend only as an operational recovery action. LocalAI keeps it non-routable while durable cleanup is pending. +**A model cannot be scheduled on a node that looks free (`no replica slot ... all models busy, cannot evict`):** +- A replica row in `staging` or `loading` holds its slot: slot allocation counts every state except `unloading`. If a worker drops out mid-transfer, that row never reaches `loaded`, and eviction only ever considers `loaded` replicas, so on a node with one replica slot per model the model became unschedulable there. +- The reconciler now reclaims a replica row stuck before serving when no load job is still driving it, and the freed slot is immediately reusable. +- Liveness is decided by the load job's progress heartbeat, not by elapsed time. Staging a large checkpoint legitimately runs for a long time without touching the replica row, so a transfer that is still progressing is never reclaimed however long it takes. +- `Reconciler: reclaimed a replica slot held by a load nobody is driving` names each row reclaimed this way. + **A request fails with `nats: no responders available for request`:** - The chosen worker was not subscribed on the bus when the frontend tried to install the backend on it. A node's status comes from its HTTP heartbeat, which is a separate channel: a worker that stops stays `healthy` until that heartbeat ages out. - The scheduler now checks that a node still answers on the bus before it commits to it, marks one that does not as unhealthy, and picks another. A request should therefore see this only when no reachable node is left. From 7a78ec82ebcea79c360ea35b04a5a61348c1ecea Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Sun, 23 Aug 2026 23:46:34 +0200 Subject: [PATCH 11/13] chore(model-gallery): :arrow_up: update checksum (#11690) :arrow_up: Checksum updates in gallery/index.yaml Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- gallery/index.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gallery/index.yaml b/gallery/index.yaml index 31cb6a925c34..c9a240db2178 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -50,8 +50,8 @@ use_tokenizer_template: true files: - filename: llama-cpp/models/Huihui-Qwen3.8-27B-abliterated-bf16/Huihui-Qwen3.8-27B-abliterated-bf16.gguf - sha256: a64a5e5464d7d0ea7ffcbc937cf28f8a7bc9b0a6e87be6034e6b854418d5abd5 uri: https://huggingface.co/huihui-ai/Huihui-Qwen3.8-27B-abliterated-GGUF/resolve/main/Huihui-Qwen3.8-27B-abliterated-bf16.gguf + sha256: b880f2042df16a9a493800c83f1ee16b7cd46e8ca695f193655940a1949e9097 - filename: llama-cpp/mmproj/Huihui-Qwen3.8-27B-abliterated-bf16/mmproj-model-bf16.gguf sha256: c9a09064683620bea3d3bfed5d4462e1a97a7d2fff7e5045d6862a0a85eeb5b5 uri: https://huggingface.co/huihui-ai/Huihui-Qwen3.8-27B-abliterated-GGUF/resolve/main/mmproj-model-bf16.gguf @@ -1180,7 +1180,7 @@ files: - filename: llama-cpp/models/nemotron-3.5-lightning-30b-a3b/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-Q8_0.gguf uri: huggingface://ggml-org/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-Q8_0.gguf - sha256: d4cc4c9fffaa356db8b49cfaba9233609cfb29b4dd89d827665210dc1cc8dbbb + sha256: b0e25ce2d301930e706d549a59d18c5969dcec664fdf7466435287b03fefda36 - &muse-glimmer-30b name: "muse-glimmer-30b" variants: @@ -18663,11 +18663,11 @@ model: SmolVLM2-256M-Video-Instruct-Q8_0.gguf files: - filename: SmolVLM2-256M-Video-Instruct-Q8_0.gguf - sha256: af7ce9951a2f46c4f6e5def253e5b896ca5e417010e7a9949fdc9e5175c27767 uri: huggingface://ggml-org/SmolVLM2-256M-Video-Instruct-GGUF/SmolVLM2-256M-Video-Instruct-Q8_0.gguf + sha256: 1202d1c54493bddff5b0ecbc36fcb7520ff720b9fa3d7224aeb293581f90529a - filename: mmproj-SmolVLM2-256M-Video-Instruct-Q8_0.gguf - sha256: d34913a588464ff7215f086193e0426a4f045eaba74456ee5e2667d8ed6798b1 uri: huggingface://ggml-org/SmolVLM2-256M-Video-Instruct-GGUF/mmproj-SmolVLM2-256M-Video-Instruct-Q8_0.gguf + sha256: 05d5751132244a6ebd64cba9b34898c0d874b2cb78159d758e1d4da3aad91581 - name: qwen3-30b-a3b url: github:mudler/LocalAI/gallery/qwen3.yaml@master urls: From eadc005b861a715b2efa296bee60caaf11210740 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Sun, 23 Aug 2026 23:49:32 +0200 Subject: [PATCH 12/13] chore: :arrow_up: Update antirez/ds4 to `c1d4597a80e300b803dc642519718f2c999589da` (#11685) :arrow_up: Update antirez/ds4 Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/cpp/ds4/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/cpp/ds4/Makefile b/backend/cpp/ds4/Makefile index bdf126122160..5e73c1328bbb 100644 --- a/backend/cpp/ds4/Makefile +++ b/backend/cpp/ds4/Makefile @@ -1,10 +1,10 @@ # ds4 backend Makefile. # -# Upstream pin lives below as DS4_VERSION?=84cc882352757baf628a1776badf7cc54d584e28 +# Upstream pin lives below as DS4_VERSION?=c1d4597a80e300b803dc642519718f2c999589da # (.github/bump_deps.sh) can find and update it - matches the # llama-cpp / ik-llama-cpp / turboquant convention. -DS4_VERSION?=84cc882352757baf628a1776badf7cc54d584e28 +DS4_VERSION?=c1d4597a80e300b803dc642519718f2c999589da DS4_REPO?=https://github.com/antirez/ds4 CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) From 3953448f600ebf0e5a4b305c281d3d5790316a07 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 23 Aug 2026 22:17:57 +0000 Subject: [PATCH 13/13] fix(distributed): resync stored config revisions at startup The controller pins a model's replicas to a stored revision and rejects any request carrying a different one. Nothing ever re-derived that value from the configuration on disk: it moved only on an edit, a gallery install, or a peer's change broadcast. An inference request may only establish a revision, never replace one. So any other way for the two to diverge left the model permanently unroutable. A configuration edited while a frontend was down lands there, and so does a change in what the revision is computed over: an upgrade that alters the hashed form leaves every stored revision describing a configuration that no longer exists. The only recovery was deleting the row by hand, which is not something a cluster should need. Each frontend now reconciles the stored revisions against the loaded configurations at startup and republishes the ones that disagree. Only those: republishing quarantines every replica loaded under the old revision, so doing it for a model that did not drift would unload a healthy replica for nothing. A model with no stored revision has never been served and is left for its first request to establish. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude Code:claude-opus-5 [golangci-lint] --- core/application/startup.go | 10 ++ core/services/modeladmin/revision_resync.go | 118 +++++++++++++++ .../modeladmin/revision_resync_test.go | 142 ++++++++++++++++++ docs/content/features/distributed-mode.md | 5 +- 4 files changed, 273 insertions(+), 2 deletions(-) create mode 100644 core/services/modeladmin/revision_resync.go create mode 100644 core/services/modeladmin/revision_resync_test.go diff --git a/core/application/startup.go b/core/application/startup.go index 66a8131620da..a5aec50908c7 100644 --- a/core/application/startup.go +++ b/core/application/startup.go @@ -373,6 +373,16 @@ func New(opts ...config.AppOption) (*Application, error) { cfgLoaderOpts := options.ToConfigLoaderOptions() modelRevisionLifecycle := modeladmin.NewDistributedModelRevisionLifecycle(distSvc.Registry, distSvc.ModelCleanup) gs.SetModelRevisionLifecycle(modelRevisionLifecycle) + // Bring the controller's stored revisions back in line with the + // configuration on disk. An inference request may only establish a + // revision, never replace one, so a model whose stored value had + // drifted stayed unroutable until someone deleted the row. + if err := modeladmin.ResyncModelConfigRevisions(options.Context, + application.ModelConfigLoader(), + modeladmin.NewRevisionStore(distSvc.Registry, modelRevisionLifecycle), + ); err != nil { + xlog.Warn("Failed to resync model config revisions", "error", err) + } gs.OnModelsChanged = func(evt messaging.CacheInvalidateEvent) { // ApplyRemoteChange honors the op: a "delete" prunes the element // (a reload-from-path is additive and cannot drop it), anything diff --git a/core/services/modeladmin/revision_resync.go b/core/services/modeladmin/revision_resync.go new file mode 100644 index 000000000000..4724f60e03b0 --- /dev/null +++ b/core/services/modeladmin/revision_resync.go @@ -0,0 +1,118 @@ +package modeladmin + +import ( + "context" + "errors" + "fmt" + + "github.com/mudler/xlog" + "gorm.io/gorm" + + "github.com/mudler/LocalAI/core/config" +) + +// ErrNoStoredRevision reports that the controller holds no revision for a +// model, which is the normal state for one that has never been served. +var ErrNoStoredRevision = gorm.ErrRecordNotFound + +// RevisionStore is the controller state this resync reads and corrects. +type RevisionStore interface { + GetModelConfigRevision(ctx context.Context, modelName string) (string, error) + ApplyConfigRevisions(ctx context.Context, transitions []ModelRevisionTransition) (int, error) +} + +// RevisionReader is the read half, satisfied by the node registry. +type RevisionReader interface { + GetModelConfigRevision(ctx context.Context, modelName string) (string, error) +} + +type revisionStore struct { + RevisionReader + lifecycle ModelRevisionLifecycle +} + +func (s revisionStore) ApplyConfigRevisions(ctx context.Context, t []ModelRevisionTransition) (int, error) { + return s.lifecycle.ApplyConfigRevisions(ctx, t) +} + +// NewRevisionStore pairs the registry that holds the stored revisions with the +// lifecycle that publishes new ones. Returns nil when either half is missing, +// which ResyncModelConfigRevisions treats as "nothing to reconcile". +func NewRevisionStore(reader RevisionReader, lifecycle ModelRevisionLifecycle) RevisionStore { + if reader == nil || lifecycle == nil { + return nil + } + return revisionStore{RevisionReader: reader, lifecycle: lifecycle} +} + +// ResyncModelConfigRevisions makes the controller's stored revision for each +// model agree with what this build computes from the configuration on disk. +// +// The stored revision is what every inference request is checked against, but +// nothing ever re-derived it from the persisted configuration: it moved only on +// an edit, a gallery install, or a peer's change broadcast. Any other way for +// the two to diverge left the model permanently unroutable, because an +// inference request may only establish a revision, never replace one. A +// configuration edited while this frontend was down, or a change in what the +// revision is computed over, both landed there, and the only recovery was +// deleting the row by hand. +// +// Running this at startup makes that self-correcting. Only a model whose stored +// revision disagrees is republished, so replicas of models that did not drift +// keep serving: republishing is not free, it quarantines every replica loaded +// under the old revision. +// +// A model with no stored revision is left alone. It has never been served, and +// inventing controller state for it here would quarantine nothing and describe +// a model that may never be requested. +func ResyncModelConfigRevisions(ctx context.Context, loader *config.ModelConfigLoader, store RevisionStore) error { + if loader == nil || store == nil { + return nil + } + + var transitions []ModelRevisionTransition + for _, cfg := range loader.GetAllModelsConfigs() { + want, err := config.ModelConfigRevision(&cfg) + if err != nil { + return fmt.Errorf("compute config revision for %q: %w", cfg.Name, err) + } + + stored, err := store.GetModelConfigRevision(ctx, cfg.Name) + if errors.Is(err, ErrNoStoredRevision) { + continue + } + if err != nil { + return fmt.Errorf("read stored config revision for %q: %w", cfg.Name, err) + } + if stored == want { + continue + } + + xlog.Warn("Stored model config revision disagrees with the configuration on disk, republishing", + "model", cfg.Name, "stored", shortRevision(stored), "computed", shortRevision(want)) + transitions = append(transitions, ModelRevisionTransition{ + ModelName: cfg.Name, ConfigRevision: want, Disabled: cfg.IsDisabled(), + }) + } + + if len(transitions) == 0 { + return nil + } + if _, err := store.ApplyConfigRevisions(ctx, transitions); err != nil { + return fmt.Errorf("republish model config revisions: %w", err) + } + xlog.Info("Republished model config revisions to match the configuration on disk", "models", len(transitions)) + return nil +} + +// shortRevision trims a revision for log output; the leading bytes identify it +// well enough to tell two apart. +func shortRevision(revision string) string { + if revision == "" { + return "(none)" + } + if len(revision) > 12 { + return revision[:12] + } + return revision +} diff --git a/core/services/modeladmin/revision_resync_test.go b/core/services/modeladmin/revision_resync_test.go new file mode 100644 index 000000000000..3979ccf75298 --- /dev/null +++ b/core/services/modeladmin/revision_resync_test.go @@ -0,0 +1,142 @@ +package modeladmin + +import ( + "context" + "errors" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/pkg/system" +) + +// stubRevisionStore stands in for the controller's stored revisions. +type stubRevisionStore struct { + stored map[string]string + getErr error + applied []ModelRevisionTransition + applyEr error +} + +func (s *stubRevisionStore) GetModelConfigRevision(_ context.Context, name string) (string, error) { + if s.getErr != nil { + return "", s.getErr + } + rev, ok := s.stored[name] + if !ok { + return "", ErrNoStoredRevision + } + return rev, nil +} + +func (s *stubRevisionStore) ApplyConfigRevisions(_ context.Context, t []ModelRevisionTransition) (int, error) { + s.applied = append(s.applied, t...) + return 0, s.applyEr +} + +// The controller pins a model's replicas to a stored revision and rejects any +// request carrying a different one. Nothing ever re-derived that stored value +// from the configuration on disk: it only moved on an edit, a gallery install +// or a peer's change event. So whenever the stored value stopped matching what +// this build computes for an unchanged file, every request for that model was +// rejected until an operator deleted the row by hand. +var _ = Describe("ResyncModelConfigRevisions", func() { + var ( + dir string + loader *config.ModelConfigLoader + store *stubRevisionStore + appConfig *config.ApplicationConfig + ) + + write := func(name, body string) { + Expect(os.WriteFile(filepath.Join(dir, name+".yaml"), []byte(body), 0o600)).To(Succeed()) + } + + revisionOf := func(name string) string { + cfg, ok := loader.GetModelConfig(name) + Expect(ok).To(BeTrue()) + rev, err := config.ModelConfigRevision(&cfg) + Expect(err).ToNot(HaveOccurred()) + return rev + } + + BeforeEach(func() { + dir = GinkgoT().TempDir() + appConfig = config.NewApplicationConfig() + appConfig.SystemState = &system.SystemState{Model: system.Model{ModelsPath: dir}} + loader = config.NewModelConfigLoader(dir) + store = &stubRevisionStore{stored: map[string]string{}} + }) + + load := func() { + Expect(loader.LoadModelConfigsFromPath(dir, appConfig.ToConfigLoaderOptions()...)).To(Succeed()) + } + + It("republishes the revision when the stored one no longer matches the config on disk", func() { + write("drifted", "name: drifted\nbackend: llama-cpp\ncontext_size: 4096\n") + load() + store.stored["drifted"] = "a-revision-from-an-earlier-build" + + Expect(ResyncModelConfigRevisions(context.Background(), loader, store)).To(Succeed()) + + Expect(store.applied).To(HaveLen(1)) + Expect(store.applied[0].ModelName).To(Equal("drifted")) + Expect(store.applied[0].ConfigRevision).To(Equal(revisionOf("drifted"))) + }) + + It("leaves a model alone when the stored revision already matches", func() { + write("agreed", "name: agreed\nbackend: llama-cpp\n") + load() + store.stored["agreed"] = revisionOf("agreed") + + Expect(ResyncModelConfigRevisions(context.Background(), loader, store)).To(Succeed()) + + Expect(store.applied).To(BeEmpty(), "republishing an unchanged revision would quarantine live replicas for nothing") + }) + + // A model nobody has served has no stored revision. Creating one here would + // invent controller state for a model that may never be requested; the first + // request establishes it. + It("does not create state for a model that has never been served", func() { + write("never-served", "name: never-served\nbackend: llama-cpp\n") + load() + + Expect(ResyncModelConfigRevisions(context.Background(), loader, store)).To(Succeed()) + + Expect(store.applied).To(BeEmpty()) + }) + + It("republishes only the models that actually drifted", func() { + write("drifted", "name: drifted\nbackend: llama-cpp\n") + write("agreed", "name: agreed\nbackend: llama-cpp\ncontext_size: 2048\n") + load() + store.stored["drifted"] = "stale" + store.stored["agreed"] = revisionOf("agreed") + + Expect(ResyncModelConfigRevisions(context.Background(), loader, store)).To(Succeed()) + + Expect(store.applied).To(HaveLen(1)) + Expect(store.applied[0].ModelName).To(Equal("drifted")) + }) + + It("reports a store failure instead of continuing silently", func() { + write("drifted", "name: drifted\nbackend: llama-cpp\n") + load() + store.stored["drifted"] = "stale" + store.applyEr = errors.New("database is down") + + Expect(ResyncModelConfigRevisions(context.Background(), loader, store)).ToNot(Succeed()) + }) + + It("skips a model whose stored revision cannot be read rather than guessing", func() { + write("unreadable", "name: unreadable\nbackend: llama-cpp\n") + load() + store.getErr = errors.New("connection reset") + + Expect(ResyncModelConfigRevisions(context.Background(), loader, store)).ToNot(Succeed()) + Expect(store.applied).To(BeEmpty()) + }) +}) diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index d82616024928..70d63684cb8c 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -1041,8 +1041,9 @@ Notes: **Requests fail with `stale model config revision` although nobody edited the model:** - A model's stored revision must describe its persisted configuration. Releases before this fix also hashed the per-request prediction parameters, so the first request after a restart pinned the revision to its own `temperature`, `top_p`, `stop` and similar values. Every later request that sent different values was then rejected. - Upgrade the frontend replicas first. After the upgrade the revision is stamped when the configuration is loaded, so it no longer depends on the request body. -- The stored revision does not heal on its own, because the recorded value belongs to no persisted configuration. Clear it once per affected model so the next request establishes the correct revision: `DELETE FROM model_config_states WHERE model_name = '';` -- Saving any edit for the model through the API or the WebUI has the same effect, because an edit publishes the current revision. +- Each frontend now reconciles the stored revisions against the configuration on disk at startup, and republishes any that disagree, so a drifted revision heals on the next restart. Only models that actually drifted are republished, because republishing quarantines the replicas loaded under the old revision. +- A model that has never been served has no stored revision and is left alone; its first request establishes one. +- On a release without that reconciliation, clear the row once per affected model so the next request establishes the correct revision: `DELETE FROM model_config_states WHERE model_name = '';` Saving any edit through the API or the WebUI has the same effect. **Port conflicts on workers:** - Each model gets its own gRPC process on an incrementing port (50051, 50052, ...)