From a7440f032d5e741cff965f19b604321b2ae53550 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Sat, 1 Aug 2026 09:11:48 +0200 Subject: [PATCH 1/9] chore: :arrow_up: Update PrismML-Eng/llama.cpp to `4dd165625bb6c020285eec8b342af25cf60233dd` (#11259) :arrow_up: Update PrismML-Eng/llama.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/cpp/bonsai/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cpp/bonsai/Makefile b/backend/cpp/bonsai/Makefile index 9707372a9a11..673d7ab6d503 100644 --- a/backend/cpp/bonsai/Makefile +++ b/backend/cpp/bonsai/Makefile @@ -1,7 +1,7 @@ # Pinned to the HEAD of the `prism` branch on https://github.com/PrismML-Eng/llama.cpp. # Auto-bumped nightly by .github/workflows/bump_deps.yaml. -BONSAI_VERSION?=7529fdaaf99ffdc5ca71ace9c7409a56b27ad92f +BONSAI_VERSION?=4dd165625bb6c020285eec8b342af25cf60233dd LLAMA_REPO?=https://github.com/PrismML-Eng/llama.cpp CMAKE_ARGS?= From 2f3dd404b5c6fa0edcdeabf03c96f8ea95fb229d Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Sat, 1 Aug 2026 09:12:54 +0200 Subject: [PATCH 2/9] feat(import): route MLX TTS models to mlx-audio (#11267) Detect text-to-speech MLX repositories during model import and emit a TTS-ready mlx-audio configuration. Expose mlx-audio in the backend preference dropdown for repositories without complete metadata. Assisted-by: Codex:gpt-5 Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- core/gallery/importers/mlx.go | 24 +++++++++--- core/gallery/importers/mlx_test.go | 42 +++++++++++++++++++++ core/http/endpoints/localai/backend.go | 1 + core/http/endpoints/localai/backend_test.go | 1 + docs/content/getting-started/models.md | 6 +++ 5 files changed, 69 insertions(+), 5 deletions(-) diff --git a/core/gallery/importers/mlx.go b/core/gallery/importers/mlx.go index 2698fe72fe5f..1f23b48dc2ce 100644 --- a/core/gallery/importers/mlx.go +++ b/core/gallery/importers/mlx.go @@ -3,6 +3,7 @@ package importers import ( "encoding/json" "path/filepath" + "slices" "strings" "github.com/mudler/LocalAI/core/config" @@ -31,7 +32,7 @@ func (i *MLXImporter) Match(details Details) bool { } b, ok := preferencesMap["backend"].(string) - if ok && b == "mlx" || b == "mlx-vlm" { + if ok && slices.Contains([]string{"mlx", "mlx-vlm", "mlx-audio"}, b) { return true } @@ -71,19 +72,32 @@ func (i *MLXImporter) Import(details Details) (gallery.ModelConfig, error) { // (issue #10269). Send them to the mlx-vlm backend, which applies the // processor-aware chat template. backend := "mlx" - if details.HuggingFace != nil && details.HuggingFace.PipelineTag == "image-text-to-text" { - backend = "mlx-vlm" + usecases := []string{config.UsecaseChat} + useTokenizerTemplate := true + if details.HuggingFace != nil { + switch details.HuggingFace.PipelineTag { + case "image-text-to-text": + backend = "mlx-vlm" + case "text-to-speech": + backend = "mlx-audio" + usecases = []string{config.UsecaseTTS} + useTokenizerTemplate = false + } } // An explicit backend preference always wins. b, ok := preferencesMap["backend"].(string) if ok { backend = b + if backend == "mlx-audio" { + usecases = []string{config.UsecaseTTS} + useTokenizerTemplate = false + } } modelConfig := config.ModelConfig{ Name: name, Description: description, - KnownUsecaseStrings: []string{config.UsecaseChat}, + KnownUsecaseStrings: usecases, Backend: backend, PredictionOptions: schema.PredictionOptions{ BasicModelRequest: schema.BasicModelRequest{ @@ -91,7 +105,7 @@ func (i *MLXImporter) Import(details Details) (gallery.ModelConfig, error) { }, }, TemplateConfig: config.TemplateConfig{ - UseTokenizerTemplate: true, + UseTokenizerTemplate: useTokenizerTemplate, }, } diff --git a/core/gallery/importers/mlx_test.go b/core/gallery/importers/mlx_test.go index 2eeaef3fbc9e..9c346455766c 100644 --- a/core/gallery/importers/mlx_test.go +++ b/core/gallery/importers/mlx_test.go @@ -48,6 +48,16 @@ var _ = Describe("MLXImporter", func() { Expect(result).To(BeTrue()) }) + It("should match when backend preference is mlx-audio", func() { + preferences := json.RawMessage(`{"backend": "mlx-audio"}`) + details := importers.Details{ + URI: "https://example.com/model", + Preferences: preferences, + } + + Expect(importer.Match(details)).To(BeTrue()) + }) + It("should not match when URI does not contain mlx-community/ and no backend preference", func() { details := importers.Details{ URI: "https://huggingface.co/other-org/test-model", @@ -123,6 +133,21 @@ var _ = Describe("MLXImporter", func() { Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: mlx-vlm")) }) + It("should configure explicit mlx-audio imports for text-to-speech", func() { + preferences := json.RawMessage(`{"backend": "mlx-audio"}`) + details := importers.Details{ + URI: "https://huggingface.co/mlx-community/Kokoro-82M-4bit", + Preferences: preferences, + } + + modelConfig, err := importer.Import(details) + + Expect(err).ToNot(HaveOccurred()) + Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: mlx-audio")) + Expect(modelConfig.ConfigFile).To(ContainSubstring("- tts")) + Expect(modelConfig.ConfigFile).ToNot(ContainSubstring("use_tokenizer_template: true")) + }) + It("should auto-route vision-language models to the mlx-vlm backend", func() { // gemma-4 E4B and similar VLMs declare pipeline_tag // "image-text-to-text" on HuggingFace. The text-only mlx-lm @@ -143,6 +168,23 @@ var _ = Describe("MLXImporter", func() { Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: mlx-vlm")) }) + It("should auto-route text-to-speech models to the mlx-audio backend", func() { + details := importers.Details{ + URI: "https://huggingface.co/mlx-community/Kokoro-82M-4bit", + HuggingFace: &hfapi.ModelDetails{ + ModelID: "mlx-community/Kokoro-82M-4bit", + PipelineTag: "text-to-speech", + }, + } + + modelConfig, err := importer.Import(details) + + Expect(err).ToNot(HaveOccurred()) + Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: mlx-audio")) + Expect(modelConfig.ConfigFile).To(ContainSubstring("- tts")) + Expect(modelConfig.ConfigFile).ToNot(ContainSubstring("use_tokenizer_template: true")) + }) + It("should keep text-only models on the plain mlx backend", func() { details := importers.Details{ URI: "https://huggingface.co/mlx-community/Llama-3.2-1B-Instruct-4bit", diff --git a/core/http/endpoints/localai/backend.go b/core/http/endpoints/localai/backend.go index 3d4a25b3313a..221e2829a199 100644 --- a/core/http/endpoints/localai/backend.go +++ b/core/http/endpoints/localai/backend.go @@ -38,6 +38,7 @@ var knownPrefOnlyBackends = []schema.KnownBackend{ {Name: "whisperx", Modality: "asr", AutoDetect: false, Description: "WhisperX transcription (preference-only)"}, {Name: "crispasr", Modality: "asr", AutoDetect: false, Description: "CrispASR multi-architecture transcription (preference-only)"}, // TTS + {Name: "mlx-audio", Modality: "tts", AutoDetect: false, Description: "MLX-Audio text-to-speech models (auto-detected; pref-only fallback)"}, {Name: "kokoros", Modality: "tts", AutoDetect: false, Description: "Kokoros TTS (preference-only)"}, {Name: "qwen-tts", Modality: "tts", AutoDetect: false, Description: "Qwen TTS (preference-only)"}, {Name: "qwen3-tts-cpp", Modality: "tts", AutoDetect: false, Description: "Qwen3 TTS C++ (preference-only)"}, diff --git a/core/http/endpoints/localai/backend_test.go b/core/http/endpoints/localai/backend_test.go index 3c43fb60c870..04a486354619 100644 --- a/core/http/endpoints/localai/backend_test.go +++ b/core/http/endpoints/localai/backend_test.go @@ -152,6 +152,7 @@ var _ = Describe("Backend Endpoints", func() { expectPrefOnly("tinygrad", "text") expectPrefOnly("trl", "text") expectPrefOnly("mlx-vlm", "text") + expectPrefOnly("mlx-audio", "tts") expectPrefOnly("whisperx", "asr") expectPrefOnly("crispasr", "asr") expectPrefOnly("kokoros", "tts") diff --git a/docs/content/getting-started/models.md b/docs/content/getting-started/models.md index d3345a346541..1bd612358e85 100644 --- a/docs/content/getting-started/models.md +++ b/docs/content/getting-started/models.md @@ -80,6 +80,12 @@ The WebUI provides a powerful model import interface that supports both simple a - Custom preferences 5. Click "Import Model" to start the import process +Repositories under `mlx-community` are imported with the native MLX backend. +LocalAI uses Hugging Face's pipeline metadata to select `mlx-vlm` for +vision-language models and `mlx-audio` for text-to-speech models; other MLX +repositories use `mlx`. An explicit backend selection in the import form always +overrides this automatic routing. + ### Advanced Import Mode For full control over model configuration: From 04764bbe894cf47a825d6db1b849c34cd9b0bcf2 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Sat, 1 Aug 2026 09:13:25 +0200 Subject: [PATCH 3/9] chore(model gallery): :robot: add 1 new models via gallery agent (#11268) chore(model gallery): :robot: add new models via gallery agent Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- gallery/index.yaml | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/gallery/index.yaml b/gallery/index.yaml index 617dc2466bb7..3c53c10cf143 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -1,4 +1,48 @@ --- +- name: "deepseek-v4-flash-0731" + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/unsloth/DeepSeek-V4-Flash-0731-GGUF + description: | + # DeepSeek-V4-Flash-0731 + + Technical Report👁️ + + ## Introduction + + **DeepSeek-V4-Flash-0731** is the official release of **DeepSeek-V4-Flash**, superseding the preview version, with substantially enhanced agentic capabilities. It has the same model structure as DeepSeek-V4-Flash-DSpark, i.e. it comes with a speculative decoding module attached. + + DeepSeek-V4-Flash-0731 outperforms DeepSeek-V4-Pro (Preview) on benchmarks listed below despite its far smaller activated parameter count, and is broadly competitive with the strongest proprietary models available. + + Notes: + + 1. For the Code Agent tasks among the public benchmarks above, DeepSeek-V4-Flash-0731 is evaluated with the minimal mode of DeepSeek Harness (to be released) as the agent framework, using the `max` reasoning effort level with `temperature = 1.0, top_p = 0.95`. + 2. † DSBench-FullStack is an internal full-stack development test set; DSBench-Hard is an internal test set of difficult coding-agent problems. + + ## Chat Template + + ... + license: "mit" + tags: + - llm + - gguf + - deepseek + icon: https://github.com/deepseek-ai/DeepSeek-V2/blob/main/figures/logo.svg + overrides: + backend: ds4 + function: + grammar: + disable: true + known_usecases: + - chat + parameters: + model: ds4flash.gguf + template: + use_tokenizer_template: true + files: + - filename: ds4flash.gguf + sha256: "" + uri: https://huggingface.co/unsloth/DeepSeek-V4-Flash-0731-GGUF - name: "parable-granite-4.1-3b-claude-fable-5" url: "github:mudler/LocalAI/gallery/virtual.yaml@master" urls: From fca7ab2df4ec438f0b2275e71191f482233149c5 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Sat, 1 Aug 2026 09:13:59 +0200 Subject: [PATCH 4/9] fix(gallery): correct Nanbeige 4.2 artifacts (#11269) Use the case-sensitive Hugging Face filenames and refresh the linked SHA256 values for both gallery variants. Assisted-by: Codex:gpt-5 Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- gallery/index.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/gallery/index.yaml b/gallery/index.yaml index 3c53c10cf143..5162c9604ac2 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -6054,14 +6054,14 @@ - instruction-tuned - code - math - last_checked: "2026-07-28" + last_checked: "2026-08-01" overrides: parameters: - model: nanbeige4.2-3b-Q8_0.gguf + model: Nanbeige4.2-3B-Q8_0.gguf files: - - filename: nanbeige4.2-3b-Q8_0.gguf - sha256: 44707bb25e7ba3f2b0b5f3c2311da95ee3676986dd2014ce2aaeb14113590e33 - uri: huggingface://owao/Nanbeige4.2-3B-GGUF/nanbeige4.2-3b-Q8_0.gguf + - filename: Nanbeige4.2-3B-Q8_0.gguf + sha256: 4f8bd17cdf58bea2a94aef03457e0b8f019c26fe4daee7ae49b61bfa935a9126 + uri: huggingface://owao/Nanbeige4.2-3B-GGUF/Nanbeige4.2-3B-Q8_0.gguf - name: nanbeige4.2-3b url: github:mudler/LocalAI/gallery/nanbeige4.2.yaml@master urls: @@ -6084,16 +6084,16 @@ - instruction-tuned - code - math - last_checked: "2026-07-28" + last_checked: "2026-08-01" variants: - model: nanbeige4.2-3b-q8 overrides: parameters: - model: nanbeige4.2-3b-Q4_K_M.gguf + model: Nanbeige4.2-3B-Q4_K_M.gguf files: - - filename: nanbeige4.2-3b-Q4_K_M.gguf - sha256: 9ffd17d14472ff208409b3f51a6d87a5e5ec1b878b9a6f4dfe15c2a883366104 - uri: huggingface://owao/Nanbeige4.2-3B-GGUF/nanbeige4.2-3b-Q4_K_M.gguf + - filename: Nanbeige4.2-3B-Q4_K_M.gguf + sha256: ffe1b9b8ee95ec4b962c379905aa8be6f72ae9c4645c6c70e3b6ff7b197e6ef4 + uri: huggingface://owao/Nanbeige4.2-3B-GGUF/Nanbeige4.2-3B-Q4_K_M.gguf - name: nemo-parakeet-tdt-0.6b url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: From 76927ccde3cb40b687aa15b8102a110524fc7c73 Mon Sep 17 00:00:00 2001 From: Zelys Date: Sat, 1 Aug 2026 02:25:35 -0500 Subject: [PATCH 5/9] fix(utils): reject tar hardlinks that escape the extraction root (#11266) * fix(utils): reject tar hardlinks that escape the extraction root ExtractArchive pre-scans archive members and rejects symlinks, but tar hardlink entries carry a regular file mode and so pass that check. Header.Linkname was never validated, so an archive could create a link to a path outside the destination directory. Validate Linkname with the same path check already applied to member names. Hardlinks that resolve inside the extraction root still extract, so ordinary archives are unaffected. pkg/oci/image.go already resolves tar.TypeLink targets before using them; this brings the archive extraction path in line with it. Assisted-by: Claude:claude-opus-5 Signed-off-by: Zelys-DFKH * test(utils): cover hardlink overwrite and in-root hardlinks The existing hardlink test names a link target two levels above the extraction root, so its final assertion checked a path the link never resolved to and could not fail. Point the target one level up instead, at the path that assertion already names. Add two cases. The first uses a .tar.gz, where ExtractArchive binds a Tar config with OverwriteExisting set, and follows the link entry with a regular entry of the same name. Before the fix that pair linked to a file outside the root and then truncated it through the link, which the plain .tar case does not reach. The second extracts a hardlink whose target is an earlier member of the same archive, covering the claim that ordinary archives are unaffected. Assisted-by: Claude:claude-opus-5 Signed-off-by: Zelys-DFKH --------- Signed-off-by: Zelys-DFKH --- pkg/utils/untar.go | 17 ++++ pkg/utils/untar_test.go | 166 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 183 insertions(+) diff --git a/pkg/utils/untar.go b/pkg/utils/untar.go index cb65bb6b5ec0..c8521648e240 100644 --- a/pkg/utils/untar.go +++ b/pkg/utils/untar.go @@ -70,6 +70,11 @@ func ExtractArchive(archive, dst string) error { if f.FileInfo.Mode()&os.ModeSymlink != 0 { return fmt.Errorf("archive contains a symlink") } + if linkname, ok := archiveMemberLinkname(f); ok { + if err := validateArchiveMemberPath(extractRoot, linkname); err != nil { + return err + } + } return nil }) @@ -95,6 +100,18 @@ func archiveMemberName(f archiver.File) string { } } +// archiveMemberLinkname reports the target of a tar hardlink member, which carries a regular file mode and so is not caught by the symlink check. +func archiveMemberLinkname(f archiver.File) (string, bool) { + switch h := f.Header.(type) { + case tar.Header: + return h.Linkname, h.Typeflag == tar.TypeLink + case *tar.Header: + return h.Linkname, h.Typeflag == tar.TypeLink + default: + return "", false + } +} + func validateArchiveMemberPath(root, name string) error { if name == "" { return fmt.Errorf("archive contains an empty path") diff --git a/pkg/utils/untar_test.go b/pkg/utils/untar_test.go index e82b3611f790..7d40125803b7 100644 --- a/pkg/utils/untar_test.go +++ b/pkg/utils/untar_test.go @@ -3,6 +3,7 @@ package utils_test import ( "archive/tar" "archive/zip" + "compress/gzip" "os" "path/filepath" @@ -59,6 +60,53 @@ var _ = Describe("utils/archive tests", func() { Expect(err.Error()).To(ContainSubstring("unsafe path")) Expect(filepath.Join(tmpDir, "escaped.txt")).ToNot(BeAnExistingFile()) }) + + It("rejects tar hardlinks that overwrite a file outside the destination", func() { + tmpDir := GinkgoT().TempDir() + archivePath := filepath.Join(tmpDir, "model.tar.gz") + extractPath := filepath.Join(tmpDir, "models") + outsidePath := filepath.Join(tmpDir, "outside.txt") + + Expect(os.WriteFile(outsidePath, []byte("original"), 0o600)).To(Succeed()) + Expect(writeTarGzArchiveWithHardlinkedFile(archivePath, "payload.bin", "../outside.txt", "overwritten")).To(Succeed()) + + err := ExtractArchive(archivePath, extractPath) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("unsafe path")) + + contents, readErr := os.ReadFile(outsidePath) + Expect(readErr).ToNot(HaveOccurred()) + Expect(string(contents)).To(Equal("original")) + }) + + It("extracts tar hardlinks that stay inside the destination", func() { + tmpDir := GinkgoT().TempDir() + archivePath := filepath.Join(tmpDir, "model.tar.gz") + extractPath := filepath.Join(tmpDir, "models") + + Expect(writeTarGzArchiveWithInternalHardlink(archivePath, "model.bin", "alias.bin", "weights")).To(Succeed()) + + Expect(ExtractArchive(archivePath, extractPath)).To(Succeed()) + + extracted, err := os.ReadFile(filepath.Join(extractPath, "alias.bin")) + Expect(err).ToNot(HaveOccurred()) + Expect(string(extracted)).To(Equal("weights")) + }) + + It("rejects tar hardlinks that point outside the destination", func() { + tmpDir := GinkgoT().TempDir() + archivePath := filepath.Join(tmpDir, "model.tar") + extractPath := filepath.Join(tmpDir, "models") + + Expect(writeTarArchiveWithHardlink(archivePath, "payload.bin", "../escaped.txt")).To(Succeed()) + + err := ExtractArchive(archivePath, extractPath) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("unsafe path")) + Expect(filepath.Join(tmpDir, "escaped.txt")).ToNot(BeAnExistingFile()) + }) }) func writeZipArchive(path string, files map[string]string) (err error) { @@ -126,3 +174,121 @@ func writeTarArchive(path string, files map[string]string) (err error) { return nil } + +func writeTarArchiveWithHardlink(path, name, linkname string) (err error) { + out, err := os.Create(path) + if err != nil { + return err + } + defer func() { + if closeErr := out.Close(); err == nil { + err = closeErr + } + }() + + writer := tar.NewWriter(out) + defer func() { + if closeErr := writer.Close(); err == nil { + err = closeErr + } + }() + + return writer.WriteHeader(&tar.Header{ + Name: name, + Linkname: linkname, + Typeflag: tar.TypeLink, + Mode: 0o600, + }) +} + +func writeTarGzArchiveWithHardlinkedFile(path, name, linkname, contents string) (err error) { + out, err := os.Create(path) + if err != nil { + return err + } + defer func() { + if closeErr := out.Close(); err == nil { + err = closeErr + } + }() + + compressor := gzip.NewWriter(out) + defer func() { + if closeErr := compressor.Close(); err == nil { + err = closeErr + } + }() + + writer := tar.NewWriter(compressor) + defer func() { + if closeErr := writer.Close(); err == nil { + err = closeErr + } + }() + + if err := writer.WriteHeader(&tar.Header{ + Name: name, + Linkname: linkname, + Typeflag: tar.TypeLink, + Mode: 0o600, + }); err != nil { + return err + } + + data := []byte(contents) + if err := writer.WriteHeader(&tar.Header{ + Name: name, + Mode: 0o600, + Size: int64(len(data)), + }); err != nil { + return err + } + _, err = writer.Write(data) + + return err +} + +func writeTarGzArchiveWithInternalHardlink(path, targetName, linkName, contents string) (err error) { + out, err := os.Create(path) + if err != nil { + return err + } + defer func() { + if closeErr := out.Close(); err == nil { + err = closeErr + } + }() + + compressor := gzip.NewWriter(out) + defer func() { + if closeErr := compressor.Close(); err == nil { + err = closeErr + } + }() + + writer := tar.NewWriter(compressor) + defer func() { + if closeErr := writer.Close(); err == nil { + err = closeErr + } + }() + + data := []byte(contents) + if err := writer.WriteHeader(&tar.Header{ + Name: targetName, + Mode: 0o600, + Size: int64(len(data)), + }); err != nil { + return err + } + if _, err := writer.Write(data); err != nil { + return err + } + + return writer.WriteHeader(&tar.Header{ + Name: linkName, + Linkname: targetName, + Typeflag: tar.TypeLink, + Mode: 0o600, + }) +} From 7e4a60c70140c96828a874e8045f6dc6af770d0f Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Sat, 1 Aug 2026 09:25:51 +0200 Subject: [PATCH 6/9] chore: :arrow_up: Update TheTom/llama-cpp-turboquant to `8a891f4b566efdbd3cea92fafee3227a0a267683` (#11258) :arrow_up: Update TheTom/llama-cpp-turboquant Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/cpp/turboquant/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cpp/turboquant/Makefile b/backend/cpp/turboquant/Makefile index 5ec20b86af8c..162d09fd1268 100644 --- a/backend/cpp/turboquant/Makefile +++ b/backend/cpp/turboquant/Makefile @@ -1,7 +1,7 @@ # Pinned to the HEAD of feature/turboquant-kv-cache on https://github.com/TheTom/llama-cpp-turboquant. # Auto-bumped nightly by .github/workflows/bump_deps.yaml. -TURBOQUANT_VERSION?=c26cbdffcf6fc9b7430cd6b117757e9a3f70b7ea +TURBOQUANT_VERSION?=8a891f4b566efdbd3cea92fafee3227a0a267683 LLAMA_REPO?=https://github.com/TheTom/llama-cpp-turboquant CMAKE_ARGS?= From cedcbf97a9df97a52aa0422902c1404b84fedb03 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Sat, 1 Aug 2026 09:26:23 +0200 Subject: [PATCH 7/9] fix(llama-cpp): retain CPU variants in GPU builds (#11255) Build the runtime CPU variant set alongside x86 GPU backends so partial offload uses the host's SIMD kernels instead of the scalar fallback. Keep arm64 GPU images on the portable binary until their builders consistently provide gcc-14. Assisted-by: Codex:gpt-5 Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- .docker/llama-cpp-build-target.sh | 14 +++++++++++ .docker/llama-cpp-compile.sh | 18 ++++++-------- backend/cpp/llama-cpp/run.sh | 8 +++--- scripts/build/llama-cpp-build-target_test.sh | 26 ++++++++++++++++++++ 4 files changed, 51 insertions(+), 15 deletions(-) create mode 100755 .docker/llama-cpp-build-target.sh create mode 100755 scripts/build/llama-cpp-build-target_test.sh diff --git a/.docker/llama-cpp-build-target.sh b/.docker/llama-cpp-build-target.sh new file mode 100755 index 000000000000..177fdc7ad7cc --- /dev/null +++ b/.docker/llama-cpp-build-target.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +arch=${1:?target architecture is required} +build_type=${2-} + +# GPU arm64 base images do not consistently provide the gcc-14 toolchain needed +# to compile ggml's armv9.2 CPU variants. Keep their portable fallback until the +# builder images can supply that compiler. +if [ "$arch" = "arm64" ] && [ -n "$build_type" ]; then + echo llama-cpp-fallback +else + echo llama-cpp-cpu-all +fi diff --git a/.docker/llama-cpp-compile.sh b/.docker/llama-cpp-compile.sh index 112d43c160ed..32ff2a2392f0 100755 --- a/.docker/llama-cpp-compile.sh +++ b/.docker/llama-cpp-compile.sh @@ -18,10 +18,12 @@ if [[ -n "${CUDA_DOCKER_ARCH:-}" ]]; then fi cd /LocalAI/backend/cpp/llama-cpp -if [ -z "${BUILD_TYPE:-}" ]; then - # Pure CPU image (BUILD_TYPE empty): one build with ggml CPU_ALL_VARIANTS replaces the - # per-microarch binaries (x86: avx/avx2/avx512/fallback; arm64: armv8.x/armv9.x). ggml - # dlopens the best libggml-cpu-*.so at runtime by probing host CPU features. +BUILD_TARGET=$(/LocalAI/.docker/llama-cpp-build-target.sh "${TARGETARCH}" "${BUILD_TYPE:-}") +if [ "$BUILD_TARGET" = "llama-cpp-cpu-all" ]; then + # One build with ggml CPU_ALL_VARIANTS replaces the per-microarch binaries (x86: + # avx/avx2/avx512/fallback; arm64: armv8.x/armv9.x). BUILD_TYPE remains in the + # environment, so GPU builds retain their accelerator backend while ggml dlopens the + # best CPU library when work is offloaded to the host. # # arm64: the CPU_ALL_VARIANTS table includes armv9.2 SME variants whose -march=...+sme is # rejected by the Ubuntu 24.04 default gcc-13. gcc-14 accepts it, so build the arm64 @@ -35,14 +37,8 @@ if [ -z "${BUILD_TYPE:-}" ]; then apt-get update -qq && apt-get install -y -qq gcc-14 g++-14 export CC=gcc-14 CXX=g++-14 fi - make llama-cpp-cpu-all -else - # GPU build (cublas/hipblas/sycl/vulkan/...): the accelerator does the compute, so a - # single fallback CPU build is enough - no per-microarch CPU variants needed. (This also - # keeps the heavy GPU backend compile from also building the whole CPU variant matrix, - # and avoids the gcc-14 apt step on GPU base images such as nvidia l4t.) - make llama-cpp-fallback fi +make "$BUILD_TARGET" make llama-cpp-grpc make llama-cpp-rpc-server diff --git a/backend/cpp/llama-cpp/run.sh b/backend/cpp/llama-cpp/run.sh index 7610983194ea..3182801f18bd 100755 --- a/backend/cpp/llama-cpp/run.sh +++ b/backend/cpp/llama-cpp/run.sh @@ -12,10 +12,10 @@ grep -e "flags" /proc/cpuinfo | head -1 BINARY=llama-cpp-fallback -# CPU images (x86, arm64, darwin) ship a single llama-cpp-cpu-all built with ggml +# CPU images and x86 GPU images ship a single llama-cpp-cpu-all built with ggml # CPU_ALL_VARIANTS: ggml's backend registry dlopens the best libggml-cpu-*.so for this -# host, so no shell-side AVX probing. GPU images (cublas/sycl/vulkan/hipblas) ship only -# llama-cpp-fallback (the accelerator does the compute), so fall back to it when absent. +# host, so no shell-side AVX probing. GPU arm64 images still ship llama-cpp-fallback +# until their builder toolchains support ggml's complete arm variant matrix. if [ -e "$CURDIR"/llama-cpp-cpu-all ]; then BINARY=llama-cpp-cpu-all fi @@ -76,4 +76,4 @@ echo "Using binary: $BINARY" exec "$CURDIR"/$BINARY "$@" # We should never reach this point, however just in case we do, run fallback -exec "$CURDIR"/llama-cpp-fallback "$@" \ No newline at end of file +exec "$CURDIR"/llama-cpp-fallback "$@" diff --git a/scripts/build/llama-cpp-build-target_test.sh b/scripts/build/llama-cpp-build-target_test.sh new file mode 100755 index 000000000000..3d3970b5551e --- /dev/null +++ b/scripts/build/llama-cpp-build-target_test.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail + +CURDIR=$(dirname "$(realpath "$0")") +SELECTOR="$CURDIR/../../.docker/llama-cpp-build-target.sh" + +assert_target() { + local arch=$1 + local build_type=$2 + local expected=$3 + local actual + + actual=$("$SELECTOR" "$arch" "$build_type") + if [ "$actual" != "$expected" ]; then + echo "FAIL: $arch/$build_type selected $actual, expected $expected" + exit 1 + fi +} + +assert_target amd64 cublas llama-cpp-cpu-all +assert_target amd64 vulkan llama-cpp-cpu-all +assert_target amd64 "" llama-cpp-cpu-all +assert_target arm64 cublas llama-cpp-fallback +assert_target arm64 "" llama-cpp-cpu-all + +echo "PASS: llama.cpp build target preserves CPU variants where supported" From c0a9c427717da4366ba5d6f95cca4ceaa9463f88 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Sat, 1 Aug 2026 11:49:26 +0200 Subject: [PATCH 8/9] gallery: add Fara1.5 9B GGUF variants (#11271) Add the new 9B Fara computer-use model alongside its existing 27B sibling, with Q4_K_M and Q8_0 llama.cpp variants plus the required vision projector. Assisted-by: Codex:gpt-5 [web] Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- gallery/index.yaml | 76 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/gallery/index.yaml b/gallery/index.yaml index 5162c9604ac2..216163aeed87 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -5306,6 +5306,82 @@ - filename: llama-cpp/models/Qwen3.5-35B-A3B-APEX-GGUF/Qwen3.5-35B-A3B-APEX-Quality.gguf sha256: 50887b60c77ee5c95bc3657814ae993abcab7b2d71868b9af1e84d6badd09a57 uri: https://huggingface.co/mudler/Qwen3.5-35B-A3B-APEX-GGUF/resolve/main/Qwen3.5-35B-A3B-APEX-Quality.gguf +- &fara1-5-9b + name: fara1.5-9b + url: github:mudler/LocalAI/gallery/virtual.yaml@master + variants: + - model: fara1.5-9b-q8 + urls: + - https://huggingface.co/microsoft/Fara1.5-9B + - https://huggingface.co/bartowski/Fara1.5-9B-GGUF + description: | + Fara1.5-9B is Microsoft's 9B-parameter multimodal computer-use agent for web browsers, fine-tuned from Qwen3.5-9B. It accepts screenshots and text, emits structured browser actions, supports a 262K-token context, and should be deployed with appropriate sandboxing and user-confirmation controls. This entry uses the recommended Q4_K_M GGUF quantization. + license: mit + tags: + - fara + - qwen + - qwen3.5 + - 9b + - llm + - gguf + - quantized + - chat + - vision + - multimodal + - agent + - computer-use + - gpu + - cpu + last_checked: "2026-08-01" + overrides: + backend: llama-cpp + function: + grammar: + disable: true + known_usecases: + - chat + - vision + mmproj: llama-cpp/mmproj/Fara1.5-9B-GGUF/mmproj-Fara1.5-9B-f16.gguf + options: + - use_jinja:true + parameters: + model: llama-cpp/models/Fara1.5-9B-GGUF/Fara1.5-9B-Q4_K_M.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/Fara1.5-9B-GGUF/Fara1.5-9B-Q4_K_M.gguf + sha256: a02e7220337b87290bca7ef7225ba4afa7104efa05c0851de6c29ec9c5d04c7d + uri: huggingface://bartowski/Fara1.5-9B-GGUF/Fara1.5-9B-Q4_K_M.gguf + - filename: llama-cpp/mmproj/Fara1.5-9B-GGUF/mmproj-Fara1.5-9B-f16.gguf + sha256: 97b423c81719ffc367124a9739d6feb6f62d62f60869a6d385a701b963ce1906 + uri: huggingface://bartowski/Fara1.5-9B-GGUF/mmproj-Fara1.5-9B-f16.gguf +- !!merge <<: *fara1-5-9b + name: fara1.5-9b-q8 + variants: [] + description: | + Fara1.5-9B is Microsoft's 9B-parameter multimodal computer-use agent for web browsers, fine-tuned from Qwen3.5-9B. It accepts screenshots and text, emits structured browser actions, supports a 262K-token context, and should be deployed with appropriate sandboxing and user-confirmation controls. This entry uses the higher-quality Q8_0 GGUF quantization. + overrides: + backend: llama-cpp + function: + grammar: + disable: true + known_usecases: + - chat + - vision + mmproj: llama-cpp/mmproj/Fara1.5-9B-GGUF/mmproj-Fara1.5-9B-f16.gguf + options: + - use_jinja:true + parameters: + model: llama-cpp/models/Fara1.5-9B-GGUF/Fara1.5-9B-Q8_0.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/Fara1.5-9B-GGUF/Fara1.5-9B-Q8_0.gguf + sha256: a2e30cca7aec006266308153ae781347505af16baa514bbd4e0e3f4a79ea3a22 + uri: huggingface://bartowski/Fara1.5-9B-GGUF/Fara1.5-9B-Q8_0.gguf + - filename: llama-cpp/mmproj/Fara1.5-9B-GGUF/mmproj-Fara1.5-9B-f16.gguf + sha256: 97b423c81719ffc367124a9739d6feb6f62d62f60869a6d385a701b963ce1906 + uri: huggingface://bartowski/Fara1.5-9B-GGUF/mmproj-Fara1.5-9B-f16.gguf - name: fara1.5-27b url: github:mudler/LocalAI/gallery/virtual.yaml@master variants: From 3c02d2aa4d64b2e202596f0df7534d32fa1a31ad Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Sat, 1 Aug 2026 14:14:59 +0200 Subject: [PATCH 9/9] gallery: add Inkling Small GGUF variants (#11273) Add Q4_K_M and IQ2_M sharded llama.cpp entries with the BF16 multimodal projector. Assisted-by: Codex:gpt-5 Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- gallery/index.yaml | 84 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/gallery/index.yaml b/gallery/index.yaml index 216163aeed87..05ea2ddec38b 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -941,6 +941,90 @@ - filename: llama-cpp/mmproj/inkling-UD-Q4_K_XL/mmproj-BF16.gguf sha256: 662c925e1df293cfba16ffd6bd53dac31d3c73160ba65dff7270d7a70f351e91 uri: https://huggingface.co/unsloth/inkling-GGUF/resolve/main/mmproj-BF16.gguf +- &inkling-small + name: "inkling-small" + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/thinkingmachines/Inkling-Small + - https://huggingface.co/unsloth/Inkling-Small-GGUF + description: | + Inkling Small is a 276B-parameter mixture-of-experts multimodal model with 12B active parameters for text, image, and audio understanding, instruction following, coding, and tool use. This entry uses the Q4_K_M GGUF quantization, whose five language-model shards total approximately 162.5 GB. + license: "apache-2.0" + tags: + - llm + - gguf + - vision + - audio + - multimodal + overrides: + backend: llama-cpp + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + mmproj: llama-cpp/mmproj/Inkling-Small-UD-Q4_K_M/mmproj-BF16.gguf + options: + - use_jinja:true + parameters: + model: llama-cpp/models/Inkling-Small-UD-Q4_K_M/Inkling-Small-UD-Q4_K_M-00001-of-00005.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/Inkling-Small-UD-Q4_K_M/Inkling-Small-UD-Q4_K_M-00001-of-00005.gguf + sha256: a51ac3f439198f2817219edd582be4b600c273be24e78cbd58ebff982d9f007e + uri: https://huggingface.co/unsloth/Inkling-Small-GGUF/resolve/main/UD-Q4_K_M/Inkling-Small-UD-Q4_K_M-00001-of-00005.gguf + - filename: llama-cpp/models/Inkling-Small-UD-Q4_K_M/Inkling-Small-UD-Q4_K_M-00002-of-00005.gguf + sha256: 3dccdd473cc3a191e6028f6105b01831ed3a8dc30ec4e02679f9e0c2ccb59671 + uri: https://huggingface.co/unsloth/Inkling-Small-GGUF/resolve/main/UD-Q4_K_M/Inkling-Small-UD-Q4_K_M-00002-of-00005.gguf + - filename: llama-cpp/models/Inkling-Small-UD-Q4_K_M/Inkling-Small-UD-Q4_K_M-00003-of-00005.gguf + sha256: 1a7edf29bda1d278b4668e1a082d7db634b845a53ea58c918a8cea1f9006c21c + uri: https://huggingface.co/unsloth/Inkling-Small-GGUF/resolve/main/UD-Q4_K_M/Inkling-Small-UD-Q4_K_M-00003-of-00005.gguf + - filename: llama-cpp/models/Inkling-Small-UD-Q4_K_M/Inkling-Small-UD-Q4_K_M-00004-of-00005.gguf + sha256: 376f67568438da96b10566730e8a9e17e3f665ab1b9d82eba48ec33708b172f7 + uri: https://huggingface.co/unsloth/Inkling-Small-GGUF/resolve/main/UD-Q4_K_M/Inkling-Small-UD-Q4_K_M-00004-of-00005.gguf + - filename: llama-cpp/models/Inkling-Small-UD-Q4_K_M/Inkling-Small-UD-Q4_K_M-00005-of-00005.gguf + sha256: e34364af0d04d2d295bc374f1a4fa80002e4277db1bcf4b56240dbd443ac21d3 + uri: https://huggingface.co/unsloth/Inkling-Small-GGUF/resolve/main/UD-Q4_K_M/Inkling-Small-UD-Q4_K_M-00005-of-00005.gguf + - filename: llama-cpp/mmproj/Inkling-Small-UD-Q4_K_M/mmproj-BF16.gguf + sha256: 05d4475a956030be87b099865d6552a541a476db8cc3e266fcfa7c5a24846248 + uri: https://huggingface.co/unsloth/Inkling-Small-GGUF/resolve/main/mmproj-BF16.gguf + variants: + - model: inkling-small-iq2-m +- !!merge <<: *inkling-small + name: "inkling-small-iq2-m" + description: | + Inkling Small is a 276B-parameter mixture-of-experts multimodal model with 12B active parameters for text, image, and audio understanding, instruction following, coding, and tool use. This entry uses the IQ2_M GGUF quantization, whose three language-model shards total approximately 82.4 GB. + overrides: + backend: llama-cpp + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + mmproj: llama-cpp/mmproj/Inkling-Small-UD-IQ2_M/mmproj-BF16.gguf + options: + - use_jinja:true + parameters: + model: llama-cpp/models/Inkling-Small-UD-IQ2_M/Inkling-Small-UD-IQ2_M-00001-of-00003.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/Inkling-Small-UD-IQ2_M/Inkling-Small-UD-IQ2_M-00001-of-00003.gguf + sha256: 3b6ace30e488ad26e816cdba4e42714f40110a3142a210bd5c2e48f69e27cb31 + uri: https://huggingface.co/unsloth/Inkling-Small-GGUF/resolve/main/UD-IQ2_M/Inkling-Small-UD-IQ2_M-00001-of-00003.gguf + - filename: llama-cpp/models/Inkling-Small-UD-IQ2_M/Inkling-Small-UD-IQ2_M-00002-of-00003.gguf + sha256: 5ca94e858ae116eb513a2af1facd35844d42ef4a209e1d85cc4ecc73cd21b894 + uri: https://huggingface.co/unsloth/Inkling-Small-GGUF/resolve/main/UD-IQ2_M/Inkling-Small-UD-IQ2_M-00002-of-00003.gguf + - filename: llama-cpp/models/Inkling-Small-UD-IQ2_M/Inkling-Small-UD-IQ2_M-00003-of-00003.gguf + sha256: 8a84e00d4625d52491969f88f24f0999bc77527b7fa1d15cbe44ef88235bb377 + uri: https://huggingface.co/unsloth/Inkling-Small-GGUF/resolve/main/UD-IQ2_M/Inkling-Small-UD-IQ2_M-00003-of-00003.gguf + - filename: llama-cpp/mmproj/Inkling-Small-UD-IQ2_M/mmproj-BF16.gguf + sha256: 05d4475a956030be87b099865d6552a541a476db8cc3e266fcfa7c5a24846248 + uri: https://huggingface.co/unsloth/Inkling-Small-GGUF/resolve/main/mmproj-BF16.gguf + variants: [] - name: "qwythos-9b-v2" url: "github:mudler/LocalAI/gallery/virtual.yaml@master" urls: