diff --git a/.docker/llama-cpp-build-target.sh b/.docker/llama-cpp-build-target.sh new file mode 100755 index 000000000000..177fdc7ad7cc --- /dev/null +++ b/.docker/llama-cpp-build-target.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +arch=${1:?target architecture is required} +build_type=${2-} + +# GPU arm64 base images do not consistently provide the gcc-14 toolchain needed +# to compile ggml's armv9.2 CPU variants. Keep their portable fallback until the +# builder images can supply that compiler. +if [ "$arch" = "arm64" ] && [ -n "$build_type" ]; then + echo llama-cpp-fallback +else + echo llama-cpp-cpu-all +fi diff --git a/.docker/llama-cpp-compile.sh b/.docker/llama-cpp-compile.sh index 112d43c160ed..32ff2a2392f0 100755 --- a/.docker/llama-cpp-compile.sh +++ b/.docker/llama-cpp-compile.sh @@ -18,10 +18,12 @@ if [[ -n "${CUDA_DOCKER_ARCH:-}" ]]; then fi cd /LocalAI/backend/cpp/llama-cpp -if [ -z "${BUILD_TYPE:-}" ]; then - # Pure CPU image (BUILD_TYPE empty): one build with ggml CPU_ALL_VARIANTS replaces the - # per-microarch binaries (x86: avx/avx2/avx512/fallback; arm64: armv8.x/armv9.x). ggml - # dlopens the best libggml-cpu-*.so at runtime by probing host CPU features. +BUILD_TARGET=$(/LocalAI/.docker/llama-cpp-build-target.sh "${TARGETARCH}" "${BUILD_TYPE:-}") +if [ "$BUILD_TARGET" = "llama-cpp-cpu-all" ]; then + # One build with ggml CPU_ALL_VARIANTS replaces the per-microarch binaries (x86: + # avx/avx2/avx512/fallback; arm64: armv8.x/armv9.x). BUILD_TYPE remains in the + # environment, so GPU builds retain their accelerator backend while ggml dlopens the + # best CPU library when work is offloaded to the host. # # arm64: the CPU_ALL_VARIANTS table includes armv9.2 SME variants whose -march=...+sme is # rejected by the Ubuntu 24.04 default gcc-13. gcc-14 accepts it, so build the arm64 @@ -35,14 +37,8 @@ if [ -z "${BUILD_TYPE:-}" ]; then apt-get update -qq && apt-get install -y -qq gcc-14 g++-14 export CC=gcc-14 CXX=g++-14 fi - make llama-cpp-cpu-all -else - # GPU build (cublas/hipblas/sycl/vulkan/...): the accelerator does the compute, so a - # single fallback CPU build is enough - no per-microarch CPU variants needed. (This also - # keeps the heavy GPU backend compile from also building the whole CPU variant matrix, - # and avoids the gcc-14 apt step on GPU base images such as nvidia l4t.) - make llama-cpp-fallback fi +make "$BUILD_TARGET" make llama-cpp-grpc make llama-cpp-rpc-server diff --git a/backend/cpp/bonsai/Makefile b/backend/cpp/bonsai/Makefile index 9707372a9a11..673d7ab6d503 100644 --- a/backend/cpp/bonsai/Makefile +++ b/backend/cpp/bonsai/Makefile @@ -1,7 +1,7 @@ # Pinned to the HEAD of the `prism` branch on https://github.com/PrismML-Eng/llama.cpp. # Auto-bumped nightly by .github/workflows/bump_deps.yaml. -BONSAI_VERSION?=7529fdaaf99ffdc5ca71ace9c7409a56b27ad92f +BONSAI_VERSION?=4dd165625bb6c020285eec8b342af25cf60233dd LLAMA_REPO?=https://github.com/PrismML-Eng/llama.cpp CMAKE_ARGS?= diff --git a/backend/cpp/llama-cpp/run.sh b/backend/cpp/llama-cpp/run.sh index 7610983194ea..3182801f18bd 100755 --- a/backend/cpp/llama-cpp/run.sh +++ b/backend/cpp/llama-cpp/run.sh @@ -12,10 +12,10 @@ grep -e "flags" /proc/cpuinfo | head -1 BINARY=llama-cpp-fallback -# CPU images (x86, arm64, darwin) ship a single llama-cpp-cpu-all built with ggml +# CPU images and x86 GPU images ship a single llama-cpp-cpu-all built with ggml # CPU_ALL_VARIANTS: ggml's backend registry dlopens the best libggml-cpu-*.so for this -# host, so no shell-side AVX probing. GPU images (cublas/sycl/vulkan/hipblas) ship only -# llama-cpp-fallback (the accelerator does the compute), so fall back to it when absent. +# host, so no shell-side AVX probing. GPU arm64 images still ship llama-cpp-fallback +# until their builder toolchains support ggml's complete arm variant matrix. if [ -e "$CURDIR"/llama-cpp-cpu-all ]; then BINARY=llama-cpp-cpu-all fi @@ -76,4 +76,4 @@ echo "Using binary: $BINARY" exec "$CURDIR"/$BINARY "$@" # We should never reach this point, however just in case we do, run fallback -exec "$CURDIR"/llama-cpp-fallback "$@" \ No newline at end of file +exec "$CURDIR"/llama-cpp-fallback "$@" diff --git a/backend/cpp/turboquant/Makefile b/backend/cpp/turboquant/Makefile index 5ec20b86af8c..162d09fd1268 100644 --- a/backend/cpp/turboquant/Makefile +++ b/backend/cpp/turboquant/Makefile @@ -1,7 +1,7 @@ # Pinned to the HEAD of feature/turboquant-kv-cache on https://github.com/TheTom/llama-cpp-turboquant. # Auto-bumped nightly by .github/workflows/bump_deps.yaml. -TURBOQUANT_VERSION?=c26cbdffcf6fc9b7430cd6b117757e9a3f70b7ea +TURBOQUANT_VERSION?=8a891f4b566efdbd3cea92fafee3227a0a267683 LLAMA_REPO?=https://github.com/TheTom/llama-cpp-turboquant CMAKE_ARGS?= diff --git a/core/gallery/importers/mlx.go b/core/gallery/importers/mlx.go index 2698fe72fe5f..1f23b48dc2ce 100644 --- a/core/gallery/importers/mlx.go +++ b/core/gallery/importers/mlx.go @@ -3,6 +3,7 @@ package importers import ( "encoding/json" "path/filepath" + "slices" "strings" "github.com/mudler/LocalAI/core/config" @@ -31,7 +32,7 @@ func (i *MLXImporter) Match(details Details) bool { } b, ok := preferencesMap["backend"].(string) - if ok && b == "mlx" || b == "mlx-vlm" { + if ok && slices.Contains([]string{"mlx", "mlx-vlm", "mlx-audio"}, b) { return true } @@ -71,19 +72,32 @@ func (i *MLXImporter) Import(details Details) (gallery.ModelConfig, error) { // (issue #10269). Send them to the mlx-vlm backend, which applies the // processor-aware chat template. backend := "mlx" - if details.HuggingFace != nil && details.HuggingFace.PipelineTag == "image-text-to-text" { - backend = "mlx-vlm" + usecases := []string{config.UsecaseChat} + useTokenizerTemplate := true + if details.HuggingFace != nil { + switch details.HuggingFace.PipelineTag { + case "image-text-to-text": + backend = "mlx-vlm" + case "text-to-speech": + backend = "mlx-audio" + usecases = []string{config.UsecaseTTS} + useTokenizerTemplate = false + } } // An explicit backend preference always wins. b, ok := preferencesMap["backend"].(string) if ok { backend = b + if backend == "mlx-audio" { + usecases = []string{config.UsecaseTTS} + useTokenizerTemplate = false + } } modelConfig := config.ModelConfig{ Name: name, Description: description, - KnownUsecaseStrings: []string{config.UsecaseChat}, + KnownUsecaseStrings: usecases, Backend: backend, PredictionOptions: schema.PredictionOptions{ BasicModelRequest: schema.BasicModelRequest{ @@ -91,7 +105,7 @@ func (i *MLXImporter) Import(details Details) (gallery.ModelConfig, error) { }, }, TemplateConfig: config.TemplateConfig{ - UseTokenizerTemplate: true, + UseTokenizerTemplate: useTokenizerTemplate, }, } diff --git a/core/gallery/importers/mlx_test.go b/core/gallery/importers/mlx_test.go index 2eeaef3fbc9e..9c346455766c 100644 --- a/core/gallery/importers/mlx_test.go +++ b/core/gallery/importers/mlx_test.go @@ -48,6 +48,16 @@ var _ = Describe("MLXImporter", func() { Expect(result).To(BeTrue()) }) + It("should match when backend preference is mlx-audio", func() { + preferences := json.RawMessage(`{"backend": "mlx-audio"}`) + details := importers.Details{ + URI: "https://example.com/model", + Preferences: preferences, + } + + Expect(importer.Match(details)).To(BeTrue()) + }) + It("should not match when URI does not contain mlx-community/ and no backend preference", func() { details := importers.Details{ URI: "https://huggingface.co/other-org/test-model", @@ -123,6 +133,21 @@ var _ = Describe("MLXImporter", func() { Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: mlx-vlm")) }) + It("should configure explicit mlx-audio imports for text-to-speech", func() { + preferences := json.RawMessage(`{"backend": "mlx-audio"}`) + details := importers.Details{ + URI: "https://huggingface.co/mlx-community/Kokoro-82M-4bit", + Preferences: preferences, + } + + modelConfig, err := importer.Import(details) + + Expect(err).ToNot(HaveOccurred()) + Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: mlx-audio")) + Expect(modelConfig.ConfigFile).To(ContainSubstring("- tts")) + Expect(modelConfig.ConfigFile).ToNot(ContainSubstring("use_tokenizer_template: true")) + }) + It("should auto-route vision-language models to the mlx-vlm backend", func() { // gemma-4 E4B and similar VLMs declare pipeline_tag // "image-text-to-text" on HuggingFace. The text-only mlx-lm @@ -143,6 +168,23 @@ var _ = Describe("MLXImporter", func() { Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: mlx-vlm")) }) + It("should auto-route text-to-speech models to the mlx-audio backend", func() { + details := importers.Details{ + URI: "https://huggingface.co/mlx-community/Kokoro-82M-4bit", + HuggingFace: &hfapi.ModelDetails{ + ModelID: "mlx-community/Kokoro-82M-4bit", + PipelineTag: "text-to-speech", + }, + } + + modelConfig, err := importer.Import(details) + + Expect(err).ToNot(HaveOccurred()) + Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: mlx-audio")) + Expect(modelConfig.ConfigFile).To(ContainSubstring("- tts")) + Expect(modelConfig.ConfigFile).ToNot(ContainSubstring("use_tokenizer_template: true")) + }) + It("should keep text-only models on the plain mlx backend", func() { details := importers.Details{ URI: "https://huggingface.co/mlx-community/Llama-3.2-1B-Instruct-4bit", diff --git a/core/http/endpoints/localai/backend.go b/core/http/endpoints/localai/backend.go index 3d4a25b3313a..221e2829a199 100644 --- a/core/http/endpoints/localai/backend.go +++ b/core/http/endpoints/localai/backend.go @@ -38,6 +38,7 @@ var knownPrefOnlyBackends = []schema.KnownBackend{ {Name: "whisperx", Modality: "asr", AutoDetect: false, Description: "WhisperX transcription (preference-only)"}, {Name: "crispasr", Modality: "asr", AutoDetect: false, Description: "CrispASR multi-architecture transcription (preference-only)"}, // TTS + {Name: "mlx-audio", Modality: "tts", AutoDetect: false, Description: "MLX-Audio text-to-speech models (auto-detected; pref-only fallback)"}, {Name: "kokoros", Modality: "tts", AutoDetect: false, Description: "Kokoros TTS (preference-only)"}, {Name: "qwen-tts", Modality: "tts", AutoDetect: false, Description: "Qwen TTS (preference-only)"}, {Name: "qwen3-tts-cpp", Modality: "tts", AutoDetect: false, Description: "Qwen3 TTS C++ (preference-only)"}, diff --git a/core/http/endpoints/localai/backend_test.go b/core/http/endpoints/localai/backend_test.go index 3c43fb60c870..04a486354619 100644 --- a/core/http/endpoints/localai/backend_test.go +++ b/core/http/endpoints/localai/backend_test.go @@ -152,6 +152,7 @@ var _ = Describe("Backend Endpoints", func() { expectPrefOnly("tinygrad", "text") expectPrefOnly("trl", "text") expectPrefOnly("mlx-vlm", "text") + expectPrefOnly("mlx-audio", "tts") expectPrefOnly("whisperx", "asr") expectPrefOnly("crispasr", "asr") expectPrefOnly("kokoros", "tts") diff --git a/docs/content/getting-started/models.md b/docs/content/getting-started/models.md index d3345a346541..1bd612358e85 100644 --- a/docs/content/getting-started/models.md +++ b/docs/content/getting-started/models.md @@ -80,6 +80,12 @@ The WebUI provides a powerful model import interface that supports both simple a - Custom preferences 5. Click "Import Model" to start the import process +Repositories under `mlx-community` are imported with the native MLX backend. +LocalAI uses Hugging Face's pipeline metadata to select `mlx-vlm` for +vision-language models and `mlx-audio` for text-to-speech models; other MLX +repositories use `mlx`. An explicit backend selection in the import form always +overrides this automatic routing. + ### Advanced Import Mode For full control over model configuration: diff --git a/gallery/index.yaml b/gallery/index.yaml index 617dc2466bb7..05ea2ddec38b 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -1,4 +1,48 @@ --- +- name: "deepseek-v4-flash-0731" + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/unsloth/DeepSeek-V4-Flash-0731-GGUF + description: | + # DeepSeek-V4-Flash-0731 + + Technical Report👁️ + + ## Introduction + + **DeepSeek-V4-Flash-0731** is the official release of **DeepSeek-V4-Flash**, superseding the preview version, with substantially enhanced agentic capabilities. It has the same model structure as DeepSeek-V4-Flash-DSpark, i.e. it comes with a speculative decoding module attached. + + DeepSeek-V4-Flash-0731 outperforms DeepSeek-V4-Pro (Preview) on benchmarks listed below despite its far smaller activated parameter count, and is broadly competitive with the strongest proprietary models available. + + Notes: + + 1. For the Code Agent tasks among the public benchmarks above, DeepSeek-V4-Flash-0731 is evaluated with the minimal mode of DeepSeek Harness (to be released) as the agent framework, using the `max` reasoning effort level with `temperature = 1.0, top_p = 0.95`. + 2. † DSBench-FullStack is an internal full-stack development test set; DSBench-Hard is an internal test set of difficult coding-agent problems. + + ## Chat Template + + ... + license: "mit" + tags: + - llm + - gguf + - deepseek + icon: https://github.com/deepseek-ai/DeepSeek-V2/blob/main/figures/logo.svg + overrides: + backend: ds4 + function: + grammar: + disable: true + known_usecases: + - chat + parameters: + model: ds4flash.gguf + template: + use_tokenizer_template: true + files: + - filename: ds4flash.gguf + sha256: "" + uri: https://huggingface.co/unsloth/DeepSeek-V4-Flash-0731-GGUF - name: "parable-granite-4.1-3b-claude-fable-5" url: "github:mudler/LocalAI/gallery/virtual.yaml@master" urls: @@ -897,6 +941,90 @@ - filename: llama-cpp/mmproj/inkling-UD-Q4_K_XL/mmproj-BF16.gguf sha256: 662c925e1df293cfba16ffd6bd53dac31d3c73160ba65dff7270d7a70f351e91 uri: https://huggingface.co/unsloth/inkling-GGUF/resolve/main/mmproj-BF16.gguf +- &inkling-small + name: "inkling-small" + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/thinkingmachines/Inkling-Small + - https://huggingface.co/unsloth/Inkling-Small-GGUF + description: | + Inkling Small is a 276B-parameter mixture-of-experts multimodal model with 12B active parameters for text, image, and audio understanding, instruction following, coding, and tool use. This entry uses the Q4_K_M GGUF quantization, whose five language-model shards total approximately 162.5 GB. + license: "apache-2.0" + tags: + - llm + - gguf + - vision + - audio + - multimodal + overrides: + backend: llama-cpp + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + mmproj: llama-cpp/mmproj/Inkling-Small-UD-Q4_K_M/mmproj-BF16.gguf + options: + - use_jinja:true + parameters: + model: llama-cpp/models/Inkling-Small-UD-Q4_K_M/Inkling-Small-UD-Q4_K_M-00001-of-00005.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/Inkling-Small-UD-Q4_K_M/Inkling-Small-UD-Q4_K_M-00001-of-00005.gguf + sha256: a51ac3f439198f2817219edd582be4b600c273be24e78cbd58ebff982d9f007e + uri: https://huggingface.co/unsloth/Inkling-Small-GGUF/resolve/main/UD-Q4_K_M/Inkling-Small-UD-Q4_K_M-00001-of-00005.gguf + - filename: llama-cpp/models/Inkling-Small-UD-Q4_K_M/Inkling-Small-UD-Q4_K_M-00002-of-00005.gguf + sha256: 3dccdd473cc3a191e6028f6105b01831ed3a8dc30ec4e02679f9e0c2ccb59671 + uri: https://huggingface.co/unsloth/Inkling-Small-GGUF/resolve/main/UD-Q4_K_M/Inkling-Small-UD-Q4_K_M-00002-of-00005.gguf + - filename: llama-cpp/models/Inkling-Small-UD-Q4_K_M/Inkling-Small-UD-Q4_K_M-00003-of-00005.gguf + sha256: 1a7edf29bda1d278b4668e1a082d7db634b845a53ea58c918a8cea1f9006c21c + uri: https://huggingface.co/unsloth/Inkling-Small-GGUF/resolve/main/UD-Q4_K_M/Inkling-Small-UD-Q4_K_M-00003-of-00005.gguf + - filename: llama-cpp/models/Inkling-Small-UD-Q4_K_M/Inkling-Small-UD-Q4_K_M-00004-of-00005.gguf + sha256: 376f67568438da96b10566730e8a9e17e3f665ab1b9d82eba48ec33708b172f7 + uri: https://huggingface.co/unsloth/Inkling-Small-GGUF/resolve/main/UD-Q4_K_M/Inkling-Small-UD-Q4_K_M-00004-of-00005.gguf + - filename: llama-cpp/models/Inkling-Small-UD-Q4_K_M/Inkling-Small-UD-Q4_K_M-00005-of-00005.gguf + sha256: e34364af0d04d2d295bc374f1a4fa80002e4277db1bcf4b56240dbd443ac21d3 + uri: https://huggingface.co/unsloth/Inkling-Small-GGUF/resolve/main/UD-Q4_K_M/Inkling-Small-UD-Q4_K_M-00005-of-00005.gguf + - filename: llama-cpp/mmproj/Inkling-Small-UD-Q4_K_M/mmproj-BF16.gguf + sha256: 05d4475a956030be87b099865d6552a541a476db8cc3e266fcfa7c5a24846248 + uri: https://huggingface.co/unsloth/Inkling-Small-GGUF/resolve/main/mmproj-BF16.gguf + variants: + - model: inkling-small-iq2-m +- !!merge <<: *inkling-small + name: "inkling-small-iq2-m" + description: | + Inkling Small is a 276B-parameter mixture-of-experts multimodal model with 12B active parameters for text, image, and audio understanding, instruction following, coding, and tool use. This entry uses the IQ2_M GGUF quantization, whose three language-model shards total approximately 82.4 GB. + overrides: + backend: llama-cpp + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + mmproj: llama-cpp/mmproj/Inkling-Small-UD-IQ2_M/mmproj-BF16.gguf + options: + - use_jinja:true + parameters: + model: llama-cpp/models/Inkling-Small-UD-IQ2_M/Inkling-Small-UD-IQ2_M-00001-of-00003.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/Inkling-Small-UD-IQ2_M/Inkling-Small-UD-IQ2_M-00001-of-00003.gguf + sha256: 3b6ace30e488ad26e816cdba4e42714f40110a3142a210bd5c2e48f69e27cb31 + uri: https://huggingface.co/unsloth/Inkling-Small-GGUF/resolve/main/UD-IQ2_M/Inkling-Small-UD-IQ2_M-00001-of-00003.gguf + - filename: llama-cpp/models/Inkling-Small-UD-IQ2_M/Inkling-Small-UD-IQ2_M-00002-of-00003.gguf + sha256: 5ca94e858ae116eb513a2af1facd35844d42ef4a209e1d85cc4ecc73cd21b894 + uri: https://huggingface.co/unsloth/Inkling-Small-GGUF/resolve/main/UD-IQ2_M/Inkling-Small-UD-IQ2_M-00002-of-00003.gguf + - filename: llama-cpp/models/Inkling-Small-UD-IQ2_M/Inkling-Small-UD-IQ2_M-00003-of-00003.gguf + sha256: 8a84e00d4625d52491969f88f24f0999bc77527b7fa1d15cbe44ef88235bb377 + uri: https://huggingface.co/unsloth/Inkling-Small-GGUF/resolve/main/UD-IQ2_M/Inkling-Small-UD-IQ2_M-00003-of-00003.gguf + - filename: llama-cpp/mmproj/Inkling-Small-UD-IQ2_M/mmproj-BF16.gguf + sha256: 05d4475a956030be87b099865d6552a541a476db8cc3e266fcfa7c5a24846248 + uri: https://huggingface.co/unsloth/Inkling-Small-GGUF/resolve/main/mmproj-BF16.gguf + variants: [] - name: "qwythos-9b-v2" url: "github:mudler/LocalAI/gallery/virtual.yaml@master" urls: @@ -5262,6 +5390,82 @@ - filename: llama-cpp/models/Qwen3.5-35B-A3B-APEX-GGUF/Qwen3.5-35B-A3B-APEX-Quality.gguf sha256: 50887b60c77ee5c95bc3657814ae993abcab7b2d71868b9af1e84d6badd09a57 uri: https://huggingface.co/mudler/Qwen3.5-35B-A3B-APEX-GGUF/resolve/main/Qwen3.5-35B-A3B-APEX-Quality.gguf +- &fara1-5-9b + name: fara1.5-9b + url: github:mudler/LocalAI/gallery/virtual.yaml@master + variants: + - model: fara1.5-9b-q8 + urls: + - https://huggingface.co/microsoft/Fara1.5-9B + - https://huggingface.co/bartowski/Fara1.5-9B-GGUF + description: | + Fara1.5-9B is Microsoft's 9B-parameter multimodal computer-use agent for web browsers, fine-tuned from Qwen3.5-9B. It accepts screenshots and text, emits structured browser actions, supports a 262K-token context, and should be deployed with appropriate sandboxing and user-confirmation controls. This entry uses the recommended Q4_K_M GGUF quantization. + license: mit + tags: + - fara + - qwen + - qwen3.5 + - 9b + - llm + - gguf + - quantized + - chat + - vision + - multimodal + - agent + - computer-use + - gpu + - cpu + last_checked: "2026-08-01" + overrides: + backend: llama-cpp + function: + grammar: + disable: true + known_usecases: + - chat + - vision + mmproj: llama-cpp/mmproj/Fara1.5-9B-GGUF/mmproj-Fara1.5-9B-f16.gguf + options: + - use_jinja:true + parameters: + model: llama-cpp/models/Fara1.5-9B-GGUF/Fara1.5-9B-Q4_K_M.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/Fara1.5-9B-GGUF/Fara1.5-9B-Q4_K_M.gguf + sha256: a02e7220337b87290bca7ef7225ba4afa7104efa05c0851de6c29ec9c5d04c7d + uri: huggingface://bartowski/Fara1.5-9B-GGUF/Fara1.5-9B-Q4_K_M.gguf + - filename: llama-cpp/mmproj/Fara1.5-9B-GGUF/mmproj-Fara1.5-9B-f16.gguf + sha256: 97b423c81719ffc367124a9739d6feb6f62d62f60869a6d385a701b963ce1906 + uri: huggingface://bartowski/Fara1.5-9B-GGUF/mmproj-Fara1.5-9B-f16.gguf +- !!merge <<: *fara1-5-9b + name: fara1.5-9b-q8 + variants: [] + description: | + Fara1.5-9B is Microsoft's 9B-parameter multimodal computer-use agent for web browsers, fine-tuned from Qwen3.5-9B. It accepts screenshots and text, emits structured browser actions, supports a 262K-token context, and should be deployed with appropriate sandboxing and user-confirmation controls. This entry uses the higher-quality Q8_0 GGUF quantization. + overrides: + backend: llama-cpp + function: + grammar: + disable: true + known_usecases: + - chat + - vision + mmproj: llama-cpp/mmproj/Fara1.5-9B-GGUF/mmproj-Fara1.5-9B-f16.gguf + options: + - use_jinja:true + parameters: + model: llama-cpp/models/Fara1.5-9B-GGUF/Fara1.5-9B-Q8_0.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/Fara1.5-9B-GGUF/Fara1.5-9B-Q8_0.gguf + sha256: a2e30cca7aec006266308153ae781347505af16baa514bbd4e0e3f4a79ea3a22 + uri: huggingface://bartowski/Fara1.5-9B-GGUF/Fara1.5-9B-Q8_0.gguf + - filename: llama-cpp/mmproj/Fara1.5-9B-GGUF/mmproj-Fara1.5-9B-f16.gguf + sha256: 97b423c81719ffc367124a9739d6feb6f62d62f60869a6d385a701b963ce1906 + uri: huggingface://bartowski/Fara1.5-9B-GGUF/mmproj-Fara1.5-9B-f16.gguf - name: fara1.5-27b url: github:mudler/LocalAI/gallery/virtual.yaml@master variants: @@ -6010,14 +6214,14 @@ - instruction-tuned - code - math - last_checked: "2026-07-28" + last_checked: "2026-08-01" overrides: parameters: - model: nanbeige4.2-3b-Q8_0.gguf + model: Nanbeige4.2-3B-Q8_0.gguf files: - - filename: nanbeige4.2-3b-Q8_0.gguf - sha256: 44707bb25e7ba3f2b0b5f3c2311da95ee3676986dd2014ce2aaeb14113590e33 - uri: huggingface://owao/Nanbeige4.2-3B-GGUF/nanbeige4.2-3b-Q8_0.gguf + - filename: Nanbeige4.2-3B-Q8_0.gguf + sha256: 4f8bd17cdf58bea2a94aef03457e0b8f019c26fe4daee7ae49b61bfa935a9126 + uri: huggingface://owao/Nanbeige4.2-3B-GGUF/Nanbeige4.2-3B-Q8_0.gguf - name: nanbeige4.2-3b url: github:mudler/LocalAI/gallery/nanbeige4.2.yaml@master urls: @@ -6040,16 +6244,16 @@ - instruction-tuned - code - math - last_checked: "2026-07-28" + last_checked: "2026-08-01" variants: - model: nanbeige4.2-3b-q8 overrides: parameters: - model: nanbeige4.2-3b-Q4_K_M.gguf + model: Nanbeige4.2-3B-Q4_K_M.gguf files: - - filename: nanbeige4.2-3b-Q4_K_M.gguf - sha256: 9ffd17d14472ff208409b3f51a6d87a5e5ec1b878b9a6f4dfe15c2a883366104 - uri: huggingface://owao/Nanbeige4.2-3B-GGUF/nanbeige4.2-3b-Q4_K_M.gguf + - filename: Nanbeige4.2-3B-Q4_K_M.gguf + sha256: ffe1b9b8ee95ec4b962c379905aa8be6f72ae9c4645c6c70e3b6ff7b197e6ef4 + uri: huggingface://owao/Nanbeige4.2-3B-GGUF/Nanbeige4.2-3B-Q4_K_M.gguf - name: nemo-parakeet-tdt-0.6b url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: diff --git a/pkg/utils/untar.go b/pkg/utils/untar.go index cb65bb6b5ec0..c8521648e240 100644 --- a/pkg/utils/untar.go +++ b/pkg/utils/untar.go @@ -70,6 +70,11 @@ func ExtractArchive(archive, dst string) error { if f.FileInfo.Mode()&os.ModeSymlink != 0 { return fmt.Errorf("archive contains a symlink") } + if linkname, ok := archiveMemberLinkname(f); ok { + if err := validateArchiveMemberPath(extractRoot, linkname); err != nil { + return err + } + } return nil }) @@ -95,6 +100,18 @@ func archiveMemberName(f archiver.File) string { } } +// archiveMemberLinkname reports the target of a tar hardlink member, which carries a regular file mode and so is not caught by the symlink check. +func archiveMemberLinkname(f archiver.File) (string, bool) { + switch h := f.Header.(type) { + case tar.Header: + return h.Linkname, h.Typeflag == tar.TypeLink + case *tar.Header: + return h.Linkname, h.Typeflag == tar.TypeLink + default: + return "", false + } +} + func validateArchiveMemberPath(root, name string) error { if name == "" { return fmt.Errorf("archive contains an empty path") diff --git a/pkg/utils/untar_test.go b/pkg/utils/untar_test.go index e82b3611f790..7d40125803b7 100644 --- a/pkg/utils/untar_test.go +++ b/pkg/utils/untar_test.go @@ -3,6 +3,7 @@ package utils_test import ( "archive/tar" "archive/zip" + "compress/gzip" "os" "path/filepath" @@ -59,6 +60,53 @@ var _ = Describe("utils/archive tests", func() { Expect(err.Error()).To(ContainSubstring("unsafe path")) Expect(filepath.Join(tmpDir, "escaped.txt")).ToNot(BeAnExistingFile()) }) + + It("rejects tar hardlinks that overwrite a file outside the destination", func() { + tmpDir := GinkgoT().TempDir() + archivePath := filepath.Join(tmpDir, "model.tar.gz") + extractPath := filepath.Join(tmpDir, "models") + outsidePath := filepath.Join(tmpDir, "outside.txt") + + Expect(os.WriteFile(outsidePath, []byte("original"), 0o600)).To(Succeed()) + Expect(writeTarGzArchiveWithHardlinkedFile(archivePath, "payload.bin", "../outside.txt", "overwritten")).To(Succeed()) + + err := ExtractArchive(archivePath, extractPath) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("unsafe path")) + + contents, readErr := os.ReadFile(outsidePath) + Expect(readErr).ToNot(HaveOccurred()) + Expect(string(contents)).To(Equal("original")) + }) + + It("extracts tar hardlinks that stay inside the destination", func() { + tmpDir := GinkgoT().TempDir() + archivePath := filepath.Join(tmpDir, "model.tar.gz") + extractPath := filepath.Join(tmpDir, "models") + + Expect(writeTarGzArchiveWithInternalHardlink(archivePath, "model.bin", "alias.bin", "weights")).To(Succeed()) + + Expect(ExtractArchive(archivePath, extractPath)).To(Succeed()) + + extracted, err := os.ReadFile(filepath.Join(extractPath, "alias.bin")) + Expect(err).ToNot(HaveOccurred()) + Expect(string(extracted)).To(Equal("weights")) + }) + + It("rejects tar hardlinks that point outside the destination", func() { + tmpDir := GinkgoT().TempDir() + archivePath := filepath.Join(tmpDir, "model.tar") + extractPath := filepath.Join(tmpDir, "models") + + Expect(writeTarArchiveWithHardlink(archivePath, "payload.bin", "../escaped.txt")).To(Succeed()) + + err := ExtractArchive(archivePath, extractPath) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("unsafe path")) + Expect(filepath.Join(tmpDir, "escaped.txt")).ToNot(BeAnExistingFile()) + }) }) func writeZipArchive(path string, files map[string]string) (err error) { @@ -126,3 +174,121 @@ func writeTarArchive(path string, files map[string]string) (err error) { return nil } + +func writeTarArchiveWithHardlink(path, name, linkname string) (err error) { + out, err := os.Create(path) + if err != nil { + return err + } + defer func() { + if closeErr := out.Close(); err == nil { + err = closeErr + } + }() + + writer := tar.NewWriter(out) + defer func() { + if closeErr := writer.Close(); err == nil { + err = closeErr + } + }() + + return writer.WriteHeader(&tar.Header{ + Name: name, + Linkname: linkname, + Typeflag: tar.TypeLink, + Mode: 0o600, + }) +} + +func writeTarGzArchiveWithHardlinkedFile(path, name, linkname, contents string) (err error) { + out, err := os.Create(path) + if err != nil { + return err + } + defer func() { + if closeErr := out.Close(); err == nil { + err = closeErr + } + }() + + compressor := gzip.NewWriter(out) + defer func() { + if closeErr := compressor.Close(); err == nil { + err = closeErr + } + }() + + writer := tar.NewWriter(compressor) + defer func() { + if closeErr := writer.Close(); err == nil { + err = closeErr + } + }() + + if err := writer.WriteHeader(&tar.Header{ + Name: name, + Linkname: linkname, + Typeflag: tar.TypeLink, + Mode: 0o600, + }); err != nil { + return err + } + + data := []byte(contents) + if err := writer.WriteHeader(&tar.Header{ + Name: name, + Mode: 0o600, + Size: int64(len(data)), + }); err != nil { + return err + } + _, err = writer.Write(data) + + return err +} + +func writeTarGzArchiveWithInternalHardlink(path, targetName, linkName, contents string) (err error) { + out, err := os.Create(path) + if err != nil { + return err + } + defer func() { + if closeErr := out.Close(); err == nil { + err = closeErr + } + }() + + compressor := gzip.NewWriter(out) + defer func() { + if closeErr := compressor.Close(); err == nil { + err = closeErr + } + }() + + writer := tar.NewWriter(compressor) + defer func() { + if closeErr := writer.Close(); err == nil { + err = closeErr + } + }() + + data := []byte(contents) + if err := writer.WriteHeader(&tar.Header{ + Name: targetName, + Mode: 0o600, + Size: int64(len(data)), + }); err != nil { + return err + } + if _, err := writer.Write(data); err != nil { + return err + } + + return writer.WriteHeader(&tar.Header{ + Name: linkName, + Linkname: targetName, + Typeflag: tar.TypeLink, + Mode: 0o600, + }) +} diff --git a/scripts/build/llama-cpp-build-target_test.sh b/scripts/build/llama-cpp-build-target_test.sh new file mode 100755 index 000000000000..3d3970b5551e --- /dev/null +++ b/scripts/build/llama-cpp-build-target_test.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail + +CURDIR=$(dirname "$(realpath "$0")") +SELECTOR="$CURDIR/../../.docker/llama-cpp-build-target.sh" + +assert_target() { + local arch=$1 + local build_type=$2 + local expected=$3 + local actual + + actual=$("$SELECTOR" "$arch" "$build_type") + if [ "$actual" != "$expected" ]; then + echo "FAIL: $arch/$build_type selected $actual, expected $expected" + exit 1 + fi +} + +assert_target amd64 cublas llama-cpp-cpu-all +assert_target amd64 vulkan llama-cpp-cpu-all +assert_target amd64 "" llama-cpp-cpu-all +assert_target arm64 cublas llama-cpp-fallback +assert_target arm64 "" llama-cpp-cpu-all + +echo "PASS: llama.cpp build target preserves CPU variants where supported"