[pull] master from mudler:master

.github/bump_vllm_metal.sh

@@ -3,9 +3,9 @@
 # darwin (Apple Silicon) install path. The macOS/Metal build
 # (backend/python/vllm/install.sh, Darwin branch) installs vllm-metal, which is
 # version-locked to a specific vLLM source release. install.sh derives that vLLM
-# version at build time from vllm-metal's own installer at the pinned
-# tag, so there is only ONE value to bump here -- mirroring bump_vllm_wheel.sh,
-# which bumps the Linux cu130 wheel pin.
+# version, and the wheel asset name, at build time from the pinned tag, so there
+# is only ONE value to bump here -- mirroring bump_vllm_wheel.sh, which bumps the
+# Linux cu130 wheel pin.
 #
 # This deliberately tracks vllm-project/vllm-metal, NOT vllm-project/vllm: the
 # darwin build can only use the exact vLLM version vllm-metal supports, so it may
@@ -23,15 +23,20 @@ if [ -z "$FILE" ] || [ -z "$REPO" ] || [ -z "$VAR" ]; then
     exit 1
 fi
 
-# vllm-metal ships frequent dev releases, all flagged as non-prerelease, so
-# /releases/latest returns the newest one (with its cp312 wheel asset).
+# vllm-metal ships frequent .dev releases, flagged as prereleases, alongside the
+# stable ones. /releases/latest skips the prereleases and returns the newest
+# stable tag, which is what darwin should pin: upstream deletes and re-cuts .dev
+# tags, and a pin to a deleted tag 404s the whole build.
 LATEST_TAG=$(gh_curl -H "Accept: application/vnd.github+json" \
     "https://api.github.com/repos/$REPO/releases/latest" \
     | python3 -c "import json,sys; print(json.load(sys.stdin)['tag_name'])")
 
-# The coupled vLLM source version lives in vllm-metal's installer at that tag.
-NEW_VLLM_VERSION=$(gh_curl \
-    "https://raw.githubusercontent.com/$REPO/$LATEST_TAG/install.sh" \
+# The coupled vLLM release lives in .github/vllm-release-tag.commit at that tag
+# (since vllm-metal 0.28); releases predating that file pinned it inline in their
+# own install.sh. The extractor reads both forms.
+NEW_VLLM_VERSION=$( { gh_curl \
+    "https://raw.githubusercontent.com/$REPO/$LATEST_TAG/.github/vllm-release-tag.commit" \
+    || gh_curl "https://raw.githubusercontent.com/$REPO/$LATEST_TAG/install.sh"; } \
     | "$(dirname "${BASH_SOURCE[0]}")/../scripts/lib/extract-vllm-metal-version.sh")
 
 if [ -z "$LATEST_TAG" ] || [ -z "$NEW_VLLM_VERSION" ]; then

backend/cpp/audio-cpp/Makefile

@@ -9,7 +9,7 @@
 # recipe is a make target (not a prepare.sh) so 'make purge && make' is a clean
 # rebuild and so the bump bot can see the pin.
 
-AUDIO_CPP_VERSION?=f334cff70a68ea3d2e40d6638733e8c1ec434164
+AUDIO_CPP_VERSION?=c18b7f737aac0a2855e9f963a427498739ad40fe
 AUDIO_CPP_REPO?=https://github.com/0xShug0/audio.cpp
 
 CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))

backend/python/vllm/install.sh

@@ -119,14 +119,18 @@ if [ "$(uname -s)" = "Darwin" ]; then
     # can rewrite it. Darwin therefore follows vllm-metal and can lag the Linux
     # vllm pin (requirements-cublas13-after.txt, bumped independently against
     # vllm/vllm) until vllm-metal supports a newer vLLM.
-    VLLM_METAL_VERSION="v0.3.0.dev20260818075955"
+    VLLM_METAL_VERSION="v0.28.0"
 
     # The coupled vLLM source version is whatever this vllm-metal release builds
-    # against. Derive it from
-    # the PINNED tag rather than hardcoding a second value that could drift. The
-    # tag is immutable, so this stays reproducible across rebuilds.
-    VLLM_VERSION=$(curl -fsSL "https://raw.githubusercontent.com/vllm-project/vllm-metal/${VLLM_METAL_VERSION}/install.sh" \
-        | "$backend_dir/../../../scripts/lib/extract-vllm-metal-version.sh")
+    # against. Derive it from the PINNED tag rather than hardcoding a second value
+    # that could drift. The tag is immutable, so this stays reproducible across
+    # rebuilds. Since vllm-metal 0.28 the coupling is declared in
+    # .github/vllm-release-tag.commit; older releases pinned it inline in their
+    # own install.sh, so fall back to that. The extractor reads both forms.
+    _vllm_metal_raw="https://raw.githubusercontent.com/vllm-project/vllm-metal/${VLLM_METAL_VERSION}"
+    VLLM_VERSION=$( { curl -fsSL "${_vllm_metal_raw}/.github/vllm-release-tag.commit" \
+        || curl -fsSL "${_vllm_metal_raw}/install.sh"; } \
+        | "$backend_dir/../../../scripts/lib/extract-vllm-metal-version.sh" || true)
     if [ -z "${VLLM_VERSION}" ]; then
         echo "ERROR: could not derive the vLLM version from vllm-metal ${VLLM_METAL_VERSION}" >&2
         exit 1
@@ -153,10 +157,18 @@ if [ "$(uname -s)" = "Darwin" ]; then
     # 2) Install the prebuilt vllm-metal wheel for the PINNED release. It pulls
     #    mlx / mlx-metal as deps and registers the `metal` platform plugin that
     #    backend.py resolves to at engine-init time. Build the release-asset URL
-    #    deterministically (tag + the cp312/arm64 wheel name) rather than querying
-    #    api.github.com, whose unauthenticated rate limit (60/hr per IP) 403s on
-    #    shared CI runners. The wheel version is the tag without its leading 'v'.
-    _metal_wheel="vllm_metal-${VLLM_METAL_VERSION#v}-cp312-cp312-macosx_11_0_arm64.whl"
+    #    from the release's OWN asset listing rather than composing it from a
+    #    hardcoded platform tag: upstream raised its macOS deployment target
+    #    (macosx_11_0 -> macosx_15_0) and every composed URL started to 404.
+    #    expanded_assets is the plain release page, not api.github.com, whose
+    #    unauthenticated rate limit (60/hr per IP) 403s on shared CI runners.
+    #    The wheel version is the tag without its leading 'v'.
+    _metal_wheel=$(curl -fsSL "https://github.com/vllm-project/vllm-metal/releases/expanded_assets/${VLLM_METAL_VERSION}" \
+        | grep -oE "vllm_metal-${VLLM_METAL_VERSION#v}-cp312-cp312-[A-Za-z0-9_]+\.whl" | head -1 || true)
+    if [ -z "${_metal_wheel}" ]; then
+        echo "ERROR: no cp312 wheel asset on vllm-metal release ${VLLM_METAL_VERSION}" >&2
+        exit 1
+    fi
     _metal_wheel_url="https://github.com/vllm-project/vllm-metal/releases/download/${VLLM_METAL_VERSION}/${_metal_wheel}"
     echo "Installing vllm-metal wheel: ${_metal_wheel_url}"
     uv pip install "${_metal_wheel_url}"

core/gallery/empty_base_install_test.go

@@ -236,8 +236,15 @@ var _ = Describe("InstallModelFromGallery with an empty base config", func() {
 		Expect(install(e.Name, gallery.GalleryModel{})).To(Succeed())
 		cfg := installedConfig(e.Name)
 		Expect(cfg["name"]).To(Equal(e.Name))
-		// The catalog's own overrides, verbatim, laid over the empty base.
-		Expect(cfg["parameters"]).To(Equal(e.Overrides["parameters"]))
+		// The catalog's own overrides, laid over the empty base. parameters is
+		// checked key by key rather than as a whole map: the install also merges
+		// the model family's inference defaults into it, and what matters here is
+		// that the authored keys survive that.
+		authored, ok := e.Overrides["parameters"].(map[string]any)
+		Expect(ok).To(BeTrue())
+		for key, want := range authored {
+			Expect(cfg["parameters"]).To(HaveKeyWithValue(key, want))
+		}
 		Expect(cfg["known_usecases"]).To(Equal(e.Overrides["known_usecases"]))
 	})
 })

core/gallery/inference_defaults_install_test.go

@@ -0,0 +1,189 @@
+package gallery_test
+
+import (
+	"context"
+	"fmt"
+	"maps"
+	"os"
+	"path/filepath"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"gopkg.in/yaml.v3"
+
+	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/core/gallery"
+	"github.com/mudler/LocalAI/pkg/modelartifacts"
+	"github.com/mudler/LocalAI/pkg/system"
+)
+
+// The recommended sampling parameters for a model family are applied at install
+// and persisted into the model YAML. Persisting them is only worth anything if
+// they are written where the loader reads them back: PredictionOptions is nested
+// under "parameters" in ModelConfig, so a top level "temperature" key parses
+// without error and is then ignored for the life of the model.
+//
+// The expected values are read from the family table rather than written out
+// here, so that retuning a family stays a one file change.
+//
+// Nothing here reaches the network.
+var _ = Describe("Inference defaults persisted at install", func() {
+	var tempdir string
+	var galleries []config.Gallery
+	var systemState *system.SystemState
+	// The gallery listing is cached on the name and URL pair, so every spec
+	// needs a gallery of its own or it reads the previous spec's catalog.
+	galleryRevision := 0
+
+	// The name has to contain a pattern from inference_defaults.json, otherwise
+	// no defaults are applied and every assertion below passes vacuously.
+	const modelName = "qwen3.5-install-defaults"
+
+	newGallery := func(entries ...gallery.GalleryModel) {
+		out, err := yaml.Marshal(entries)
+		Expect(err).ToNot(HaveOccurred())
+		name := fmt.Sprintf("inference-defaults-%d", galleryRevision)
+		galleryRevision++
+		galleryPath := filepath.Join(tempdir, name+".yaml")
+		Expect(os.WriteFile(galleryPath, out, 0600)).To(Succeed())
+		galleries = []config.Gallery{{Name: name, URL: "file://" + galleryPath}}
+	}
+
+	install := func(name string) error {
+		return gallery.InstallModelFromGallery(
+			context.TODO(), galleries, []config.Gallery{}, systemState, nil,
+			name, gallery.GalleryModel{}, func(string, string, string, float64) {}, false, false, false)
+	}
+
+	installedConfig := func(name string) map[string]any {
+		dat, err := os.ReadFile(filepath.Join(tempdir, name+".yaml"))
+		Expect(err).ToNot(HaveOccurred())
+		content := map[string]any{}
+		Expect(yaml.Unmarshal(dat, &content)).To(Succeed())
+		return content
+	}
+
+	// Seeding the weights keeps the install off the network: the downloader
+	// treats an already-present destination with no declared sha256 as fetched.
+	// extra goes into parameters:, so a spec can pin a value the defaults would
+	// otherwise supply.
+	seedGallery := func(extra map[string]any) {
+		Expect(os.WriteFile(filepath.Join(tempdir, "weights.gguf"), []byte("weights"), 0600)).To(Succeed())
+
+		params := map[string]any{"model": "weights.gguf"}
+		maps.Copy(params, extra)
+
+		e := gallery.GalleryModel{Overrides: map[string]any{
+			"backend":    "llama-cpp",
+			"parameters": params,
+		}}
+		e.Name = modelName
+		e.AdditionalFiles = []gallery.File{{Filename: "weights.gguf", URI: "https://example.com/weights.gguf"}}
+		newGallery(e)
+	}
+
+	// Guards the fixture itself. If the name stops matching a family the specs
+	// below would still pass while asserting nothing at all.
+	expectedFamily := func() map[string]float64 {
+		family := config.MatchModelFamily(modelName)
+		Expect(family).ToNot(BeEmpty(), "fixture name no longer matches a family in inference_defaults.json")
+		return family
+	}
+
+	BeforeEach(func() {
+		var err error
+		tempdir, err = os.MkdirTemp("", "inference-defaults-install")
+		Expect(err).ToNot(HaveOccurred())
+		DeferCleanup(func() { Expect(os.RemoveAll(tempdir)).To(Succeed()) })
+
+		systemState, err = system.GetSystemState(system.WithModelPath(tempdir))
+		Expect(err).ToNot(HaveOccurred())
+	})
+
+	It("writes them under parameters, where the loader reads them back", func() {
+		family := expectedFamily()
+		seedGallery(nil)
+
+		Expect(install(modelName)).To(Succeed())
+
+		params, ok := installedConfig(modelName)["parameters"].(map[string]any)
+		Expect(ok).To(BeTrue(), "parameters should be a map")
+
+		for key, want := range family {
+			Expect(params).To(HaveKey(key))
+			Expect(params[key]).To(BeNumerically("==", want), "parameters.%s", key)
+		}
+	})
+
+	It("does not leave them at the top level, where they are ignored", func() {
+		family := expectedFamily()
+		seedGallery(nil)
+
+		Expect(install(modelName)).To(Succeed())
+
+		cfg := installedConfig(modelName)
+		for key := range family {
+			Expect(cfg).ToNot(HaveKey(key), "%s at the top level is never read", key)
+		}
+	})
+
+	It("leaves a value the entry already sets alone", func() {
+		family := expectedFamily()
+		Expect(family).To(HaveKey("temperature"))
+		Expect(family["temperature"]).ToNot(BeNumerically("==", 0.05), "pick a value the family does not use")
+
+		seedGallery(map[string]any{"temperature": 0.05})
+
+		Expect(install(modelName)).To(Succeed())
+
+		params, ok := installedConfig(modelName)["parameters"].(map[string]any)
+		Expect(ok).To(BeTrue(), "parameters should be a map")
+		Expect(params["temperature"]).To(BeNumerically("==", 0.05))
+	})
+
+	// An entry that binds a primary artifact carries no files: of its own, so it
+	// takes the other branch of the install and none of the specs above reach it.
+	// It is also the one branch that already re-marshalled, which is why the
+	// defaults did land on disk there, at the top level where nothing reads them.
+	It("writes them under parameters on the artifact binding path too", func() {
+		family := expectedFamily()
+
+		definition := &gallery.ModelConfig{ConfigFile: `
+backend: transformers
+artifacts:
+  - name: model
+    target: model
+    source:
+      type: huggingface
+      repo: owner/repo
+parameters:
+  model: owner/repo
+`}
+		// Standing in for the materializer keeps the install off the network.
+		materializer := &fakeArtifactMaterializer{result: modelartifacts.Result{
+			Spec: modelartifacts.Spec{
+				Name: "model", Target: "model",
+				Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo", Revision: "main"},
+				Resolved: &modelartifacts.Resolved{
+					Endpoint: "https://huggingface.co",
+					Revision: "0123456789abcdef0123456789abcdef01234567",
+					CacheKey: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
+				},
+			},
+			RelativePath: ".artifacts/huggingface/0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef/snapshot",
+		}}
+
+		_, err := gallery.InstallModel(context.TODO(), systemState, modelName, definition, nil, nil, false,
+			gallery.WithArtifactMaterializer(materializer))
+		Expect(err).ToNot(HaveOccurred())
+
+		cfg := installedConfig(modelName)
+		params, ok := cfg["parameters"].(map[string]any)
+		Expect(ok).To(BeTrue(), "parameters should be a map")
+		for key, want := range family {
+			Expect(params).To(HaveKey(key))
+			Expect(params[key]).To(BeNumerically("==", want), "parameters.%s", key)
+			Expect(cfg).ToNot(HaveKey(key), "%s at the top level is never read", key)
+		}
+	})
+})

core/gallery/models.go

@@ -622,35 +622,51 @@ func InstallModel(ctx context.Context, systemState *system.SystemState, nameOver
 		lconfig.ApplyInferenceDefaults(&modelConfig, name, modelConfig.Model)
 
 		// Merge inference defaults into configMap so they are persisted without losing unknown fields.
-		if modelConfig.Temperature != nil {
-			if _, exists := configMap["temperature"]; !exists {
-				configMap["temperature"] = *modelConfig.Temperature
-			}
+		// They belong under "parameters": ModelConfig embeds PredictionOptions with
+		// that yaml key, so a top level "temperature" parses without error and is
+		// then ignored for the life of the model.
+		params, mergeable := configMap["parameters"].(map[string]any)
+		if configMap["parameters"] == nil {
+			params, mergeable = map[string]any{}, true
 		}
-		if modelConfig.TopP != nil {
-			if _, exists := configMap["top_p"]; !exists {
-				configMap["top_p"] = *modelConfig.TopP
+		if mergeable {
+			// An entry that sets one of these keeps its own value. ApplyInferenceDefaults
+			// already skipped those fields; this keeps the write side symmetric.
+			setDefault := func(key string, value any) {
+				if _, exists := params[key]; !exists {
+					params[key] = value
+				}
 			}
-		}
-		if modelConfig.TopK != nil {
-			if _, exists := configMap["top_k"]; !exists {
-				configMap["top_k"] = *modelConfig.TopK
+			if modelConfig.Temperature != nil {
+				setDefault("temperature", *modelConfig.Temperature)
 			}
-		}
-		if modelConfig.MinP != nil {
-			if _, exists := configMap["min_p"]; !exists {
-				configMap["min_p"] = *modelConfig.MinP
+			if modelConfig.TopP != nil {
+				setDefault("top_p", *modelConfig.TopP)
 			}
-		}
-		if modelConfig.RepeatPenalty != 0 {
-			if _, exists := configMap["repeat_penalty"]; !exists {
-				configMap["repeat_penalty"] = modelConfig.RepeatPenalty
+			if modelConfig.TopK != nil {
+				setDefault("top_k", *modelConfig.TopK)
 			}
-		}
-		if modelConfig.PresencePenalty != 0 {
-			if _, exists := configMap["presence_penalty"]; !exists {
-				configMap["presence_penalty"] = modelConfig.PresencePenalty
+			if modelConfig.MinP != nil {
+				setDefault("min_p", *modelConfig.MinP)
+			}
+			if modelConfig.RepeatPenalty != 0 {
+				setDefault("repeat_penalty", modelConfig.RepeatPenalty)
 			}
+			if modelConfig.PresencePenalty != 0 {
+				setDefault("presence_penalty", modelConfig.PresencePenalty)
+			}
+			if len(params) > 0 {
+				configMap["parameters"] = params
+			}
+		}
+
+		// The marshal above predates this merge, and the only other re-marshal is
+		// behind the artifact binding below, which an entry carrying files: never
+		// reaches. Without this the defaults are computed and then dropped on the
+		// way to disk.
+		updatedConfigYAML, err = yaml.Marshal(configMap)
+		if err != nil {
+			return nil, fmt.Errorf("failed to marshal config with inference defaults: %v", err)
 		}
 
 		if valid, err := modelConfig.Validate(); !valid {