From 49945fdd7555f1b37dc9e12e2d2dc339d9351203 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Thu, 3 Sep 2026 18:03:20 +0200 Subject: [PATCH 1/4] chore: :arrow_up: Update 0xShug0/audio.cpp to `c18b7f737aac0a2855e9f963a427498739ad40fe` (#11843) :arrow_up: Update 0xShug0/audio.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/cpp/audio-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cpp/audio-cpp/Makefile b/backend/cpp/audio-cpp/Makefile index 4c836a0cb24d..572667f60bed 100644 --- a/backend/cpp/audio-cpp/Makefile +++ b/backend/cpp/audio-cpp/Makefile @@ -9,7 +9,7 @@ # recipe is a make target (not a prepare.sh) so 'make purge && make' is a clean # rebuild and so the bump bot can see the pin. -AUDIO_CPP_VERSION?=f334cff70a68ea3d2e40d6638733e8c1ec434164 +AUDIO_CPP_VERSION?=c18b7f737aac0a2855e9f963a427498739ad40fe AUDIO_CPP_REPO?=https://github.com/0xShug0/audio.cpp CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) From 8aeea4cddee0f1e5b484550ee520db7def505253 Mon Sep 17 00:00:00 2001 From: Dimitris Karakasilis Date: Thu, 3 Sep 2026 19:30:26 +0300 Subject: [PATCH 2/4] fix(gallery): persist inference defaults where the loader reads them (#11232) The recommended sampling parameters for a model family were applied at install and then never took effect. Two things went wrong on the way to disk. They were written as top level keys. ModelConfig embeds PredictionOptions under the "parameters" yaml key, so temperature, top_p, top_k, min_p, repeat_penalty and presence_penalty are only read from there. At the top level they parse without error and are then ignored for the life of the model. They were also merged in after the YAML had already been marshalled. The only re-marshal sat behind the artifact binding, which an entry carrying files: never reaches, so for those entries the defaults were computed and then dropped before anything was written. Neither failure was visible in normal use. ApplyInferenceDefaults runs again at load time and fills the same values from the same table, so the model ends up tuned correctly while the file on disk pins nothing. It surfaces when someone edits one of those values expecting it to win, or when a family is absent from inference_defaults.json and there is nothing to refill from. Both install paths are covered: an entry carrying files:, and one that binds a primary artifact instead. The empty base spec asserted that the authored parameters block landed verbatim. It now checks the authored keys individually, because the family defaults are merged into that same block. Assisted-by: Claude:claude-opus-5 Signed-off-by: Dimitris Karakasilis --- core/gallery/empty_base_install_test.go | 11 +- .../inference_defaults_install_test.go | 189 ++++++++++++++++++ core/gallery/models.go | 62 +++--- 3 files changed, 237 insertions(+), 25 deletions(-) create mode 100644 core/gallery/inference_defaults_install_test.go diff --git a/core/gallery/empty_base_install_test.go b/core/gallery/empty_base_install_test.go index 6b104e233f02..b2a15c1ea5a0 100644 --- a/core/gallery/empty_base_install_test.go +++ b/core/gallery/empty_base_install_test.go @@ -236,8 +236,15 @@ var _ = Describe("InstallModelFromGallery with an empty base config", func() { Expect(install(e.Name, gallery.GalleryModel{})).To(Succeed()) cfg := installedConfig(e.Name) Expect(cfg["name"]).To(Equal(e.Name)) - // The catalog's own overrides, verbatim, laid over the empty base. - Expect(cfg["parameters"]).To(Equal(e.Overrides["parameters"])) + // The catalog's own overrides, laid over the empty base. parameters is + // checked key by key rather than as a whole map: the install also merges + // the model family's inference defaults into it, and what matters here is + // that the authored keys survive that. + authored, ok := e.Overrides["parameters"].(map[string]any) + Expect(ok).To(BeTrue()) + for key, want := range authored { + Expect(cfg["parameters"]).To(HaveKeyWithValue(key, want)) + } Expect(cfg["known_usecases"]).To(Equal(e.Overrides["known_usecases"])) }) }) diff --git a/core/gallery/inference_defaults_install_test.go b/core/gallery/inference_defaults_install_test.go new file mode 100644 index 000000000000..11f46bc42d68 --- /dev/null +++ b/core/gallery/inference_defaults_install_test.go @@ -0,0 +1,189 @@ +package gallery_test + +import ( + "context" + "fmt" + "maps" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/pkg/modelartifacts" + "github.com/mudler/LocalAI/pkg/system" +) + +// The recommended sampling parameters for a model family are applied at install +// and persisted into the model YAML. Persisting them is only worth anything if +// they are written where the loader reads them back: PredictionOptions is nested +// under "parameters" in ModelConfig, so a top level "temperature" key parses +// without error and is then ignored for the life of the model. +// +// The expected values are read from the family table rather than written out +// here, so that retuning a family stays a one file change. +// +// Nothing here reaches the network. +var _ = Describe("Inference defaults persisted at install", func() { + var tempdir string + var galleries []config.Gallery + var systemState *system.SystemState + // The gallery listing is cached on the name and URL pair, so every spec + // needs a gallery of its own or it reads the previous spec's catalog. + galleryRevision := 0 + + // The name has to contain a pattern from inference_defaults.json, otherwise + // no defaults are applied and every assertion below passes vacuously. + const modelName = "qwen3.5-install-defaults" + + newGallery := func(entries ...gallery.GalleryModel) { + out, err := yaml.Marshal(entries) + Expect(err).ToNot(HaveOccurred()) + name := fmt.Sprintf("inference-defaults-%d", galleryRevision) + galleryRevision++ + galleryPath := filepath.Join(tempdir, name+".yaml") + Expect(os.WriteFile(galleryPath, out, 0600)).To(Succeed()) + galleries = []config.Gallery{{Name: name, URL: "file://" + galleryPath}} + } + + install := func(name string) error { + return gallery.InstallModelFromGallery( + context.TODO(), galleries, []config.Gallery{}, systemState, nil, + name, gallery.GalleryModel{}, func(string, string, string, float64) {}, false, false, false) + } + + installedConfig := func(name string) map[string]any { + dat, err := os.ReadFile(filepath.Join(tempdir, name+".yaml")) + Expect(err).ToNot(HaveOccurred()) + content := map[string]any{} + Expect(yaml.Unmarshal(dat, &content)).To(Succeed()) + return content + } + + // Seeding the weights keeps the install off the network: the downloader + // treats an already-present destination with no declared sha256 as fetched. + // extra goes into parameters:, so a spec can pin a value the defaults would + // otherwise supply. + seedGallery := func(extra map[string]any) { + Expect(os.WriteFile(filepath.Join(tempdir, "weights.gguf"), []byte("weights"), 0600)).To(Succeed()) + + params := map[string]any{"model": "weights.gguf"} + maps.Copy(params, extra) + + e := gallery.GalleryModel{Overrides: map[string]any{ + "backend": "llama-cpp", + "parameters": params, + }} + e.Name = modelName + e.AdditionalFiles = []gallery.File{{Filename: "weights.gguf", URI: "https://example.com/weights.gguf"}} + newGallery(e) + } + + // Guards the fixture itself. If the name stops matching a family the specs + // below would still pass while asserting nothing at all. + expectedFamily := func() map[string]float64 { + family := config.MatchModelFamily(modelName) + Expect(family).ToNot(BeEmpty(), "fixture name no longer matches a family in inference_defaults.json") + return family + } + + BeforeEach(func() { + var err error + tempdir, err = os.MkdirTemp("", "inference-defaults-install") + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { Expect(os.RemoveAll(tempdir)).To(Succeed()) }) + + systemState, err = system.GetSystemState(system.WithModelPath(tempdir)) + Expect(err).ToNot(HaveOccurred()) + }) + + It("writes them under parameters, where the loader reads them back", func() { + family := expectedFamily() + seedGallery(nil) + + Expect(install(modelName)).To(Succeed()) + + params, ok := installedConfig(modelName)["parameters"].(map[string]any) + Expect(ok).To(BeTrue(), "parameters should be a map") + + for key, want := range family { + Expect(params).To(HaveKey(key)) + Expect(params[key]).To(BeNumerically("==", want), "parameters.%s", key) + } + }) + + It("does not leave them at the top level, where they are ignored", func() { + family := expectedFamily() + seedGallery(nil) + + Expect(install(modelName)).To(Succeed()) + + cfg := installedConfig(modelName) + for key := range family { + Expect(cfg).ToNot(HaveKey(key), "%s at the top level is never read", key) + } + }) + + It("leaves a value the entry already sets alone", func() { + family := expectedFamily() + Expect(family).To(HaveKey("temperature")) + Expect(family["temperature"]).ToNot(BeNumerically("==", 0.05), "pick a value the family does not use") + + seedGallery(map[string]any{"temperature": 0.05}) + + Expect(install(modelName)).To(Succeed()) + + params, ok := installedConfig(modelName)["parameters"].(map[string]any) + Expect(ok).To(BeTrue(), "parameters should be a map") + Expect(params["temperature"]).To(BeNumerically("==", 0.05)) + }) + + // An entry that binds a primary artifact carries no files: of its own, so it + // takes the other branch of the install and none of the specs above reach it. + // It is also the one branch that already re-marshalled, which is why the + // defaults did land on disk there, at the top level where nothing reads them. + It("writes them under parameters on the artifact binding path too", func() { + family := expectedFamily() + + definition := &gallery.ModelConfig{ConfigFile: ` +backend: transformers +artifacts: + - name: model + target: model + source: + type: huggingface + repo: owner/repo +parameters: + model: owner/repo +`} + // Standing in for the materializer keeps the install off the network. + materializer := &fakeArtifactMaterializer{result: modelartifacts.Result{ + Spec: modelartifacts.Spec{ + Name: "model", Target: "model", + Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo", Revision: "main"}, + Resolved: &modelartifacts.Resolved{ + Endpoint: "https://huggingface.co", + Revision: "0123456789abcdef0123456789abcdef01234567", + CacheKey: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + }, + }, + RelativePath: ".artifacts/huggingface/0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef/snapshot", + }} + + _, err := gallery.InstallModel(context.TODO(), systemState, modelName, definition, nil, nil, false, + gallery.WithArtifactMaterializer(materializer)) + Expect(err).ToNot(HaveOccurred()) + + cfg := installedConfig(modelName) + params, ok := cfg["parameters"].(map[string]any) + Expect(ok).To(BeTrue(), "parameters should be a map") + for key, want := range family { + Expect(params).To(HaveKey(key)) + Expect(params[key]).To(BeNumerically("==", want), "parameters.%s", key) + Expect(cfg).ToNot(HaveKey(key), "%s at the top level is never read", key) + } + }) +}) diff --git a/core/gallery/models.go b/core/gallery/models.go index 7664b6e8c652..c787dcde4be8 100644 --- a/core/gallery/models.go +++ b/core/gallery/models.go @@ -622,35 +622,51 @@ func InstallModel(ctx context.Context, systemState *system.SystemState, nameOver lconfig.ApplyInferenceDefaults(&modelConfig, name, modelConfig.Model) // Merge inference defaults into configMap so they are persisted without losing unknown fields. - if modelConfig.Temperature != nil { - if _, exists := configMap["temperature"]; !exists { - configMap["temperature"] = *modelConfig.Temperature - } + // They belong under "parameters": ModelConfig embeds PredictionOptions with + // that yaml key, so a top level "temperature" parses without error and is + // then ignored for the life of the model. + params, mergeable := configMap["parameters"].(map[string]any) + if configMap["parameters"] == nil { + params, mergeable = map[string]any{}, true } - if modelConfig.TopP != nil { - if _, exists := configMap["top_p"]; !exists { - configMap["top_p"] = *modelConfig.TopP + if mergeable { + // An entry that sets one of these keeps its own value. ApplyInferenceDefaults + // already skipped those fields; this keeps the write side symmetric. + setDefault := func(key string, value any) { + if _, exists := params[key]; !exists { + params[key] = value + } } - } - if modelConfig.TopK != nil { - if _, exists := configMap["top_k"]; !exists { - configMap["top_k"] = *modelConfig.TopK + if modelConfig.Temperature != nil { + setDefault("temperature", *modelConfig.Temperature) } - } - if modelConfig.MinP != nil { - if _, exists := configMap["min_p"]; !exists { - configMap["min_p"] = *modelConfig.MinP + if modelConfig.TopP != nil { + setDefault("top_p", *modelConfig.TopP) } - } - if modelConfig.RepeatPenalty != 0 { - if _, exists := configMap["repeat_penalty"]; !exists { - configMap["repeat_penalty"] = modelConfig.RepeatPenalty + if modelConfig.TopK != nil { + setDefault("top_k", *modelConfig.TopK) } - } - if modelConfig.PresencePenalty != 0 { - if _, exists := configMap["presence_penalty"]; !exists { - configMap["presence_penalty"] = modelConfig.PresencePenalty + if modelConfig.MinP != nil { + setDefault("min_p", *modelConfig.MinP) + } + if modelConfig.RepeatPenalty != 0 { + setDefault("repeat_penalty", modelConfig.RepeatPenalty) } + if modelConfig.PresencePenalty != 0 { + setDefault("presence_penalty", modelConfig.PresencePenalty) + } + if len(params) > 0 { + configMap["parameters"] = params + } + } + + // The marshal above predates this merge, and the only other re-marshal is + // behind the artifact binding below, which an entry carrying files: never + // reaches. Without this the defaults are computed and then dropped on the + // way to disk. + updatedConfigYAML, err = yaml.Marshal(configMap) + if err != nil { + return nil, fmt.Errorf("failed to marshal config with inference defaults: %v", err) } if valid, err := modelConfig.Validate(); !valid { From 9901103aac55d61daf7c1eeb1cb7640dd79ad798 Mon Sep 17 00:00:00 2001 From: Tai An Date: Thu, 3 Sep 2026 09:30:31 -0700 Subject: [PATCH 3/4] fix(downloader): make file:// installs reachable again (#11701) (#11734) fix(downloader): make file:// installs reachable again DownloadFileWithContext already has a branch that copies from a local file, but it could never run. Before reaching it the function decides whether the destination is fetchable with } else if !os.IsNotExist(err) || !URI(url).LooksLikeHTTPURL() { and LooksLikeHTTPURL is http(s) only, so any URI resolving to a local path is rejected there. Falling through requires the destination to be missing AND the source to be an HTTP URL, which a file:// source never is -- leaving the local-source branch below unreachable. A first import always has a missing destination, so importing file:///path/to/model.gguf always failed, with an error that listed file:// among the supported schemes (#11701). Name the local-source condition once as URI.hasLocalSource and use it both to admit the destination and to pick the source, so the two cannot drift apart again. Signed-off-by: Tai An --- pkg/downloader/local_source_test.go | 48 +++++++++++++++++++++++++++++ pkg/downloader/uri.go | 18 +++++++++-- 2 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 pkg/downloader/local_source_test.go diff --git a/pkg/downloader/local_source_test.go b/pkg/downloader/local_source_test.go new file mode 100644 index 000000000000..3625ec7c44bd --- /dev/null +++ b/pkg/downloader/local_source_test.go @@ -0,0 +1,48 @@ +package downloader_test + +import ( + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + + . "github.com/mudler/LocalAI/pkg/downloader" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("file:// sources", func() { + var noProgress = func(string, string, string, float64) {} + + It("copies a file:// source into a destination that does not exist yet", func() { + // the layout from issue #11701: the source sits in a nested directory, + // the destination is the flat models dir, so it cannot already exist + srcDir := GinkgoT().TempDir() + nested := filepath.Join(srcDir, "InternScience", "Agents-A1-4B-Q8_0-GGUF") + Expect(os.MkdirAll(nested, 0o755)).To(Succeed()) + srcPath := filepath.Join(nested, "Agents-A1-4B-Q8_0.gguf") + payload := []byte("GGUF-not-really-but-enough-bytes") + Expect(os.WriteFile(srcPath, payload, 0o600)).To(Succeed()) + + sum := sha256.Sum256(payload) + destPath := filepath.Join(GinkgoT().TempDir(), "Agents-A1-4B-Q8_0.gguf") + + uri := URI("file://" + srcPath) + Expect(uri.DownloadFile(destPath, hex.EncodeToString(sum[:]), 1, 1, noProgress)).To(Succeed()) + + got, err := os.ReadFile(destPath) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(Equal(payload)) + }) + + It("reports the missing source when a file:// source does not exist", func() { + destPath := filepath.Join(GinkgoT().TempDir(), "absent.gguf") + missing := filepath.Join(GinkgoT().TempDir(), "absent.gguf") + + err := URI("file://"+missing).DownloadFile(destPath, "", 1, 1, noProgress) + Expect(err).To(HaveOccurred()) + // the error must name the source that could not be read, not claim the + // destination is an unrecognized URL + Expect(err.Error()).To(ContainSubstring(missing)) + }) +}) diff --git a/pkg/downloader/uri.go b/pkg/downloader/uri.go index 90df65b0a465..afa5e0d81674 100644 --- a/pkg/downloader/uri.go +++ b/pkg/downloader/uri.go @@ -272,6 +272,20 @@ func (u URI) LooksLikeURL() bool { strings.HasPrefix(string(u), GithubURI2) } +// hasLocalSource reports whether the URI names a local file to copy from +// rather than a URL to fetch. DownloadFileWithContext both decides whether the +// destination is reachable and picks its source with this, so that the two +// cannot drift apart: they did, and every "file://" install failed because the +// reachability check admitted http(s) only, leaving the local-source branch +// unreachable for any destination that did not already exist. +func (u URI) hasLocalSource() bool { + if strings.HasPrefix(string(u), LocalPrefix) { + return true + } + _, err := os.Stat(u.ResolveURL()) + return err == nil +} + func (u URI) LooksLikeHTTPURL() bool { return strings.HasPrefix(string(u), HTTPPrefix) || strings.HasPrefix(string(u), HTTPSPrefix) @@ -643,7 +657,7 @@ func (uri URI) DownloadFileWithContext(ctx context.Context, filePath, sha string return nil } } - } else if !os.IsNotExist(err) || !URI(url).LooksLikeHTTPURL() { + } else if !os.IsNotExist(err) || !(URI(url).LooksLikeHTTPURL() || uri.hasLocalSource()) { // Error occurred while checking file existence return fmt.Errorf("could not fetch %q: local file does not exist (%v) and %q is not a recognized downloadable URL (supported schemes: %s)", filePath, err, url, strings.Join([]string{HTTPPrefix, HTTPSPrefix, LocalPrefix, HuggingFacePrefix, HuggingFacePrefix1, OllamaPrefix, OCIPrefix, OCIFilePrefix, GithubURI2}, ", ")) } @@ -720,7 +734,7 @@ func (uri URI) DownloadFileWithContext(ctx context.Context, filePath, sha string var source io.ReadCloser var contentLength int64 - if _, e := os.Stat(uri.ResolveURL()); strings.HasPrefix(string(uri), LocalPrefix) || e == nil { + if uri.hasLocalSource() { file, err := os.Open(uri.ResolveURL()) if err != nil { return fmt.Errorf("failed to open file %q: %v", uri.ResolveURL(), err) From 7a234473e8d77f5f85d2258e5eda15f356ed90d2 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Thu, 3 Sep 2026 18:32:02 +0200 Subject: [PATCH 4/4] fix(ci): unbreak the e2e build and the darwin vllm-metal pin (#11849) Two independent breakages on master make every open pull request red, for reasons unrelated to the changes under review. The e2e backend suite stopped compiling. Reply.message is `bytes` in backend.proto, so res.GetMessage() returns []byte, and strings.ToUpper wants a string. Every other call site in the file already converts. tests/e2e-backends sits behind a build tag, so `go build ./...` never compiled it and the breakage reached master unnoticed. The darwin vllm build stopped resolving. Upstream vllm-metal deleted its old dev tags and re-versioned to track the vLLM release it targets, so the pinned wheel 404s. The coupled vLLM release also moved out of upstream's install.sh into .github/vllm-release-tag.commit, and the wheel's platform tag moved from macosx_11_0 to macosx_15_0. Read the wheel name from the release's own asset listing rather than composing it from a hardcoded platform segment, so a platform-tag change cannot silently 404 again, and resolve the vLLM version from the new metadata file with a fallback to the legacy installer. The bump script and the extractor learn the same two-source lookup, so the next nightly run converges on the pin checked in here instead of reintroducing the break. Assisted-by: Claude:claude-opus-5 Signed-off-by: Ettore Di Giacinto Co-authored-by: Ettore Di Giacinto --- .github/bump_vllm_metal.sh | 21 +++++++----- backend/python/vllm/install.sh | 32 +++++++++++++------ .../build/extract-vllm-metal-version_test.sh | 21 ++++++++++++ scripts/lib/extract-vllm-metal-version.sh | 10 +++++- tests/e2e-backends/backend_test.go | 2 +- 5 files changed, 66 insertions(+), 20 deletions(-) diff --git a/.github/bump_vllm_metal.sh b/.github/bump_vllm_metal.sh index d2aedf4bcb9c..459f97fd22a0 100755 --- a/.github/bump_vllm_metal.sh +++ b/.github/bump_vllm_metal.sh @@ -3,9 +3,9 @@ # darwin (Apple Silicon) install path. The macOS/Metal build # (backend/python/vllm/install.sh, Darwin branch) installs vllm-metal, which is # version-locked to a specific vLLM source release. install.sh derives that vLLM -# version at build time from vllm-metal's own installer at the pinned -# tag, so there is only ONE value to bump here -- mirroring bump_vllm_wheel.sh, -# which bumps the Linux cu130 wheel pin. +# version, and the wheel asset name, at build time from the pinned tag, so there +# is only ONE value to bump here -- mirroring bump_vllm_wheel.sh, which bumps the +# Linux cu130 wheel pin. # # This deliberately tracks vllm-project/vllm-metal, NOT vllm-project/vllm: the # darwin build can only use the exact vLLM version vllm-metal supports, so it may @@ -23,15 +23,20 @@ if [ -z "$FILE" ] || [ -z "$REPO" ] || [ -z "$VAR" ]; then exit 1 fi -# vllm-metal ships frequent dev releases, all flagged as non-prerelease, so -# /releases/latest returns the newest one (with its cp312 wheel asset). +# vllm-metal ships frequent .dev releases, flagged as prereleases, alongside the +# stable ones. /releases/latest skips the prereleases and returns the newest +# stable tag, which is what darwin should pin: upstream deletes and re-cuts .dev +# tags, and a pin to a deleted tag 404s the whole build. LATEST_TAG=$(gh_curl -H "Accept: application/vnd.github+json" \ "https://api.github.com/repos/$REPO/releases/latest" \ | python3 -c "import json,sys; print(json.load(sys.stdin)['tag_name'])") -# The coupled vLLM source version lives in vllm-metal's installer at that tag. -NEW_VLLM_VERSION=$(gh_curl \ - "https://raw.githubusercontent.com/$REPO/$LATEST_TAG/install.sh" \ +# The coupled vLLM release lives in .github/vllm-release-tag.commit at that tag +# (since vllm-metal 0.28); releases predating that file pinned it inline in their +# own install.sh. The extractor reads both forms. +NEW_VLLM_VERSION=$( { gh_curl \ + "https://raw.githubusercontent.com/$REPO/$LATEST_TAG/.github/vllm-release-tag.commit" \ + || gh_curl "https://raw.githubusercontent.com/$REPO/$LATEST_TAG/install.sh"; } \ | "$(dirname "${BASH_SOURCE[0]}")/../scripts/lib/extract-vllm-metal-version.sh") if [ -z "$LATEST_TAG" ] || [ -z "$NEW_VLLM_VERSION" ]; then diff --git a/backend/python/vllm/install.sh b/backend/python/vllm/install.sh index 68d5ba257b5c..a4124977c282 100755 --- a/backend/python/vllm/install.sh +++ b/backend/python/vllm/install.sh @@ -119,14 +119,18 @@ if [ "$(uname -s)" = "Darwin" ]; then # can rewrite it. Darwin therefore follows vllm-metal and can lag the Linux # vllm pin (requirements-cublas13-after.txt, bumped independently against # vllm/vllm) until vllm-metal supports a newer vLLM. - VLLM_METAL_VERSION="v0.3.0.dev20260818075955" + VLLM_METAL_VERSION="v0.28.0" # The coupled vLLM source version is whatever this vllm-metal release builds - # against. Derive it from - # the PINNED tag rather than hardcoding a second value that could drift. The - # tag is immutable, so this stays reproducible across rebuilds. - VLLM_VERSION=$(curl -fsSL "https://raw.githubusercontent.com/vllm-project/vllm-metal/${VLLM_METAL_VERSION}/install.sh" \ - | "$backend_dir/../../../scripts/lib/extract-vllm-metal-version.sh") + # against. Derive it from the PINNED tag rather than hardcoding a second value + # that could drift. The tag is immutable, so this stays reproducible across + # rebuilds. Since vllm-metal 0.28 the coupling is declared in + # .github/vllm-release-tag.commit; older releases pinned it inline in their + # own install.sh, so fall back to that. The extractor reads both forms. + _vllm_metal_raw="https://raw.githubusercontent.com/vllm-project/vllm-metal/${VLLM_METAL_VERSION}" + VLLM_VERSION=$( { curl -fsSL "${_vllm_metal_raw}/.github/vllm-release-tag.commit" \ + || curl -fsSL "${_vllm_metal_raw}/install.sh"; } \ + | "$backend_dir/../../../scripts/lib/extract-vllm-metal-version.sh" || true) if [ -z "${VLLM_VERSION}" ]; then echo "ERROR: could not derive the vLLM version from vllm-metal ${VLLM_METAL_VERSION}" >&2 exit 1 @@ -153,10 +157,18 @@ if [ "$(uname -s)" = "Darwin" ]; then # 2) Install the prebuilt vllm-metal wheel for the PINNED release. It pulls # mlx / mlx-metal as deps and registers the `metal` platform plugin that # backend.py resolves to at engine-init time. Build the release-asset URL - # deterministically (tag + the cp312/arm64 wheel name) rather than querying - # api.github.com, whose unauthenticated rate limit (60/hr per IP) 403s on - # shared CI runners. The wheel version is the tag without its leading 'v'. - _metal_wheel="vllm_metal-${VLLM_METAL_VERSION#v}-cp312-cp312-macosx_11_0_arm64.whl" + # from the release's OWN asset listing rather than composing it from a + # hardcoded platform tag: upstream raised its macOS deployment target + # (macosx_11_0 -> macosx_15_0) and every composed URL started to 404. + # expanded_assets is the plain release page, not api.github.com, whose + # unauthenticated rate limit (60/hr per IP) 403s on shared CI runners. + # The wheel version is the tag without its leading 'v'. + _metal_wheel=$(curl -fsSL "https://github.com/vllm-project/vllm-metal/releases/expanded_assets/${VLLM_METAL_VERSION}" \ + | grep -oE "vllm_metal-${VLLM_METAL_VERSION#v}-cp312-cp312-[A-Za-z0-9_]+\.whl" | head -1 || true) + if [ -z "${_metal_wheel}" ]; then + echo "ERROR: no cp312 wheel asset on vllm-metal release ${VLLM_METAL_VERSION}" >&2 + exit 1 + fi _metal_wheel_url="https://github.com/vllm-project/vllm-metal/releases/download/${VLLM_METAL_VERSION}/${_metal_wheel}" echo "Installing vllm-metal wheel: ${_metal_wheel_url}" uv pip install "${_metal_wheel_url}" diff --git a/scripts/build/extract-vllm-metal-version_test.sh b/scripts/build/extract-vllm-metal-version_test.sh index f04bcfea38b9..1018b30ca351 100755 --- a/scripts/build/extract-vllm-metal-version_test.sh +++ b/scripts/build/extract-vllm-metal-version_test.sh @@ -21,6 +21,17 @@ assert_version "0.26.0" ' local vllm_v="0.26.0"' assert_version "0.26.0" 'VLLM_VERSION="0.26.0"' assert_version "0.26.1" ' VLLM_VERSION = "0.26.1" # comment' +# .github/vllm-release-tag.commit form: a lone vLLM release tag. +assert_version "0.28.0" 'v0.28.0' +assert_version "0.28.0" '0.28.0' +assert_version "0.28.0" ' v0.28.0 ' + +# A whole upstream installer must still yield the inline pin, not a version-like +# fragment of some other line. +assert_version "0.26.0" 'set -e +vllm_wheel="vllm-1.2.3-cp312.whl" +VLLM_VERSION="0.26.0"' + if printf '%s\n' 'VLLM_VERSION="not-a-version"' | "$extractor"; then echo "malformed versions must be rejected" >&2 exit 1 @@ -30,3 +41,13 @@ if printf '%s\n' 'VLLM_VERSION="0.26.0"garbage' | "$extractor"; then echo "trailing assignment content must be rejected" >&2 exit 1 fi + +if printf '%s\n' 'not-a-tag' | "$extractor"; then + echo "malformed release tags must be rejected" >&2 + exit 1 +fi + +if printf '%s\n' 'vllm-1.2.3-cp312.whl' | "$extractor"; then + echo "a version embedded in a longer line must be rejected" >&2 + exit 1 +fi diff --git a/scripts/lib/extract-vllm-metal-version.sh b/scripts/lib/extract-vllm-metal-version.sh index a5e101c63c13..6019dca3e28e 100755 --- a/scripts/lib/extract-vllm-metal-version.sh +++ b/scripts/lib/extract-vllm-metal-version.sh @@ -1,5 +1,13 @@ #!/bin/bash set -euo pipefail -grep -m1 -oE '^[[:space:]]*(local[[:space:]]+)?(vllm_v|VLLM_VERSION)[[:space:]]*=[[:space:]]*"[0-9]+\.[0-9]+\.[0-9]+"[[:space:]]*(#.*)?$' \ +# Print the bare X.Y.Z vLLM version a vllm-metal release builds against, reading +# whichever form the release declares it in on stdin: +# +# * .github/vllm-release-tag.commit -- a lone "vX.Y.Z" vLLM release tag. This is +# the source of truth since vllm-metal 0.28, which also re-versioned the +# project so its own version tracks the vLLM version it targets. +# * install.sh -- releases predating that file pinned VLLM_VERSION="X.Y.Z" +# (earlier still: vllm_v="X.Y.Z") inline in their installer. +grep -m1 -oE '^[[:space:]]*((local[[:space:]]+)?(vllm_v|VLLM_VERSION)[[:space:]]*=[[:space:]]*"[0-9]+\.[0-9]+\.[0-9]+"[[:space:]]*(#.*)?|v?[0-9]+\.[0-9]+\.[0-9]+[[:space:]]*)$' \ | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' diff --git a/tests/e2e-backends/backend_test.go b/tests/e2e-backends/backend_test.go index 73b30f9909c2..9ebd044c825a 100644 --- a/tests/e2e-backends/backend_test.go +++ b/tests/e2e-backends/backend_test.go @@ -473,7 +473,7 @@ var _ = Describe("Backend container", Ordered, func() { Expect(res.GetPromptTokens()).To(BeNumerically(">", 128), "prompt is too short to span multiple prefill batches; this spec would not prove anything") } - Expect(strings.ToUpper(res.GetMessage())).To(ContainSubstring(needle), + Expect(strings.ToUpper(string(res.GetMessage()))).To(ContainSubstring(needle), "a long prompt lost information the model repeats correctly from a short one - "+ "batched prefill is corrupting state (check the backend's device architecture flags)") GinkgoWriter.Printf("LongPrefill: prompt_tokens=%d tokens=%d msg=%q\n",