[pull] master from mudler:master

backend/cpp/ds4/Makefile

@@ -1,10 +1,10 @@
 # ds4 backend Makefile.
 #
-# Upstream pin lives below as DS4_VERSION?=c1d4597a80e300b803dc642519718f2c999589da
+# Upstream pin lives below as DS4_VERSION?=8db89fe083ae4d17c9a2428ccd29803d3ae8f577
 # (.github/bump_deps.sh) can find and update it - matches the
 # llama-cpp / ik-llama-cpp / turboquant convention.
 
-DS4_VERSION?=c1d4597a80e300b803dc642519718f2c999589da
+DS4_VERSION?=8db89fe083ae4d17c9a2428ccd29803d3ae8f577
 DS4_REPO?=https://github.com/antirez/ds4
 
 CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))

backend/cpp/ik-llama-cpp/Makefile

@@ -1,5 +1,5 @@
 
-IK_LLAMA_VERSION?=7cff686d3732bfef5ce18bc4a6115fbceda29c14
+IK_LLAMA_VERSION?=15dddc60b3fc937a9e2a210359ecce392ccdf446
 LLAMA_REPO?=https://github.com/ikawrakow/ik_llama.cpp
 
 CMAKE_ARGS?=

core/gallery/cluster_env_test.go

@@ -0,0 +1,94 @@
+package gallery_test
+
+import (
+	"context"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/mudler/LocalAI/core/gallery"
+	"github.com/mudler/LocalAI/pkg/system"
+)
+
+// On a distributed controller the GPUs live on the workers, so a variant
+// picker sized against the controller tells admins a cluster of A100s can only
+// run the smallest CPU build.
+var _ = Describe("ClusterResolveEnv", func() {
+	gib := func(n uint64) uint64 { return n * 1024 * 1024 * 1024 }
+
+	// The controller as Argus actually runs it: no GPU at all.
+	var controller *system.SystemState
+
+	BeforeEach(func() {
+		controller = system.NewCapabilityState("default")
+	})
+
+	It("sizes models against the cluster reading rather than the controller", func() {
+		env := gallery.ClusterResolveEnv(context.Background(), controller, gib(80), []string{"nvidia-cuda-13"})
+
+		Expect(env.AvailableMemory).To(Equal(gib(80)))
+	})
+
+	It("accepts a CUDA backend that only the workers can run", func() {
+		env := gallery.ClusterResolveEnv(context.Background(), controller, gib(80), []string{"nvidia-cuda-13"})
+
+		Expect(env.BackendCompatible).ToNot(BeNil())
+		// A name carrying the cuda token is what the controller rejects today;
+		// a bare engine name like "vllm" passes on any host and would prove
+		// nothing about the union.
+		Expect(env.BackendCompatible("cuda-13-vllm")).To(BeTrue())
+		Expect(env.BackendCompatible("llama-cpp")).To(BeTrue())
+	})
+
+	// The union must stay a filter, not an open door: a Linux NVIDIA fleet
+	// still cannot run an Apple-only build.
+	It("still rejects a backend no node in the cluster can run", func() {
+		env := gallery.ClusterResolveEnv(context.Background(), controller, gib(80), []string{"nvidia-cuda-13"})
+
+		Expect(env.BackendCompatible("mlx")).To(BeFalse())
+	})
+
+	It("accepts a backend that any one node in a mixed fleet can run", func() {
+		env := gallery.ClusterResolveEnv(context.Background(), controller, gib(80), []string{"nvidia-cuda-13", "metal"})
+
+		Expect(env.BackendCompatible("mlx")).To(BeTrue())
+		Expect(env.BackendCompatible("cuda-13-vllm")).To(BeTrue())
+	})
+
+	// Ranking has to follow the hardware too, or a cluster of NVIDIA workers
+	// gets offered the GGUF build over the vLLM one it should prefer.
+	It("ranks engines by the workers' hardware, not the controller's", func() {
+		env := gallery.ClusterResolveEnv(context.Background(), controller, gib(80), []string{"nvidia-cuda-13"})
+
+		Expect(env.EnginePreference).To(Equal(system.NewCapabilityState("nvidia-cuda-13").EnginePreferenceTokens()))
+	})
+
+	// Every degradation path lands here, so it must be indistinguishable from
+	// the single-node behavior that shipped before any of this existed.
+	It("falls back to the host description when the cluster reports nothing", func() {
+		host := gallery.HostResolveEnv(context.Background(), controller)
+		env := gallery.ClusterResolveEnv(context.Background(), controller, 0, nil)
+
+		Expect(env.AvailableMemory).To(Equal(host.AvailableMemory))
+		Expect(env.EnginePreference).To(Equal(host.EnginePreference))
+		Expect(env.BackendCompatible("cuda-13-vllm")).To(Equal(host.BackendCompatible("cuda-13-vllm")))
+		Expect(env.BackendCompatible("mlx")).To(Equal(host.BackendCompatible("mlx")))
+	})
+
+	// A cluster that reports capabilities but no usable memory reading should
+	// still gain the hardware view; only the size question falls back.
+	It("keeps the host memory when only the memory reading is missing", func() {
+		host := gallery.HostResolveEnv(context.Background(), controller)
+		env := gallery.ClusterResolveEnv(context.Background(), controller, 0, []string{"nvidia-cuda-13"})
+
+		Expect(env.AvailableMemory).To(Equal(host.AvailableMemory))
+		Expect(env.BackendCompatible("cuda-13-vllm")).To(BeTrue())
+	})
+
+	It("keeps the probe wired so variant sizes are still measured", func() {
+		env := gallery.ClusterResolveEnv(context.Background(), controller, gib(80), []string{"nvidia-cuda-13"})
+
+		Expect(env.ProbeMemory).ToNot(BeNil())
+		Expect(env.ServingFeaturePreference).To(Equal(system.ServingFeaturePreferenceTokens()))
+	})
+})

core/gallery/models.go

@@ -938,3 +938,73 @@ func SafetyScanGalleryModel(galleryModel *GalleryModel) error {
 	}
 	return nil
 }
+
+// ClusterResolveEnv describes a CLUSTER to variant selection, where
+// HostResolveEnv describes one machine.
+//
+// It exists because a distributed controller is the wrong machine to ask. The
+// controller is typically a GPU-less pod while every model actually runs on a
+// worker, so a picker sized against it reports that a fleet of A100s can only
+// run the smallest CPU build, and auto-selection then installs exactly that.
+//
+// availableMemory is the largest single healthy node's budget, and capabilities
+// are the capability strings present in the cluster. Either may be empty: a
+// zero memory reading keeps the host's own figure and an empty capability list
+// keeps the host's own hardware verdict, so every degradation path lands back
+// on the single-node behavior rather than on a cluster described as having
+// nothing.
+func ClusterResolveEnv(ctx context.Context, systemState *system.SystemState, availableMemory uint64, capabilities []string) ResolveEnv {
+	env := HostResolveEnv(ctx, systemState)
+
+	if availableMemory > 0 {
+		env.AvailableMemory = availableMemory
+	}
+	if len(capabilities) == 0 {
+		return env
+	}
+
+	// One state pinned per capability, mirroring AvailableBackendsForCapabilities:
+	// the controller's own detection must not leak into a worker's verdict, and
+	// a forced capability on the controller image must not either.
+	nodeStates := make([]*system.SystemState, 0, len(capabilities))
+	for _, capability := range capabilities {
+		nodeStates = append(nodeStates, system.NewCapabilityState(capability,
+			system.WithBackendPath(systemState.Backend.BackendsPath)))
+	}
+
+	hostCompatible := env.BackendCompatible
+	// A union, because a variant only has to run SOMEWHERE. The controller
+	// stays in the union so a cluster whose workers all went offline still
+	// describes itself the way it did before distributed mode existed.
+	env.BackendCompatible = func(backend string) bool {
+		if hostCompatible != nil && hostCompatible(backend) {
+			return true
+		}
+		for _, nodeState := range nodeStates {
+			if nodeState.IsBackendCompatible(backend, "") {
+				return true
+			}
+		}
+		return false
+	}
+
+	// Ranking follows the same hardware as the filter. Left on the controller's
+	// tokens, an NVIDIA fleet would be offered the GGUF build over the vLLM one
+	// even though nothing filtered the vLLM build out.
+	seen := make(map[string]struct{})
+	preference := make([]string, 0, len(nodeStates))
+	for _, nodeState := range nodeStates {
+		for _, token := range nodeState.EnginePreferenceTokens() {
+			if _, dup := seen[token]; dup {
+				continue
+			}
+			seen[token] = struct{}{}
+			preference = append(preference, token)
+		}
+	}
+	if len(preference) > 0 {
+		env.EnginePreference = preference
+	}
+
+	return env
+}

core/http/endpoints/localai/backend.go

@@ -356,15 +356,15 @@ func (mgs *BackendEndpointService) UpgradeBackendEndpoint() echo.HandlerFunc {
 // local system state is the only thing worth filtering against.
 type ClusterCapabilityProvider func(ctx context.Context) ([]string, error)
 
-// resolveClusterCapabilities reads the capabilities present in the cluster,
+// ResolveClusterCapabilities reads the capabilities present in the cluster,
 // degrading to the local-only listing on error.
 //
 // Every capability-filtered discovery endpoint shares this: on a distributed
 // controller the GPUs live on the workers, so filtering against the local
 // (usually GPU-less) host hides GPU-only backends the cluster can actually
 // run. A registry hiccup must never blank the catalog, so a failure falls back
 // to the pre-existing local-only behavior rather than erroring the request.
-func resolveClusterCapabilities(ctx context.Context, provider ClusterCapabilityProvider) []string {
+func ResolveClusterCapabilities(ctx context.Context, provider ClusterCapabilityProvider) []string {
 	if provider == nil {
 		return nil
 	}
@@ -423,7 +423,7 @@ func installedInCluster(backend *gallery.GalleryBackend, clusterInstalled map[st
 // @Router /backends/available [get]
 func (mgs *BackendEndpointService) ListAvailableBackendsEndpoint(systemState *system.SystemState, clusterCapabilities ClusterCapabilityProvider, clusterInstalled ClusterInstalledProvider) echo.HandlerFunc {
 	return func(c echo.Context) error {
-		capabilities := resolveClusterCapabilities(c.Request().Context(), clusterCapabilities)
+		capabilities := ResolveClusterCapabilities(c.Request().Context(), clusterCapabilities)
 
 		backends, err := gallery.AvailableBackendsForCapabilities(mgs.galleries, systemState, capabilities)
 		if err != nil {

core/http/endpoints/localai/finetune.go

@@ -276,7 +276,7 @@ func DownloadExportedModelEndpoint(ftService *finetune.FineTuneService) echo.Han
 // ListFineTuneBackendsEndpoint returns installed backends tagged with "fine-tuning".
 func ListFineTuneBackendsEndpoint(appConfig *config.ApplicationConfig, clusterCapabilities ClusterCapabilityProvider, clusterInstalled ClusterInstalledProvider) echo.HandlerFunc {
 	return func(c echo.Context) error {
-		capabilities := resolveClusterCapabilities(c.Request().Context(), clusterCapabilities)
+		capabilities := ResolveClusterCapabilities(c.Request().Context(), clusterCapabilities)
 		installed := resolveClusterInstalled(c.Request().Context(), clusterInstalled)
 		backends, err := gallery.AvailableBackendsForCapabilities(appConfig.BackendGalleries, appConfig.SystemState, capabilities)
 		if err != nil {

core/http/endpoints/localai/quantization.go

@@ -195,7 +195,7 @@ func DownloadQuantizedModelEndpoint(qService *quantization.QuantizationService)
 // ListQuantizationBackendsEndpoint returns installed backends tagged with "quantization".
 func ListQuantizationBackendsEndpoint(appConfig *config.ApplicationConfig, clusterCapabilities ClusterCapabilityProvider, clusterInstalled ClusterInstalledProvider) echo.HandlerFunc {
 	return func(c echo.Context) error {
-		capabilities := resolveClusterCapabilities(c.Request().Context(), clusterCapabilities)
+		capabilities := ResolveClusterCapabilities(c.Request().Context(), clusterCapabilities)
 		installed := resolveClusterInstalled(c.Request().Context(), clusterInstalled)
 		backends, err := gallery.AvailableBackendsForCapabilities(appConfig.BackendGalleries, appConfig.SystemState, capabilities)
 		if err != nil {

core/http/react-ui/e2e/models-cluster-fit.spec.js

@@ -0,0 +1,120 @@
+import { test, expect } from "./coverage-fixtures.js";
+
+// On a distributed controller the models run on the workers, so every "will
+// this fit" answer on this page is about their hardware. The controller is
+// usually a GPU-less pod: sized against it, a cluster of A100s is told it can
+// only run the smallest CPU build.
+
+const GB = 1024 * 1024 * 1024;
+
+const MODELS = [
+  { name: "big-gpu-model", description: "Needs a real GPU", backend: "vllm", installed: false, tags: ["chat"] },
+];
+
+// 40GB: far past the controller's 8GB of RAM, comfortably inside one 80GB card.
+const ESTIMATES = {
+  "big-gpu-model": {
+    sizeBytes: 40 * GB,
+    sizeDisplay: "40.0 GB",
+    estimates: { 8192: { vramBytes: 40 * GB, vramDisplay: "40.0 GB" } },
+  },
+};
+
+// The controller as Argus actually runs it: 8GB of system RAM, no GPU.
+const CONTROLLER_ONLY = {
+  type: "ram",
+  available: true,
+  gpus: [],
+  aggregate: { total_memory: 8 * GB, used_memory: 2 * GB, free_memory: 6 * GB, gpu_count: 0 },
+};
+
+const WITH_CLUSTER = {
+  ...CONTROLLER_ONLY,
+  cluster: {
+    enabled: true,
+    node_id: "n-1",
+    node_name: "dgx-01",
+    total_memory: 80 * GB,
+    is_gpu: true,
+    node_count: 4,
+  },
+};
+
+async function mockModels(page, resources) {
+  await page.route("**/api/models*", (route) =>
+    route.fulfill({
+      contentType: "application/json",
+      body: JSON.stringify({
+        models: MODELS,
+        allBackends: ["vllm"],
+        allTags: ["chat"],
+        availableModels: MODELS.length,
+        installedModels: 3,
+        totalPages: 1,
+        currentPage: 1,
+      }),
+    }),
+  );
+  await page.route("**/api/models/estimate/*", (route) => {
+    const name = decodeURIComponent(new URL(route.request().url()).pathname.split("/").pop());
+    return route.fulfill({ contentType: "application/json", body: JSON.stringify(ESTIMATES[name] || {}) });
+  });
+  await page.route("**/api/resources", (route) =>
+    route.fulfill({ contentType: "application/json", body: JSON.stringify(resources) }),
+  );
+}
+
+const railItems = (page) => page.locator('[data-testid="discover-rail-item"]');
+const railItem = (page, name) => page.locator(`[data-entity="${name}"]`);
+const railReady = (page) => expect(railItems(page).first()).toBeVisible({ timeout: 20_000 });
+const PANE = '[data-testid="discover-pane"]';
+
+test.describe("Models gallery - cluster-aware fit", () => {
+  test("a model that only a worker can hold is not called too large", async ({ page }) => {
+    await mockModels(page, WITH_CLUSTER);
+    await page.goto("/app/models");
+
+    await railReady(page);
+
+    // The whole defect in one assertion: 40GB against a 4-node cluster whose
+    // largest card holds 80GB.
+    await expect(railItem(page, "big-gpu-model")).toContainText("fits", { timeout: 20_000 });
+    await expect(railItem(page, "big-gpu-model")).not.toContainText("too large");
+  });
+
+  test("the fit verdict names the node it belongs to", async ({ page }) => {
+    await mockModels(page, WITH_CLUSTER);
+    await page.goto("/app/models");
+
+    await railReady(page);
+    await railItem(page, "big-gpu-model").click();
+    // Wait for the detail itself: until it renders, the pane still holds the
+    // zero-state hero, which names the node for its own reasons.
+    await expect(page.locator(PANE).getByText("40.0 GB")).toBeVisible({ timeout: 20_000 });
+
+    // The headroom this model has is headroom SOMEWHERE, and the stat says
+    // where rather than leaving it to read as this machine's.
+    await expect(page.locator(PANE)).toContainText(/headroom on dgx-01/i);
+  });
+
+  test("the host summary describes the cluster, not the controller", async ({ page }) => {
+    await mockModels(page, WITH_CLUSTER);
+    await page.goto("/app/models");
+
+    await railReady(page);
+    // 80 GB is the cluster's best node; 8 GB is this pod's own RAM and must
+    // not be what the page advertises.
+    await expect(page.locator(".zero-pane__title")).toContainText("80 GB");
+    await expect(page.locator(".zero-pane__title")).not.toContainText("8.00 GB");
+  });
+
+  // Single-node behavior is the fallback every degradation path lands on, so
+  // it has to stay exactly as it was.
+  test("without a cluster the verdict is still the local host's", async ({ page }) => {
+    await mockModels(page, CONTROLLER_ONLY);
+    await page.goto("/app/models");
+
+    await railReady(page);
+    await expect(railItem(page, "big-gpu-model")).toContainText("too large", { timeout: 20_000 });
+  });
+});