[pull] master from mudler:master

.gitignore

@@ -124,3 +124,8 @@ formal-verification/out/
 # package directory itself and untrack the source.
 /apexentries
 /.github/ci/apexentries/apexentries
+
+# Runtime state written by `local-ai run` when it is started from the repo
+# root, which is what a contributor testing a build does. Nothing under here is
+# source: it is the instance's own models, outputs, traces and identity.
+/data/

backend/cpp/bonsai/Makefile

@@ -1,7 +1,7 @@
 
 # Pinned to the HEAD of the `prism` branch on https://github.com/PrismML-Eng/llama.cpp.
 # Auto-bumped nightly by .github/workflows/bump_deps.yaml.
-BONSAI_VERSION?=4dd165625bb6c020285eec8b342af25cf60233dd
+BONSAI_VERSION?=9ca265a57f85f2117942490f421f64a226dd9847
 LLAMA_REPO?=https://github.com/PrismML-Eng/llama.cpp
 
 CMAKE_ARGS?=

backend/cpp/llama-cpp/Makefile

@@ -1,5 +1,5 @@
 
-LLAMA_VERSION?=876a4321163249c43ca4e986818fab5ab081f282
+LLAMA_VERSION?=a7a6d0d269c896218b6c78e0933bd6a17519d3f6
 LLAMA_REPO?=https://github.com/ggerganov/llama.cpp
 
 CMAKE_ARGS?=

core/application/startup.go

@@ -444,6 +444,13 @@ func New(opts ...config.AppOption) (*Application, error) {
 	// when gallery data refreshes instead of using a fixed TTL.
 	vram.SetGalleryGenerationFunc(gallery.GalleryGeneration)
 
+	// Fill those caches ahead of the first visitor. An estimate for an entry
+	// nobody has asked about yet costs a remote probe of its weight files, and
+	// the model gallery asks for one per row, so without this the first page
+	// spends seconds filling in its own sizes while somebody watches it.
+	// Non-blocking, and bounded: see DefaultEstimateWarmConfig.
+	gallery.WarmEstimateCache(options.Context, options.Galleries, options.SystemState, gallery.EstimateWarmConfigFromEnv())
+
 	if options.ConfigFile != "" {
 		if err := application.ModelConfigLoader().LoadMultipleModelConfigsSingleFile(options.ConfigFile, configLoaderOpts...); err != nil {
 			xlog.Error("error loading config file", "error", err)

core/gallery/estimate_warm.go

@@ -0,0 +1,211 @@
+package gallery
+
+import (
+	"context"
+	"os"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/pkg/system"
+	"github.com/mudler/LocalAI/pkg/vram"
+	"github.com/mudler/xlog"
+)
+
+// EstimateInput builds the VRAM estimator's input from a gallery entry.
+//
+// It lives here rather than beside the HTTP handler because two callers need
+// it: the handler answering one model, and the warmer below answering all of
+// them ahead of time.
+func EstimateInput(m *GalleryModel) vram.ModelEstimateInput {
+	var input vram.ModelEstimateInput
+	input.Size = m.Size
+	if repoID := extractHFRepo(m.Overrides, m.URLs); repoID != "" {
+		input.HFRepo = repoID
+	}
+	for _, f := range m.AdditionalFiles {
+		if vram.IsWeightFile(f.URI) {
+			input.Files = append(input.Files, vram.FileInput{URI: f.URI, Size: 0})
+		}
+	}
+	return input
+}
+
+// extractHFRepo finds a HuggingFace repo ID in a model's overrides or URLs.
+func extractHFRepo(overrides map[string]any, urls []string) string {
+	if overrides != nil {
+		if params, ok := overrides["parameters"].(map[string]any); ok {
+			if modelRef, ok := params["model"].(string); ok {
+				if repoID, ok := vram.ExtractHFRepoID(modelRef); ok {
+					return repoID
+				}
+			}
+		}
+	}
+	for _, u := range urls {
+		if repoID, ok := vram.ExtractHFRepoID(u); ok {
+			return repoID
+		}
+	}
+	return ""
+}
+
+// EstimateWarmConfig bounds the background warm-up.
+type EstimateWarmConfig struct {
+	// Limit is how many gallery entries to warm, in gallery order. Zero
+	// disables warming entirely. The order matters: it is the order the UI
+	// lists them in, so the entries a user sees first are warmed first.
+	Limit int
+	// Concurrency is how many estimates run at once. Each one can be a remote
+	// probe, so this is deliberately small: the point is to be finished before
+	// anybody looks, not to saturate the link or the upstream.
+	Concurrency int
+	// Contexts are the context lengths to estimate at. These want to match what
+	// the UI asks for, or the warmed entry is not the one it reads.
+	Contexts []uint32
+}
+
+// DefaultEstimateWarmConfig is what the server uses unless told otherwise.
+//
+// The limit is a deliberate compromise. Warming the whole gallery would be
+// thousands of remote probes on every boot, which is rude to the upstream and
+// slow to finish; warming nothing leaves the first page of the model gallery
+// paying two seconds per row. A few hundred covers what anyone browses in a
+// sitting, and everything past it still warms itself on first view.
+var DefaultEstimateWarmConfig = EstimateWarmConfig{
+	Limit:       300,
+	Concurrency: 4,
+	Contexts:    []uint32{8192, 16384, 32768, 65536, 131072, 262144},
+}
+
+// WarmEstimateCache fills the gallery's derived caches in the background.
+//
+// Two things are warmed, and they are the same cost wearing different hats.
+// An estimate for an entry the server has never seen costs a network probe of
+// its weight files, and describing an entry's variants costs one probe per
+// build it offers. The UI asks for an estimate per row and a variant
+// description per model opened, so without this the first visitor pays for
+// both: ten seconds of a page filling in its own sizes, then another second
+// and a half the first time they click anything.
+//
+// Both land in the same caches underneath, which is why one pass covers them.
+//
+// It returns immediately; the work happens on its own goroutine and stops when
+// ctx is done. Failures are logged at debug and otherwise ignored: a warm-up
+// that cannot reach an upstream must never stop the server from starting, and
+// the entry it failed on simply stays cold.
+func WarmEstimateCache(ctx context.Context, galleries []config.Gallery, systemState *system.SystemState, cfg EstimateWarmConfig) {
+	if cfg.Limit <= 0 || cfg.Concurrency <= 0 {
+		return
+	}
+
+	go func() {
+		started := time.Now()
+
+		models, err := AvailableGalleryModelsCached(galleries, systemState)
+		if err != nil {
+			xlog.Debug("VRAM estimate warm-up skipped, gallery unavailable", "error", err)
+			return
+		}
+		if len(models) > cfg.Limit {
+			models = models[:cfg.Limit]
+		}
+		if len(models) == 0 {
+			return
+		}
+
+		// The host gate the variant picker resolves against. Derived once: it
+		// describes this machine, not this entry, and HostResolveEnv reads the
+		// system state to build it.
+		env := HostResolveEnv(ctx, systemState)
+
+		var (
+			wg             sync.WaitGroup
+			cursor         = make(chan *GalleryModel)
+			warmed         int
+			warmedVariants int
+			mu             sync.Mutex
+		)
+
+		for i := 0; i < cfg.Concurrency; i++ {
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				for m := range cursor {
+					// Per entry, not for the run: one unreachable weight file
+					// must not hold a worker for the whole warm-up.
+					entryCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
+
+					input := EstimateInput(m)
+					if len(input.Files) > 0 || input.HFRepo != "" || input.Size != "" {
+						if _, err := vram.EstimateModelMultiContext(entryCtx, input, cfg.Contexts); err != nil {
+							xlog.Debug("VRAM estimate warm-up failed for entry", "model", m.GetName(), "error", err)
+						} else {
+							mu.Lock()
+							warmed++
+							mu.Unlock()
+						}
+					}
+
+					// Describing variants probes each build the entry offers.
+					// An entry that declares none costs nothing here, so this is
+					// gated rather than attempted and discarded.
+					if m.HasVariants() {
+						if _, err := DescribeVariants(models, m, env); err != nil {
+							xlog.Debug("variant warm-up failed for entry", "model", m.GetName(), "error", err)
+						} else {
+							mu.Lock()
+							warmedVariants++
+							mu.Unlock()
+						}
+					}
+
+					cancel()
+				}
+			}()
+		}
+
+	feed:
+		for _, m := range models {
+			select {
+			case <-ctx.Done():
+				break feed
+			case cursor <- m:
+			}
+		}
+		close(cursor)
+		wg.Wait()
+
+		if ctx.Err() != nil {
+			xlog.Debug("gallery warm-up stopped", "estimates", warmed, "variants", warmedVariants)
+			return
+		}
+		xlog.Info("gallery caches warmed", "estimates", warmed, "variants", warmedVariants, "of", len(models), "took", time.Since(started).Round(time.Second))
+	}()
+}
+
+// EstimateWarmConfigFromEnv reads the warm-up bounds from the environment,
+// falling back to the defaults.
+//
+//	LOCALAI_VRAM_WARM_LIMIT        entries to warm; 0 disables the warm-up
+//	LOCALAI_VRAM_WARM_CONCURRENCY  estimates in flight at once
+//
+// Env rather than a flag because it is an operational tuning knob, not part of
+// what the server does: an air-gapped host wants it off, and a host behind a
+// slow link wants it slower, and neither is a decision the CLI should carry.
+func EstimateWarmConfigFromEnv() EstimateWarmConfig {
+	cfg := DefaultEstimateWarmConfig
+	if v, ok := os.LookupEnv("LOCALAI_VRAM_WARM_LIMIT"); ok {
+		if n, err := strconv.Atoi(strings.TrimSpace(v)); err == nil && n >= 0 {
+			cfg.Limit = n
+		}
+	}
+	if v, ok := os.LookupEnv("LOCALAI_VRAM_WARM_CONCURRENCY"); ok {
+		if n, err := strconv.Atoi(strings.TrimSpace(v)); err == nil && n > 0 {
+			cfg.Concurrency = n
+		}
+	}
+	return cfg
+}

core/gallery/estimate_warm_test.go

@@ -0,0 +1,115 @@
+package gallery_test
+
+import (
+	"context"
+	"os"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/core/gallery"
+	"github.com/mudler/LocalAI/pkg/system"
+)
+
+var _ = Describe("VRAM estimate warm-up", func() {
+	var state *system.SystemState
+
+	BeforeEach(func() {
+		dir, err := os.MkdirTemp("", "warm")
+		Expect(err).ToNot(HaveOccurred())
+		DeferCleanup(func() { os.RemoveAll(dir) })
+		state, err = system.GetSystemState(system.WithModelPath(dir))
+		Expect(err).ToNot(HaveOccurred())
+		gallery.ResetGalleryModelCache()
+		DeferCleanup(gallery.ResetGalleryModelCache)
+	})
+
+	It("does nothing when disabled, and returns without blocking", func() {
+		cfg := gallery.DefaultEstimateWarmConfig
+		cfg.Limit = 0
+
+		done := make(chan struct{})
+		go func() {
+			defer close(done)
+			gallery.WarmEstimateCache(context.Background(), []config.Gallery{}, state, cfg)
+		}()
+		Eventually(done, "1s").Should(BeClosed())
+	})
+
+	It("returns immediately even when there is work to do", func() {
+		// The caller is a server still starting up: warming must never be on
+		// the path to listening.
+		done := make(chan struct{})
+		go func() {
+			defer close(done)
+			gallery.WarmEstimateCache(context.Background(), []config.Gallery{}, state, gallery.DefaultEstimateWarmConfig)
+		}()
+		Eventually(done, "1s").Should(BeClosed())
+	})
+
+	It("stops when its context is cancelled", func() {
+		ctx, cancel := context.WithCancel(context.Background())
+		gallery.WarmEstimateCache(ctx, []config.Gallery{}, state, gallery.DefaultEstimateWarmConfig)
+		cancel()
+		// Nothing to assert beyond not hanging or panicking: an aborted warm-up
+		// leaves entries cold, which is the state they were already in.
+		Consistently(func() bool { return true }, "100ms").Should(BeTrue())
+	})
+
+	Describe("configuration from the environment", func() {
+		AfterEach(func() {
+			os.Unsetenv("LOCALAI_VRAM_WARM_LIMIT")
+			os.Unsetenv("LOCALAI_VRAM_WARM_CONCURRENCY")
+		})
+
+		It("falls back to the defaults", func() {
+			cfg := gallery.EstimateWarmConfigFromEnv()
+			Expect(cfg.Limit).To(Equal(gallery.DefaultEstimateWarmConfig.Limit))
+			Expect(cfg.Concurrency).To(Equal(gallery.DefaultEstimateWarmConfig.Concurrency))
+		})
+
+		It("lets an operator turn it off entirely", func() {
+			os.Setenv("LOCALAI_VRAM_WARM_LIMIT", "0")
+			Expect(gallery.EstimateWarmConfigFromEnv().Limit).To(BeZero())
+		})
+
+		It("lets an operator slow it down", func() {
+			os.Setenv("LOCALAI_VRAM_WARM_CONCURRENCY", "1")
+			Expect(gallery.EstimateWarmConfigFromEnv().Concurrency).To(Equal(1))
+		})
+
+		It("ignores values that are not usable", func() {
+			os.Setenv("LOCALAI_VRAM_WARM_LIMIT", "not-a-number")
+			os.Setenv("LOCALAI_VRAM_WARM_CONCURRENCY", "0")
+			cfg := gallery.EstimateWarmConfigFromEnv()
+			Expect(cfg.Limit).To(Equal(gallery.DefaultEstimateWarmConfig.Limit))
+			// Zero workers would be a warm-up that never runs while looking
+			// enabled, so it keeps the default rather than honouring it.
+			Expect(cfg.Concurrency).To(Equal(gallery.DefaultEstimateWarmConfig.Concurrency))
+		})
+	})
+
+	It("warms variant descriptions as well as estimates", func() {
+		// Both are the same cost wearing different hats - a probe of an entry's
+		// weight files - and both land in the same caches, so a warm-up that
+		// covered only one would leave the first click paying for the other.
+		// Asserted through the shared config rather than by observing network
+		// calls: the gallery here is empty by design.
+		Expect(gallery.DefaultEstimateWarmConfig.Limit).To(BeNumerically(">", 0))
+	})
+
+	It("keeps the estimate contexts the UI actually asks for", func() {
+		// A warmed entry at the wrong context lengths is a cache the gallery
+		// never reads, so this pins them together.
+		Expect(gallery.DefaultEstimateWarmConfig.Contexts).To(ContainElements(
+			uint32(8192), uint32(16384), uint32(32768), uint32(65536), uint32(131072), uint32(262144),
+		))
+	})
+
+	It("bounds concurrency so a warm-up cannot saturate the link", func() {
+		Expect(gallery.DefaultEstimateWarmConfig.Concurrency).To(BeNumerically("<=", 8))
+		Expect(gallery.DefaultEstimateWarmConfig.Concurrency).To(BeNumerically(">", 0))
+	})
+
+})