From 3f5467b4ebdd0db36161f92db07e5c0d13d295a3 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Sun, 23 Aug 2026 08:53:57 +0200 Subject: [PATCH 1/4] chore: :arrow_up: Update CrispStrobe/CrispASR to `74bb374a8cc74284348d76a0a6e944180fbe6b07` (#11650) :arrow_up: Update CrispStrobe/CrispASR Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/go/crispasr/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go/crispasr/Makefile b/backend/go/crispasr/Makefile index 45602320f80d..9cf9137621e3 100644 --- a/backend/go/crispasr/Makefile +++ b/backend/go/crispasr/Makefile @@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1) # CrispASR version (release tag) CRISPASR_REPO?=https://github.com/CrispStrobe/CrispASR -CRISPASR_VERSION?=a153b09b37c90cd55cd9336fccbdf3ba7a289596 +CRISPASR_VERSION?=74bb374a8cc74284348d76a0a6e944180fbe6b07 SO_TARGET?=libgocrispasr.so CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF From 1205f37457a56ed044c33c9c36eec69cfcb1f5e3 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Sun, 23 Aug 2026 08:54:13 +0200 Subject: [PATCH 2/4] chore: :arrow_up: Update 0xShug0/audio.cpp to `4d383be1bff107e823ffc19120dcb6c78d493c0f` (#11666) :arrow_up: Update 0xShug0/audio.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/cpp/audio-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cpp/audio-cpp/Makefile b/backend/cpp/audio-cpp/Makefile index eff3582396bb..feb7ba0a01d8 100644 --- a/backend/cpp/audio-cpp/Makefile +++ b/backend/cpp/audio-cpp/Makefile @@ -9,7 +9,7 @@ # recipe is a make target (not a prepare.sh) so 'make purge && make' is a clean # rebuild and so the bump bot can see the pin. -AUDIO_CPP_VERSION?=43001a7e0f452d80f4588e613f13332940dd4d3a +AUDIO_CPP_VERSION?=4d383be1bff107e823ffc19120dcb6c78d493c0f AUDIO_CPP_REPO?=https://github.com/0xShug0/audio.cpp CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) From ffef5398660d3c0cbeec7716ae249ee9fc58bca9 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Sun, 23 Aug 2026 08:54:33 +0200 Subject: [PATCH 3/4] chore: :arrow_up: Update ggml-org/whisper.cpp to `233fe1fc9b48a09e361d3594520838ca266537fe` (#11648) :arrow_up: Update ggml-org/whisper.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/go/whisper/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go/whisper/Makefile b/backend/go/whisper/Makefile index 2831fff80499..214fba7776e1 100644 --- a/backend/go/whisper/Makefile +++ b/backend/go/whisper/Makefile @@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1) # whisper.cpp version WHISPER_REPO?=https://github.com/ggml-org/whisper.cpp -WHISPER_CPP_VERSION?=4834a2327d008ace3ec5a9ed00f51454bcabbc1c +WHISPER_CPP_VERSION?=233fe1fc9b48a09e361d3594520838ca266537fe SO_TARGET?=libgowhisper.so CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF From 8f56e4e042d3e67890648bea63749271237262ef Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Sun, 23 Aug 2026 08:55:13 +0200 Subject: [PATCH 4/4] fix(vram): persist remote probe metadata (#11487) * fix(vram): persist remote probe metadata The startup warmer repeated remote size and GGUF metadata probes after every restart because both caches lived only in memory. Store successful HTTP probes for 24 hours so frequent restarts reuse the prior results. Bound the cache, reject invalid records, and purge it when gallery data changes. Local model files continue to bypass persistence. Assisted-by: Codex:gpt-5 * fix(vram): check temporary file cleanup The lint gate rejects the unchecked cleanup call in the persistent cache writer. Assisted-by: Codex:gpt-5.6 [golangci-lint] * fix(vram): make persistent cache optional Remote metadata probes can transfer enough data that operators need control over disk reuse and startup warming. Gallery autoload now gates both behaviors, and the runtime setting applies changes immediately. Assisted-by: Codex:gpt-5 * fix(ui): expose gallery startup pre-warm The existing gallery autoload setting also gates the startup metadata warmer. Name both effects in Settings so operators can find the requested boot control. Assisted-by: Codex:gpt-5 --------- Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- core/application/startup.go | 19 +- core/cli/run.go | 2 + core/config/application_config.go | 6 + core/config/application_config_test.go | 10 + core/config/runtime_settings.go | 1 + core/config/runtime_settings_registry.go | 4 + core/config/runtime_settings_startup.go | 1 + core/gallery/gallery.go | 4 + core/http/endpoints/localai/settings.go | 9 + .../e2e/settings-backend-logging.spec.js | 28 ++ core/http/react-ui/src/pages/Settings.jsx | 5 +- docs/content/advanced/vram-management.md | 12 +- docs/content/features/runtime-settings.md | 3 +- docs/content/reference/cli-reference.md | 1 + pkg/vram/cache.go | 335 +++++++++++++++++- pkg/vram/cache_persistent_test.go | 228 ++++++++++++ 16 files changed, 656 insertions(+), 12 deletions(-) create mode 100644 pkg/vram/cache_persistent_test.go diff --git a/core/application/startup.go b/core/application/startup.go index 17144b7a512c..66a8131620da 100644 --- a/core/application/startup.go +++ b/core/application/startup.go @@ -446,13 +446,20 @@ func New(opts ...config.AppOption) (*Application, error) { // Wire gallery generation counter into VRAM caches so they invalidate // when gallery data refreshes instead of using a fixed TTL. vram.SetGalleryGenerationFunc(gallery.GalleryGeneration) + if options.AutoloadGalleries { + if options.VRAMPersistentCache { + // Remote GGUF probes can transfer substantial metadata. Keep successful + // results across restarts so the startup warmer does not repeat that work. + vram.ConfigurePersistentCache(filepath.Join(options.SystemState.Model.ModelsPath, "..", "cache", "vram"), 24*time.Hour) + } - // Fill those caches ahead of the first visitor. An estimate for an entry - // nobody has asked about yet costs a remote probe of its weight files, and - // the model gallery asks for one per row, so without this the first page - // spends seconds filling in its own sizes while somebody watches it. - // Non-blocking, and bounded: see DefaultEstimateWarmConfig. - gallery.WarmEstimateCache(options.Context, options.Galleries, options.SystemState, gallery.EstimateWarmConfigFromEnv()) + // Fill those caches ahead of the first visitor. An estimate for an entry + // nobody has asked about yet costs a remote probe of its weight files, and + // the model gallery asks for one per row, so without this the first page + // spends seconds filling in its own sizes while somebody watches it. + // Non-blocking, and bounded: see DefaultEstimateWarmConfig. + gallery.WarmEstimateCache(options.Context, options.Galleries, options.SystemState, gallery.EstimateWarmConfigFromEnv()) + } if options.ConfigFile != "" { if err := application.ModelConfigLoader().LoadMultipleModelConfigsSingleFile(options.ConfigFile, configLoaderOpts...); err != nil { diff --git a/core/cli/run.go b/core/cli/run.go index 4bcd80f2c4ce..6b9b3e4dc0b9 100644 --- a/core/cli/run.go +++ b/core/cli/run.go @@ -53,6 +53,7 @@ type RunCMD struct { BackendGalleries string `env:"LOCALAI_BACKEND_GALLERIES,BACKEND_GALLERIES" help:"JSON list of backend galleries" group:"backends" default:"${backends}"` Galleries string `env:"LOCALAI_GALLERIES,GALLERIES" help:"JSON list of galleries" group:"models" default:"${galleries}"` AutoloadGalleries bool `env:"LOCALAI_AUTOLOAD_GALLERIES,AUTOLOAD_GALLERIES" group:"models" default:"true"` + VRAMPersistentCache bool `env:"LOCALAI_VRAM_PERSISTENT_CACHE,VRAM_PERSISTENT_CACHE" group:"models" default:"true" help:"Persist successful remote VRAM metadata probes across restarts"` AutoloadBackendGalleries bool `env:"LOCALAI_AUTOLOAD_BACKEND_GALLERIES,AUTOLOAD_BACKEND_GALLERIES" group:"backends" default:"true"` BackendImagesReleaseTag string `env:"LOCALAI_BACKEND_IMAGES_RELEASE_TAG,BACKEND_IMAGES_RELEASE_TAG" help:"Fallback release tag for backend images" group:"backends" default:"latest"` BackendImagesBranchTag string `env:"LOCALAI_BACKEND_IMAGES_BRANCH_TAG,BACKEND_IMAGES_BRANCH_TAG" help:"Fallback branch tag for backend images" group:"backends" default:"master"` @@ -302,6 +303,7 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error { config.WithF16(r.F16), config.WithStringGalleries(r.Galleries), config.WithBackendGalleries(r.BackendGalleries), + config.WithVRAMPersistentCache(r.VRAMPersistentCache), config.WithCors(r.CORS), config.WithCorsAllowOrigins(r.CORSAllowOrigins), config.WithDisableCSRF(r.DisableCSRF), diff --git a/core/config/application_config.go b/core/config/application_config.go index e6668f8822cc..b83e8fe58792 100644 --- a/core/config/application_config.go +++ b/core/config/application_config.go @@ -125,6 +125,7 @@ type ApplicationConfig struct { ExternalGRPCBackends map[string]string AutoloadGalleries, AutoloadBackendGalleries bool + VRAMPersistentCache bool AutoUpgradeBackends bool PreferDevelopmentBackends bool @@ -284,6 +285,7 @@ func NewApplicationConfig(o ...AppOption) *ApplicationConfig { // toggle can still turn it off (a persisted false wins - see // loadRuntimeSettingsFromFile). EnableBackendLogging: true, + VRAMPersistentCache: true, ArtifactDownloadConcurrency: modelartifacts.DefaultDownloadConcurrency, AgentJobRetentionDays: 30, // Default: 30 days LRUEvictionMaxRetries: 30, // Default: 30 retries @@ -596,6 +598,10 @@ func WithAutoUpgradeBackends(v bool) AppOption { return func(o *ApplicationConfig) { o.AutoUpgradeBackends = v } } +func WithVRAMPersistentCache(v bool) AppOption { + return func(o *ApplicationConfig) { o.VRAMPersistentCache = v } +} + func WithRequireBackendIntegrity(v bool) AppOption { return func(o *ApplicationConfig) { o.RequireBackendIntegrity = v } } diff --git a/core/config/application_config_test.go b/core/config/application_config_test.go index 6860388e7efd..8d8a4b753cfb 100644 --- a/core/config/application_config_test.go +++ b/core/config/application_config_test.go @@ -1,6 +1,7 @@ package config import ( + "encoding/json" "time" . "github.com/onsi/ginkgo/v2" @@ -9,6 +10,15 @@ import ( var _ = Describe("ApplicationConfig RuntimeSettings Conversion", func() { Describe("ToRuntimeSettings", func() { + It("includes the persistent VRAM cache toggle", func() { + encoded, err := json.Marshal(NewApplicationConfig().ToRuntimeSettings()) + Expect(err).NotTo(HaveOccurred()) + + var settings map[string]any + Expect(json.Unmarshal(encoded, &settings)).To(Succeed()) + Expect(settings).To(HaveKeyWithValue("vram_persistent_cache", true)) + }) + It("should convert all fields correctly", func() { appConfig := &ApplicationConfig{ WatchDog: true, diff --git a/core/config/runtime_settings.go b/core/config/runtime_settings.go index 6e4381d8c97d..bd495104a478 100644 --- a/core/config/runtime_settings.go +++ b/core/config/runtime_settings.go @@ -59,6 +59,7 @@ type RuntimeSettings struct { BackendGalleries *[]Gallery `json:"backend_galleries,omitempty"` AutoloadGalleries *bool `json:"autoload_galleries,omitempty"` AutoloadBackendGalleries *bool `json:"autoload_backend_galleries,omitempty"` + VRAMPersistentCache *bool `json:"vram_persistent_cache,omitempty"` // API keys - No omitempty as we need to save empty arrays to clear keys ApiKeys *[]string `json:"api_keys"` diff --git a/core/config/runtime_settings_registry.go b/core/config/runtime_settings_registry.go index ce5774ab48c9..fda45e5bcdda 100644 --- a/core/config/runtime_settings_registry.go +++ b/core/config/runtime_settings_registry.go @@ -328,6 +328,10 @@ var runtimeSettingsFields = []fieldSpec{ func(s *RuntimeSettings) **bool { return &s.AutoloadBackendGalleries }, func(o *ApplicationConfig) bool { return o.AutoloadBackendGalleries }, func(o *ApplicationConfig, v bool) { o.AutoloadBackendGalleries = v }), + field("vram_persistent_cache", + func(s *RuntimeSettings) **bool { return &s.VRAMPersistentCache }, + func(o *ApplicationConfig) bool { return o.VRAMPersistentCache }, + func(o *ApplicationConfig, v bool) { o.VRAMPersistentCache = v }), // API keys: echoed for the UI, but the apply loops never touch them. // The settings endpoint and the file watcher own the env+runtime merge diff --git a/core/config/runtime_settings_startup.go b/core/config/runtime_settings_startup.go index 116d2a4f1478..9808c877b1a0 100644 --- a/core/config/runtime_settings_startup.go +++ b/core/config/runtime_settings_startup.go @@ -45,6 +45,7 @@ func DefaultRuntimeBaseline() *ApplicationConfig { o.BackendGalleries = mustGalleries(DefaultBackendGalleriesJSON) o.AutoloadGalleries = true o.AutoloadBackendGalleries = true + o.VRAMPersistentCache = true // core/cli/run.go injects WithMemoryReclaimer(enabled, threshold) // unconditionally, so the kong threshold default (0.95) reaches the // config even when the reclaimer flag is off - this overlay must match diff --git a/core/gallery/gallery.go b/core/gallery/gallery.go index 1d23dfbdb130..04752b3d1d2a 100644 --- a/core/gallery/gallery.go +++ b/core/gallery/gallery.go @@ -16,6 +16,7 @@ import ( "github.com/mudler/LocalAI/pkg/downloader" "github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/utils" + "github.com/mudler/LocalAI/pkg/vram" "github.com/mudler/LocalAI/pkg/xsync" "github.com/mudler/xlog" @@ -457,6 +458,9 @@ func triggerGalleryRefresh(galleries []config.Gallery, systemState *system.Syste galleryGeneration.Add(1) } availableModelsMu.Unlock() + if changed { + vram.InvalidatePersistentCache() + } }() } diff --git a/core/http/endpoints/localai/settings.go b/core/http/endpoints/localai/settings.go index 15c6f6d925e8..606fe49c5711 100644 --- a/core/http/endpoints/localai/settings.go +++ b/core/http/endpoints/localai/settings.go @@ -4,6 +4,7 @@ import ( "encoding/json" "io" "net/http" + "path/filepath" "time" "github.com/labstack/echo/v4" @@ -12,6 +13,7 @@ import ( "github.com/mudler/LocalAI/core/http/endpoints/openresponses" "github.com/mudler/LocalAI/core/p2p" "github.com/mudler/LocalAI/core/schema" + "github.com/mudler/LocalAI/pkg/vram" "github.com/mudler/LocalAI/pkg/vrambudget" "github.com/mudler/xlog" ) @@ -185,6 +187,13 @@ func UpdateSettingsEndpoint(app *application.Application) echo.HandlerFunc { // Apply settings using centralized method watchdogChanged := appConfig.ApplyRuntimeSettings(&settings) + if settings.VRAMPersistentCache != nil || settings.AutoloadGalleries != nil { + if appConfig.VRAMPersistentCache && appConfig.AutoloadGalleries { + vram.ConfigurePersistentCache(filepath.Join(appConfig.SystemState.Model.ModelsPath, "..", "cache", "vram"), 24*time.Hour) + } else { + vram.DisablePersistentCache() + } + } // Handle API keys specially (merge with startup keys) if settings.ApiKeys != nil { diff --git a/core/http/react-ui/e2e/settings-backend-logging.spec.js b/core/http/react-ui/e2e/settings-backend-logging.spec.js index 7b5459d8973e..8b1d032563a9 100644 --- a/core/http/react-ui/e2e/settings-backend-logging.spec.js +++ b/core/http/react-ui/e2e/settings-backend-logging.spec.js @@ -18,6 +18,34 @@ test.describe('Settings - Backend Logging', () => { await expect(input).toHaveValue('4') }) + test('persistent VRAM cache can be toggled', async ({ page }) => { + const row = page.locator('.form-row', { hasText: 'Persist remote VRAM estimates' }) + await expect(row).toBeVisible() + + const checkbox = row.locator('input[type="checkbox"]') + const wasChecked = await checkbox.isChecked() + await checkbox.locator('..').click() + if (wasChecked) { + await expect(checkbox).not.toBeChecked() + } else { + await expect(checkbox).toBeChecked() + } + }) + + test('gallery startup loading and pre-warming can be toggled together', async ({ page }) => { + const row = page.locator('.form-row', { hasText: 'Load and pre-warm galleries on boot' }) + await expect(row).toBeVisible() + + const checkbox = row.locator('input[type="checkbox"]') + const wasChecked = await checkbox.isChecked() + await checkbox.locator('..').click() + if (wasChecked) { + await expect(checkbox).not.toBeChecked() + } else { + await expect(checkbox).toBeChecked() + } + }) + test('backend logging toggle can be toggled', async ({ page }) => { // Find the checkbox associated with backend logging const section = page.locator('div', { has: page.locator('text=Enable Backend Logging') }) diff --git a/core/http/react-ui/src/pages/Settings.jsx b/core/http/react-ui/src/pages/Settings.jsx index c1a95ad29ab3..df8990ff197d 100644 --- a/core/http/react-ui/src/pages/Settings.jsx +++ b/core/http/react-ui/src/pages/Settings.jsx @@ -482,12 +482,15 @@ export default function Settings() { Galleries
- + update('autoload_galleries', v)} /> update('autoload_backend_galleries', v)} /> + + update('vram_persistent_cache', v)} /> +