diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 17e85e101a4a..cefd0cefdf76 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -29,10 +29,6 @@ updates: schedule: # Check for updates to GitHub Actions every weekday interval: "weekly" - - package-ecosystem: "pip" - directory: "/backend/python/bark" - schedule: - interval: "weekly" - package-ecosystem: "pip" directory: "/backend/python/common/template" schedule: @@ -55,30 +51,10 @@ updates: ignore: - dependency-name: "torch" - dependency-name: "transformers" - - package-ecosystem: "pip" - directory: "/backend/python/exllama" - schedule: - interval: "weekly" - - package-ecosystem: "pip" - directory: "/backend/python/exllama2" - schedule: - interval: "weekly" - - package-ecosystem: "pip" - directory: "/backend/python/mamba" - schedule: - interval: "weekly" - - package-ecosystem: "pip" - directory: "/backend/python/openvoice" - schedule: - interval: "weekly" - package-ecosystem: "pip" directory: "/backend/python/rerankers" schedule: interval: "weekly" - - package-ecosystem: "pip" - directory: "/backend/python/sentencetransformers" - schedule: - interval: "weekly" - package-ecosystem: "pip" directory: "/backend/python/transformers" schedule: @@ -86,44 +62,4 @@ updates: - package-ecosystem: "pip" directory: "/backend/python/vllm" schedule: - interval: "weekly" - - package-ecosystem: "pip" - directory: "/examples/chainlit" - schedule: - interval: "weekly" - - package-ecosystem: "pip" - directory: "/examples/functions" - schedule: - interval: "weekly" - - package-ecosystem: "pip" - directory: "/examples/langchain/langchainpy-localai-example" - schedule: - interval: "weekly" - - package-ecosystem: "pip" - directory: "/examples/langchain-chroma" - schedule: - interval: "weekly" - - package-ecosystem: "pip" - directory: "/examples/streamlit-bot" - schedule: - interval: "weekly" - - package-ecosystem: "docker" - directory: "/examples/k8sgpt" - schedule: - interval: "weekly" - - package-ecosystem: "docker" - directory: "/examples/kubernetes" - schedule: - interval: "weekly" - - package-ecosystem: "docker" - directory: "/examples/langchain" - schedule: - interval: "weekly" - - package-ecosystem: "gomod" - directory: "/examples/semantic-todo" - schedule: - interval: "weekly" - - package-ecosystem: "docker" - directory: "/examples/telegram-bot" - schedule: - interval: "weekly" + interval: "weekly" \ No newline at end of file diff --git a/.github/workflows/notify-releases.yaml b/.github/workflows/notify-releases.yaml index eab8ce54f8b4..8711c474520c 100644 --- a/.github/workflows/notify-releases.yaml +++ b/.github/workflows/notify-releases.yaml @@ -31,13 +31,14 @@ jobs: messages: [ { role: "system", - content: "Write a discord message with a bullet point summary of the release notes." + content: "Write a Discord message with a bullet point summary of the release notes. Keep the complete message under 1800 characters." }, { role: "user", content: $input } - ] + ], + max_tokens: 450 }') # Send the request to LocalAI API @@ -46,7 +47,7 @@ jobs: -d "$json_payload") # Extract the summary from the response - summary=$(echo $response | jq -r '.choices[0].message.content') + summary=$(printf '%s' "$response" | jq -er '.choices[0].message.content | strings | .[0:1800]') # Print the summary # -H "Authorization: Bearer $API_KEY" \ diff --git a/backend/cpp/audio-cpp/Makefile b/backend/cpp/audio-cpp/Makefile index feb7ba0a01d8..bee144fbcba4 100644 --- a/backend/cpp/audio-cpp/Makefile +++ b/backend/cpp/audio-cpp/Makefile @@ -9,7 +9,7 @@ # recipe is a make target (not a prepare.sh) so 'make purge && make' is a clean # rebuild and so the bump bot can see the pin. -AUDIO_CPP_VERSION?=4d383be1bff107e823ffc19120dcb6c78d493c0f +AUDIO_CPP_VERSION?=288a2712316470847a730e55db9ac9e5062a2b03 AUDIO_CPP_REPO?=https://github.com/0xShug0/audio.cpp CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) diff --git a/backend/go/crispasr/Makefile b/backend/go/crispasr/Makefile index 9cf9137621e3..b87b395a7daf 100644 --- a/backend/go/crispasr/Makefile +++ b/backend/go/crispasr/Makefile @@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1) # CrispASR version (release tag) CRISPASR_REPO?=https://github.com/CrispStrobe/CrispASR -CRISPASR_VERSION?=74bb374a8cc74284348d76a0a6e944180fbe6b07 +CRISPASR_VERSION?=ae4474dd8306384a0e697183d863dfc52e69a2fb SO_TARGET?=libgocrispasr.so CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF diff --git a/core/application/startup.go b/core/application/startup.go index 12217e42687f..abc2f4a17571 100644 --- a/core/application/startup.go +++ b/core/application/startup.go @@ -433,7 +433,7 @@ func New(opts ...config.AppOption) (*Application, error) { // the load above: the loader is empty until then, and a resync against an // empty loader silently reconciles nothing. if revisionStore != nil { - if err := modeladmin.ResyncModelConfigRevisions(options.Context, application.ModelConfigLoader(), revisionStore); err != nil { + if err := modeladmin.ResyncModelConfigRevisions(options.Context, application.ModelConfigLoader(), options, revisionStore); err != nil { xlog.Warn("Failed to resync model config revisions", "error", err) } } diff --git a/core/config/model_config_loader.go b/core/config/model_config_loader.go index 4c95a96657bc..2a062c39dfcf 100644 --- a/core/config/model_config_loader.go +++ b/core/config/model_config_loader.go @@ -168,6 +168,14 @@ func readModelConfigsFromFile(file string, opts ...ConfigLoaderOption) ([]*Model if err := yaml.Unmarshal(f, &configs); err == nil && len(configs) > 0 { for _, cc := range configs { cc.modelConfigFile = file + // Stamp before SetDefaults: the revision describes what is on disk. + // SetDefaults folds in the GGUF guess, hardware defaults and + // app-level options, none of which are persisted configuration, and + // the GGUF guess in particular depends on whether the model file + // parses at that moment. + if err := cc.StampPersistedConfigRevision(); err != nil { + return nil, fmt.Errorf("stamping config revision for %q: %w", cc.Name, err) + } cc.SetDefaults(opts...) cc.syncKnownUsecasesFromString() } @@ -182,6 +190,9 @@ func readModelConfigsFromFile(file string, opts ...ConfigLoaderOption) ([]*Model c.modelConfigFile = file c.syncKnownUsecasesFromString() + if err := c.StampPersistedConfigRevision(); err != nil { + return nil, fmt.Errorf("stamping config revision for %q: %w", c.Name, err) + } c.SetDefaults(opts...) return []*ModelConfig{c}, nil @@ -218,17 +229,18 @@ func (bcl *ModelConfigLoader) LoadModelConfigFileByName(modelName, modelPath str } } - cfg.SetDefaults(append(opts, ModelPath(modelPath))...) - - // Stamp the revision here, at the boundary between the persisted - // configuration and the request that is about to override parts of it. - // Everything downstream of this point (the request middleware) merges - // per-request prediction parameters into cfg, so a revision computed later - // would identify the request rather than the configuration. - if err := cfg.StampPersistedConfigRevision(); err != nil { - return nil, fmt.Errorf("stamping config revision for %q: %w", modelName, err) + // Stamp before SetDefaults, and only when this config did not come from + // disk already carrying one (a name with no config file on disk is + // synthesized above). Re-stamping a loaded config here would hash it after + // SetDefaults and reintroduce the dependency on the GGUF guess. + if cfg.PersistedConfigRevision() == "" { + if err := cfg.StampPersistedConfigRevision(); err != nil { + return nil, fmt.Errorf("stamping config revision for %q: %w", modelName, err) + } } + cfg.SetDefaults(append(opts, ModelPath(modelPath))...) + return cfg, nil } diff --git a/core/config/model_config_revision_stability_test.go b/core/config/model_config_revision_stability_test.go index 19a0d85901d9..b353f3d1440a 100644 --- a/core/config/model_config_revision_stability_test.go +++ b/core/config/model_config_revision_stability_test.go @@ -73,18 +73,75 @@ template: }) // The request pipeline reloads the config through LoadModelConfigFileByName, - // which applies SetDefaults a second time. That must not move the revision - // away from the one model administration publishes from the loader map. + // which applies SetDefaults a second time. The stamp is taken before those + // defaults, so both the stored config and the one a request resolves carry + // the same revision. It("survives the extra SetDefaults the request path applies", func() { loader := config.NewModelConfigLoader(dir) Expect(loader.LoadModelConfigsFromPath(dir, appConfig.ToConfigLoaderOptions()...)).To(Succeed()) stored, ok := loader.GetModelConfig("example") Expect(ok).To(BeTrue()) - adminRevision, err := config.ModelConfigRevision(&stored) - Expect(err).ToNot(HaveOccurred()) + Expect(stored.PersistedConfigRevision()).ToNot(BeEmpty()) requestCfg, err := loader.LoadModelConfigFileByNameDefaultOptions("example", appConfig) Expect(err).ToNot(HaveOccurred()) - Expect(requestCfg.PersistedConfigRevision()).To(Equal(adminRevision)) + Expect(requestCfg.PersistedConfigRevision()).To(Equal(stored.PersistedConfigRevision())) + }) +}) + +// The revision must describe the configuration as persisted, and nothing else. +// SetDefaults folds in values that are not persisted config: the GGUF guess +// (which reads the model file and can fail on slow or remote storage), the +// hardware defaults, and app-level options like threads. Hashing after that +// made the revision a function of whether a multi-gigabyte file happened to +// parse, so one unchanged YAML produced two different revisions depending on +// the moment, and the controller rejected every request carrying the other one. +var _ = Describe("Model config revision independence from runtime defaults", func() { + It("does not change when SetDefaults is applied", func() { + dir := GinkgoT().TempDir() + body := "backend: llama-cpp\ncontext_size: 50000\nknown_usecases:\n - chat\n" + + "mmproj: llama-cpp/mmproj/example/mmproj.gguf\nname: example\n" + + "parameters:\n model: llama-cpp/models/example/example.gguf\n" + Expect(os.WriteFile(filepath.Join(dir, "example.yaml"), []byte(body), 0o600)).To(Succeed()) + + appConfig := config.NewApplicationConfig() + appConfig.SystemState = &system.SystemState{Model: system.Model{ModelsPath: dir}} + loader := config.NewModelConfigLoader(dir) + Expect(loader.LoadModelConfigsFromPath(dir, appConfig.ToConfigLoaderOptions()...)).To(Succeed()) + + stored, ok := loader.GetModelConfig("example") + Expect(ok).To(BeTrue()) + before := stored.PersistedConfigRevision() + Expect(before).ToNot(BeEmpty()) + + // Applying defaults again is what the request path does. + stored.SetDefaults(appConfig.ToConfigLoaderOptions()...) + Expect(stored.PersistedConfigRevision()).To(Equal(before)) + + resolved, err := loader.LoadModelConfigFileByNameDefaultOptions("example", appConfig) + Expect(err).ToNot(HaveOccurred()) + Expect(resolved.PersistedConfigRevision()).To(Equal(before), + "the request path must carry the same revision as the stored config") + }) + + It("does not change when app-level defaults differ", func() { + dir := GinkgoT().TempDir() + Expect(os.WriteFile(filepath.Join(dir, "example.yaml"), + []byte("name: example\nbackend: llama-cpp\nparameters:\n model: m.gguf\n"), 0o600)).To(Succeed()) + + revWith := func(threads int, f16 bool) string { + appConfig := config.NewApplicationConfig() + appConfig.SystemState = &system.SystemState{Model: system.Model{ModelsPath: dir}} + appConfig.Threads = threads + appConfig.F16 = f16 + loader := config.NewModelConfigLoader(dir) + Expect(loader.LoadModelConfigsFromPath(dir, appConfig.ToConfigLoaderOptions()...)).To(Succeed()) + cfg, err := loader.LoadModelConfigFileByNameDefaultOptions("example", appConfig) + Expect(err).ToNot(HaveOccurred()) + return cfg.PersistedConfigRevision() + } + + Expect(revWith(8, false)).To(Equal(revWith(1, true)), + "an operator changing threads must not make every model unroutable") }) }) diff --git a/core/http/endpoints/localai/nodes_backends_list_test.go b/core/http/endpoints/localai/nodes_backends_list_test.go index c625e8e9510f..636ab58b818e 100644 --- a/core/http/endpoints/localai/nodes_backends_list_test.go +++ b/core/http/endpoints/localai/nodes_backends_list_test.go @@ -42,6 +42,8 @@ func (s *stubNodeCommandSender) StopBackend(_, _ string) error { return nil } func (s *stubNodeCommandSender) UnloadModelOnNode(_, _ string) error { return nil } +func (s *stubNodeCommandSender) PingNode(_ string) error { return nil } + var _ = Describe("ListBackendsOnNodeEndpoint", func() { var registry *nodes.NodeRegistry diff --git a/core/http/middleware/request_config_revision_test.go b/core/http/middleware/request_config_revision_test.go index 419ae8e040ac..dec7bc04db3c 100644 --- a/core/http/middleware/request_config_revision_test.go +++ b/core/http/middleware/request_config_revision_test.go @@ -115,8 +115,8 @@ var _ = Describe("Model config revision seen by inference requests", func() { Expect(admin.LoadModelConfigsFromPath(modelDir, appConfig.ToConfigLoaderOptions()...)).To(Succeed()) loaded, ok := admin.GetModelConfig("test-model") Expect(ok).To(BeTrue()) - adminRevision, err := config.ModelConfigRevision(&loaded) - Expect(err).ToNot(HaveOccurred()) + adminRevision := loaded.PersistedConfigRevision() + Expect(adminRevision).ToNot(BeEmpty()) Expect(revisionFor(`{"model":"test-model","temperature":0.7,"messages":[{"role":"user","content":"hi"}]}`)). To(Equal(adminRevision)) diff --git a/core/services/modeladmin/config.go b/core/services/modeladmin/config.go index 23de357aa9e2..515d014f7b53 100644 --- a/core/services/modeladmin/config.go +++ b/core/services/modeladmin/config.go @@ -177,13 +177,21 @@ func (s *ConfigService) patchConfig(ctx context.Context, name string, patch map[ if err := s.Loader.LoadModelConfigsFromPath(s.modelsPath(), s.AppConfig.ToConfigLoaderOptions()...); err != nil { return fmt.Errorf("reload configs: %w", err) } - loaded, ok := s.Loader.GetModelConfig(updated.Name) - if !ok { + if _, ok := s.Loader.GetModelConfig(updated.Name); !ok { return fmt.Errorf("reload configs: model %q missing", updated.Name) } - revision, err := config.ModelConfigRevision(&loaded) + // Resolve the revision the way an inference request does. Hashing the + // stored config instead publishes a value no request will ever carry, + // because SetDefaults runs again on the request path and is not + // idempotent for every model, and the edit would leave the model + // unroutable. + resolved, err := s.Loader.LoadModelConfigFileByNameDefaultOptions(updated.Name, s.AppConfig) if err != nil { - return fmt.Errorf("compute config revision: %w", err) + return fmt.Errorf("resolve config revision: %w", err) + } + revision := resolved.PersistedConfigRevision() + if revision == "" { + return fmt.Errorf("no config revision stamped for %q", updated.Name) } _ = s.Loader.Preload(s.modelsPath()) pending, err := s.applyRevision(ctx, name, updated.Name, revision, updated.IsDisabled()) diff --git a/core/services/modeladmin/revision_resync.go b/core/services/modeladmin/revision_resync.go index 1eceff3f9529..5d84831a2ab1 100644 --- a/core/services/modeladmin/revision_resync.go +++ b/core/services/modeladmin/revision_resync.go @@ -65,8 +65,8 @@ func NewRevisionStore(reader RevisionReader, lifecycle ModelRevisionLifecycle) R // A model with no stored revision is left alone. It has never been served, and // inventing controller state for it here would quarantine nothing and describe // a model that may never be requested. -func ResyncModelConfigRevisions(ctx context.Context, loader *config.ModelConfigLoader, store RevisionStore) error { - if loader == nil || store == nil { +func ResyncModelConfigRevisions(ctx context.Context, loader *config.ModelConfigLoader, appConfig *config.ApplicationConfig, store RevisionStore) error { + if loader == nil || store == nil || appConfig == nil { return nil } @@ -81,9 +81,19 @@ func ResyncModelConfigRevisions(ctx context.Context, loader *config.ModelConfigL var transitions []ModelRevisionTransition for _, cfg := range configs { - want, err := config.ModelConfigRevision(&cfg) + // Resolve the revision the way an inference request does, through the + // loader, rather than hashing the stored config directly. SetDefaults + // is applied again on that path and is not idempotent for every model + // (it re-runs the GGUF guess and hardware defaults), so hashing the + // stored config yields a value no request will ever carry, and + // publishing it would wedge the model this resync exists to unwedge. + resolved, err := loader.LoadModelConfigFileByNameDefaultOptions(cfg.Name, appConfig) if err != nil { - return fmt.Errorf("compute config revision for %q: %w", cfg.Name, err) + return fmt.Errorf("resolve config for %q: %w", cfg.Name, err) + } + want := resolved.PersistedConfigRevision() + if want == "" { + return fmt.Errorf("no config revision stamped for %q", cfg.Name) } stored, err := store.GetModelConfigRevision(ctx, cfg.Name) diff --git a/core/services/modeladmin/revision_resync_test.go b/core/services/modeladmin/revision_resync_test.go index 12772544245a..43aceb683bfd 100644 --- a/core/services/modeladmin/revision_resync_test.go +++ b/core/services/modeladmin/revision_resync_test.go @@ -55,12 +55,13 @@ var _ = Describe("ResyncModelConfigRevisions", func() { Expect(os.WriteFile(filepath.Join(dir, name+".yaml"), []byte(body), 0o600)).To(Succeed()) } + // revisionOf resolves the revision the way an inference request does, which + // is the value the resync must publish. revisionOf := func(name string) string { - cfg, ok := loader.GetModelConfig(name) - Expect(ok).To(BeTrue()) - rev, err := config.ModelConfigRevision(&cfg) + cfg, err := loader.LoadModelConfigFileByNameDefaultOptions(name, appConfig) Expect(err).ToNot(HaveOccurred()) - return rev + Expect(cfg.PersistedConfigRevision()).ToNot(BeEmpty()) + return cfg.PersistedConfigRevision() } BeforeEach(func() { @@ -80,7 +81,7 @@ var _ = Describe("ResyncModelConfigRevisions", func() { load() store.stored["drifted"] = "a-revision-from-an-earlier-build" - Expect(ResyncModelConfigRevisions(context.Background(), loader, store)).To(Succeed()) + Expect(ResyncModelConfigRevisions(context.Background(), loader, appConfig, store)).To(Succeed()) Expect(store.applied).To(HaveLen(1)) Expect(store.applied[0].ModelName).To(Equal("drifted")) @@ -92,7 +93,7 @@ var _ = Describe("ResyncModelConfigRevisions", func() { load() store.stored["agreed"] = revisionOf("agreed") - Expect(ResyncModelConfigRevisions(context.Background(), loader, store)).To(Succeed()) + Expect(ResyncModelConfigRevisions(context.Background(), loader, appConfig, store)).To(Succeed()) Expect(store.applied).To(BeEmpty(), "republishing an unchanged revision would quarantine live replicas for nothing") }) @@ -104,7 +105,7 @@ var _ = Describe("ResyncModelConfigRevisions", func() { write("never-served", "name: never-served\nbackend: llama-cpp\n") load() - Expect(ResyncModelConfigRevisions(context.Background(), loader, store)).To(Succeed()) + Expect(ResyncModelConfigRevisions(context.Background(), loader, appConfig, store)).To(Succeed()) Expect(store.applied).To(BeEmpty()) }) @@ -116,7 +117,7 @@ var _ = Describe("ResyncModelConfigRevisions", func() { store.stored["drifted"] = "stale" store.stored["agreed"] = revisionOf("agreed") - Expect(ResyncModelConfigRevisions(context.Background(), loader, store)).To(Succeed()) + Expect(ResyncModelConfigRevisions(context.Background(), loader, appConfig, store)).To(Succeed()) Expect(store.applied).To(HaveLen(1)) Expect(store.applied[0].ModelName).To(Equal("drifted")) @@ -128,7 +129,7 @@ var _ = Describe("ResyncModelConfigRevisions", func() { store.stored["drifted"] = "stale" store.applyEr = errors.New("database is down") - Expect(ResyncModelConfigRevisions(context.Background(), loader, store)).ToNot(Succeed()) + Expect(ResyncModelConfigRevisions(context.Background(), loader, appConfig, store)).ToNot(Succeed()) }) It("skips a model whose stored revision cannot be read rather than guessing", func() { @@ -136,7 +137,7 @@ var _ = Describe("ResyncModelConfigRevisions", func() { load() store.getErr = errors.New("connection reset") - Expect(ResyncModelConfigRevisions(context.Background(), loader, store)).ToNot(Succeed()) + Expect(ResyncModelConfigRevisions(context.Background(), loader, appConfig, store)).ToNot(Succeed()) Expect(store.applied).To(BeEmpty()) }) }) @@ -149,9 +150,11 @@ var _ = Describe("ResyncModelConfigRevisions with nothing loaded", func() { It("does not touch stored revisions when no configs are loaded", func() { dir := GinkgoT().TempDir() loader := config.NewModelConfigLoader(dir) + appConfig := config.NewApplicationConfig() + appConfig.SystemState = &system.SystemState{Model: system.Model{ModelsPath: dir}} store := &stubRevisionStore{stored: map[string]string{"served-before": "stale"}} - Expect(ResyncModelConfigRevisions(context.Background(), loader, store)).To(Succeed()) + Expect(ResyncModelConfigRevisions(context.Background(), loader, appConfig, store)).To(Succeed()) Expect(store.applied).To(BeEmpty()) }) }) diff --git a/core/services/modeladmin/state.go b/core/services/modeladmin/state.go index 5d87f6675409..4f84b5859e1b 100644 --- a/core/services/modeladmin/state.go +++ b/core/services/modeladmin/state.go @@ -7,7 +7,6 @@ import ( "gopkg.in/yaml.v3" - "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/pkg/utils" ) @@ -61,13 +60,21 @@ func (s *ConfigService) toggleState(ctx context.Context, name string, action Act if err := s.Loader.LoadModelConfigsFromPath(s.modelsPath(), s.AppConfig.ToConfigLoaderOptions()...); err != nil { return fmt.Errorf("reload configs: %w", err) } - loaded, ok := s.Loader.GetModelConfig(name) - if !ok { + if _, ok := s.Loader.GetModelConfig(name); !ok { return fmt.Errorf("reload configs: model %q missing", name) } - revision, err := config.ModelConfigRevision(&loaded) + // Resolve the revision the way an inference request does. Hashing the + // stored config instead publishes a value no request will ever carry, + // because SetDefaults runs again on the request path and is not + // idempotent for every model, and the edit would leave the model + // unroutable. + resolved, err := s.Loader.LoadModelConfigFileByNameDefaultOptions(name, s.AppConfig) if err != nil { - return fmt.Errorf("compute config revision: %w", err) + return fmt.Errorf("resolve config revision: %w", err) + } + revision := resolved.PersistedConfigRevision() + if revision == "" { + return fmt.Errorf("no config revision stamped for %q", name) } pending, err := s.applyRevision(ctx, name, name, revision, action == ActionDisable) if err != nil { diff --git a/core/services/nodes/managers_agent_node_test.go b/core/services/nodes/managers_agent_node_test.go new file mode 100644 index 000000000000..8ee95c083f8e --- /dev/null +++ b/core/services/nodes/managers_agent_node_test.go @@ -0,0 +1,84 @@ +package nodes + +import ( + "context" + "runtime" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/gorm" + + "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/testutil" +) + +// Agent workers do not subscribe to the backend.* subjects, so asking one to +// list its backends can only answer "no responders". ListBackends read that as +// a node that had gone away and marked it unhealthy; the node's next heartbeat +// marked it healthy again. Every poll of the backends view therefore flapped +// every agent node in the cluster, and while it was unhealthy the router would +// not schedule onto it. +var _ = Describe("Backend listing across mixed node types", func() { + var ( + db *gorm.DB + registry *NodeRegistry + mc *scriptedMessagingClient + mgr *DistributedBackendManager + ctx context.Context + ) + + BeforeEach(func() { + if runtime.GOOS == "darwin" { + Skip("testcontainers requires Docker, not available on macOS CI") + } + db = testutil.SetupTestDB() + var err error + registry, err = NewNodeRegistry(db) + Expect(err).ToNot(HaveOccurred()) + mc = newScriptedMessagingClient() + mgr = &DistributedBackendManager{ + local: stubLocalBackendManager{}, + adapter: NewRemoteUnloaderAdapter(nil, mc, 3*time.Minute, 15*time.Minute), + registry: registry, + } + ctx = context.Background() + }) + + register := func(name, nodeType string) *BackendNode { + node := &BackendNode{Name: name, NodeType: nodeType, Address: name + ":50051"} + Expect(registry.Register(ctx, node, true)).To(Succeed()) + fetched, err := registry.GetByName(ctx, name) + Expect(err).ToNot(HaveOccurred()) + Expect(fetched.Status).To(Equal(StatusHealthy)) + return fetched + } + + statusOf := func(id string) string { + n, err := registry.Get(ctx, id) + Expect(err).ToNot(HaveOccurred()) + return n.Status + } + + It("leaves an agent node healthy instead of flapping it", func() { + agent := register("agent-worker-1", NodeTypeAgent) + mc.scriptNoResponders(messaging.SubjectNodeBackendList(agent.ID)) + + _, err := mgr.ListBackends() + Expect(err).ToNot(HaveOccurred()) + + Expect(statusOf(agent.ID)).To(Equal(StatusHealthy), + "an agent node cannot answer backend.list and must not be judged on it") + }) + + It("still marks a backend node unhealthy when it does not answer", func() { + backendNode := register("worker-a", NodeTypeBackend) + mc.scriptNoResponders(messaging.SubjectNodeBackendList(backendNode.ID)) + + _, err := mgr.ListBackends() + Expect(err).ToNot(HaveOccurred()) + + Expect(statusOf(backendNode.ID)).To(Equal(StatusUnhealthy), + "a backend worker that does not answer is genuinely gone") + }) +}) diff --git a/core/services/nodes/managers_distributed.go b/core/services/nodes/managers_distributed.go index 127425b1a611..4132eca797db 100644 --- a/core/services/nodes/managers_distributed.go +++ b/core/services/nodes/managers_distributed.go @@ -331,8 +331,9 @@ func (d *DistributedBackendManager) DeleteBackendDetailed(ctx context.Context, n // populated from the first node seen so single-node-minded callers still work. // // Pending/offline/draining nodes are skipped because they aren't expected to -// answer NATS requests; unhealthy nodes are still queried — ErrNoResponders -// then marks them unhealthy and the loop continues. +// answer NATS requests, and so are non-backend workers, which do not subscribe +// to backend.list at all; unhealthy backend nodes are still queried — +// ErrNoResponders then marks them unhealthy and the loop continues. func (d *DistributedBackendManager) ListBackends() (gallery.SystemBackends, error) { result := make(gallery.SystemBackends) allNodes, err := d.registry.List(context.Background()) @@ -344,6 +345,14 @@ func (d *DistributedBackendManager) ListBackends() (gallery.SystemBackends, erro if node.Status == StatusPending || node.Status == StatusOffline || node.Status == StatusDraining { continue } + // Only backend workers subscribe to backend.list. Asking an agent + // worker can only answer "no responders", which the error handling + // below reads as a node that has gone away, so every poll of this view + // marked every agent node unhealthy and its next heartbeat marked it + // healthy again. The backend-op fan-out skips them for the same reason. + if node.NodeType != "" && node.NodeType != NodeTypeBackend { + continue + } reply, err := d.adapter.ListBackends(node.ID) if err != nil { if errors.Is(err, nats.ErrNoResponders) { diff --git a/gallery/index.yaml b/gallery/index.yaml index c9a240db2178..c980a79fdbba 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -1,4 +1,52 @@ --- +- name: "qwen3.8-27b-dflash2" + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2-GGUF + description: | + # Qwen3.8-27B + + > [!Note] + > This repository contains model weights and configuration files for the post-trained model in the Hugging Face Transformers format. + > + > These artifacts are compatible with Hugging Face Transformers, vLLM, SGLang, TokenSpeed, etc. + + > [!Tip] + > For users seeking managed, scalable inference without infrastructure maintenance, the official Qwen API service is provided by Qwen Cloud. + > In particular, **Qwen3.8-27B** will be available as a hosted version with more production features, e.g., 1M context length by default, official built-in tools. For more information, please refer to the Qwen3.8-27B Overview. The service is coming soon. Stay tuned for updates. + + Following the widespread community adoption of the Qwen3.5 and Qwen3.6 series, we are pleased to introduce Qwen3.8, the most capable generation in the Qwen open-model family to date. + + ... + license: "apache-2.0" + tags: + - llm + - gguf + icon: https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3.5/demo/CI_Demo/mathv-1327.jpg + overrides: + backend: llama-cpp + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + parameters: + min_p: 0 + model: llama-cpp/models/Qwen3.8-27B-DFlash2-Q4_K_M/Qwen3.8-27B-DFlash2-Q4_K_M.gguf + presence_penalty: 1.5 + repeat_penalty: 1 + temperature: 0.7 + top_k: 20 + top_p: 0.8 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/Qwen3.8-27B-DFlash2-Q4_K_M/Qwen3.8-27B-DFlash2-Q4_K_M.gguf + sha256: 18a380efc9b7ed8d88677fc895f5c11ae170653434ee378f7348f715c14d0594 + uri: https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2-GGUF/resolve/main/Qwen3.8-27B-DFlash2-Q4_K_M.gguf - name: "huihui-qwen3.8-27b-abliterated" url: "github:mudler/LocalAI/gallery/virtual.yaml@master" urls: @@ -549,6 +597,100 @@ - filename: llama-cpp/mmproj/ornith-1.5-9b/mmproj-BF16.gguf uri: huggingface://ornith-ai/Ornith-1.5-9B-GGUF/mmproj-Ornith-1.5-9B-BF16.gguf sha256: d65001a94c4b6852bc7a0e7c5cc92fe8506755bb270e54483fd5feec7ae39a19 +- &qwen3-8-27b-obliterated + name: "qwen3.8-27b-obliterated-q4" + variants: + - model: qwen3.8-27b-obliterated-q8 + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/Qwen/Qwen3.8-27B + - https://huggingface.co/OBLITERATUS/Qwen3.8-27B-OBLITERATED + description: | + Qwen3.8-27B OBLITERATED is an Apache-2.0 Qwen3.8 vision-language model + modified for refusal-removal and red-team research. It retains reasoning, + coding, tool use, image, and video capabilities, but its safety guardrails + have been removed. + + This default entry uses the Q4_K_M GGUF and BF16 vision projector. The + linked variant uses the higher-quality Q8_0 model. The publisher recommends + greedy decoding with a 1.15 repetition penalty. + license: "apache-2.0" + tags: + - llm + - gguf + - cpu + - gpu + - qwen + - reasoning + - thinking + - coding + - agent + - tools + - vision + - multimodal + - long-context + - uncensored + icon: https://qianwen-res.oss-cn-beijing.aliyuncs.com/logo_qwen.jpg + last_checked: "2026-08-24" + overrides: + backend: llama-cpp + context_size: 262144 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - vision + mmproj: llama-cpp/mmproj/qwen3.8-27b-obliterated/mmproj-model-bf16.gguf + options: + - use_jinja:true + parameters: + model: llama-cpp/models/qwen3.8-27b-obliterated/Qwen3.8-27B-OBLITERATED-Q4_K_M.gguf + repeat_penalty: 1.15 + temperature: 0 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/qwen3.8-27b-obliterated/Qwen3.8-27B-OBLITERATED-Q4_K_M.gguf + uri: huggingface://OBLITERATUS/Qwen3.8-27B-OBLITERATED/Qwen3.8-27B-OBLITERATED-Q4_K_M.gguf + sha256: c5e4fe705883e244a468c9e445c8d6ba37fd310b0113e25d2b8a7f2d6f1243e8 + - filename: llama-cpp/mmproj/qwen3.8-27b-obliterated/mmproj-model-bf16.gguf + uri: huggingface://OBLITERATUS/Qwen3.8-27B-OBLITERATED/mmproj-model-bf16.gguf + sha256: e484e3b7e907ed0e0644c0de56c3f5929c7ad5c9c6cc84d35a9d8dc08d461545 +- !!merge <<: *qwen3-8-27b-obliterated + name: "qwen3.8-27b-obliterated-q8" + variants: [] + description: | + Qwen3.8-27B OBLITERATED in the higher-quality Q8_0 GGUF format. This model + is modified for refusal-removal and red-team research, and its safety + guardrails have been removed. + overrides: + backend: llama-cpp + context_size: 262144 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - vision + mmproj: llama-cpp/mmproj/qwen3.8-27b-obliterated/mmproj-model-bf16.gguf + options: + - use_jinja:true + parameters: + model: llama-cpp/models/qwen3.8-27b-obliterated/Qwen3.8-27B-OBLITERATED-Q8_0.gguf + repeat_penalty: 1.15 + temperature: 0 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/qwen3.8-27b-obliterated/Qwen3.8-27B-OBLITERATED-Q8_0.gguf + uri: huggingface://OBLITERATUS/Qwen3.8-27B-OBLITERATED/Qwen3.8-27B-OBLITERATED-Q8_0.gguf + sha256: 4ed72a101dfa7f8fd642598368c4d334f1334cedc5254b71a06b5c4a542c59fc + - filename: llama-cpp/mmproj/qwen3.8-27b-obliterated/mmproj-model-bf16.gguf + uri: huggingface://OBLITERATUS/Qwen3.8-27B-OBLITERATED/mmproj-model-bf16.gguf + sha256: e484e3b7e907ed0e0644c0de56c3f5929c7ad5c9c6cc84d35a9d8dc08d461545 - &qwen3-8-27b name: "qwen3.8-27b-q4" variants: diff --git a/pkg/oci/image.go b/pkg/oci/image.go index 8e15466478aa..c44f7e5ebf91 100644 --- a/pkg/oci/image.go +++ b/pkg/oci/image.go @@ -80,27 +80,133 @@ var layerRetryBackoff = func(attempt int) time.Duration { return d } +// blobRangeOpener re-opens a layer blob at a byte offset. It returns the +// stream and the offset it actually starts at: the requested offset when the +// server honoured the Range request, or 0 when it ignored it and is sending +// the blob from the first byte again. +type blobRangeOpener func(ctx context.Context, offset int64) (io.ReadCloser, int64, error) + +// newBlobRangeOpener returns a blobRangeOpener that re-fetches the layer's +// blob from its registry with an HTTP Range request. Registries like quay.io +// redirect blob downloads to pre-signed S3/CDN URLs that expire after ~10 +// minutes; on a slow connection a multi-GiB layer cannot finish inside that +// window, so restarting from byte zero can never succeed while resuming from +// the current offset can (docker pull survives the same expiry this way). +// Each call goes back to the registry, so it obtains a fresh redirect URL and +// a fresh auth token. Returns nil when imageRef does not name a registry blob +// (e.g. local tarballs), which disables resuming. See issue #10577. +func newBlobRangeOpener(imageRef string, layer v1.Layer, auth *registrytypes.AuthConfig, base http.RoundTripper) blobRangeOpener { + ref, err := name.ParseReference(imageRef) + if err != nil { + return nil + } + digest, err := layer.Digest() + if err != nil || digest.Hex == "" { + return nil + } + repo := ref.Context() + if base == nil { + base = http.DefaultTransport + } + var authenticator authn.Authenticator + if auth != nil { + authenticator = staticAuth{auth} + } else if authenticator, err = authn.DefaultKeychain.Resolve(repo.Registry); err != nil { + authenticator = authn.Anonymous + } + blobURL := fmt.Sprintf("%s://%s/v2/%s/blobs/%s", repo.Registry.Scheme(), repo.RegistryStr(), repo.RepositoryStr(), digest.String()) + + return func(ctx context.Context, offset int64) (io.ReadCloser, int64, error) { + tr, err := transport.NewWithContext(ctx, repo.Registry, authenticator, base, []string{repo.Scope(transport.PullScope)}) + if err != nil { + return nil, 0, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, blobURL, nil) + if err != nil { + return nil, 0, err + } + if offset > 0 { + req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset)) + } + req.Header.Set("User-Agent", UserAgent()) + resp, err := (&http.Client{Transport: tr}).Do(req) + if err != nil { + return nil, 0, err + } + switch resp.StatusCode { + case http.StatusPartialContent: + return resp.Body, offset, nil + case http.StatusOK: + return resp.Body, 0, nil + default: + _ = resp.Body.Close() + return nil, 0, fmt.Errorf("unexpected status %d resuming blob %s", resp.StatusCode, digest.String()) + } + } +} + +// verifyLayerFile proves the assembled layer file matches the digest the +// registry advertised. A resumed download splices bytes from independent HTTP +// responses and bypasses the verified reader layer.Compressed() provides, so +// the whole file must be re-checked before it is trusted. +func verifyLayerFile(layer v1.Layer, f *os.File) error { + digest, err := layer.Digest() + if err != nil || digest.Hex == "" || digest.Algorithm != "sha256" { + return nil + } + if _, err := f.Seek(0, io.SeekStart); err != nil { + return err + } + got, _, err := v1.SHA256(f) + if err != nil { + return err + } + if got.Hex != digest.Hex { + return fmt.Errorf("resumed layer digest mismatch: got %s, want %s", got, digest) + } + return nil +} + // downloadLayerToFile streams a single compressed layer into dst, retrying on // transient network errors (unexpected EOF, connection reset, ...). Large // backend images (e.g. vLLM) are several GiB and a single dropped connection // mid-stream previously failed the whole install with "unexpected EOF" and no -// recovery. The registry transport already retries manifest fetches via -// defaultRetryPredicate (see GetImage/GetImageDigest); this extends the same -// behaviour to the layer data stream. See issue #10577. -func downloadLayerToFile(ctx context.Context, layer v1.Layer, dst *os.File, progress *progressWriter) error { +// recovery. When resume is non-nil, a retry keeps the bytes already on disk +// and continues from that offset instead of starting over: registries that +// serve blobs through expiring pre-signed URLs (quay.io + S3/Akamai) cut off +// every full-length transfer on slow connections, so restarting can never +// finish while resuming makes progress each round. The retry budget only +// counts attempts that made no forward progress, so a download that keeps +// advancing keeps going. See issue #10577. +func downloadLayerToFile(ctx context.Context, layer v1.Layer, dst *os.File, progress *progressWriter, resume blobRangeOpener) error { var lastErr error + // written tracks the valid bytes currently in dst across attempts, and + // bestWritten the furthest offset any attempt has reached: only beating + // it counts as forward progress for the retry budget, so a server that + // ignores Range requests and keeps dropping mid-stream still runs out + // of attempts instead of looping forever. + var written, bestWritten int64 + // resumed records whether any byte in dst came from a resumed raw blob + // fetch, which requires re-verifying the assembled file at the end. + resumed := false + + truncate := func() error { + if _, err := dst.Seek(0, io.SeekStart); err != nil { + return err + } + if err := dst.Truncate(0); err != nil { + return err + } + written = 0 + resumed = false + if progress != nil { + progress.written = 0 + } + return nil + } + for attempt := 0; attempt <= layerDownloadRetries; attempt++ { if attempt > 0 { - // Discard any partial data from the previous failed attempt. - if _, err := dst.Seek(0, io.SeekStart); err != nil { - return err - } - if err := dst.Truncate(0); err != nil { - return err - } - if progress != nil { - progress.written = 0 - } select { case <-ctx.Done(): return ctx.Err() @@ -108,19 +214,69 @@ func downloadLayerToFile(ctx context.Context, layer v1.Layer, dst *os.File, prog } } - var w io.Writer = dst - if progress != nil { - w = io.MultiWriter(dst, progress) + var reader io.ReadCloser + if attempt > 0 && resume != nil && written > 0 { + r, offset, rerr := resume(ctx, written) + switch { + case rerr != nil: + // Keep the partial bytes: opening the resume stream can + // fail transiently (token refresh, connection refused) + // and the next attempt can still continue from here. + lastErr = rerr + case offset != written: + // The server ignored the Range request and is sending + // the blob from the first byte: drop the partial data. + if err := truncate(); err != nil { + _ = r.Close() + return err + } + reader = r + resumed = true + default: + reader = r + resumed = true + } + } else { + // First attempt, or no way to resume: restart from scratch + // through the digest-verifying layer reader. + if err := truncate(); err != nil { + return err + } + reader, lastErr = layer.Compressed() } - var reader io.ReadCloser - reader, lastErr = layer.Compressed() - if lastErr == nil { - _, lastErr = xio.Copy(ctx, w, reader) + if reader != nil { + var w io.Writer = dst + if progress != nil { + w = io.MultiWriter(dst, progress) + } + var n int64 + n, lastErr = xio.Copy(ctx, w, reader) + written += n _ = reader.Close() + if written > bestWritten { + // Forward progress: don't charge this round against the + // retry budget, or slow links would still exhaust it. + bestWritten = written + attempt = 0 + } } + if lastErr == nil { - return nil + if !resumed { + return nil + } + verr := verifyLayerFile(layer, dst) + if verr == nil { + return nil + } + // The spliced file is corrupt: discard it and retry cleanly. + logs.Warn.Printf("discarding resumed layer download: %v", verr) + lastErr = verr + if err := truncate(); err != nil { + return err + } + continue } // Stop early on context cancellation or non-retryable errors. @@ -382,8 +538,11 @@ func DownloadOCIImageTar(ctx context.Context, img v1.Image, imageRef string, tar } } - // Download the compressed layer, retrying on transient network errors. - err = downloadLayerToFile(ctx, layer, file, progress) + // Download the compressed layer, retrying on transient network + // errors and resuming from the last byte received where possible. + // Anonymous/default-keychain credentials match what GetImage uses + // for every in-tree caller (they all pass a nil auth). + err = downloadLayerToFile(ctx, layer, file, progress, newBlobRangeOpener(imageRef, layer, nil, nil)) file.Close() if err != nil { return fmt.Errorf("failed to download layer %d: %v", i, err) diff --git a/pkg/oci/image_resume_internal_test.go b/pkg/oci/image_resume_internal_test.go new file mode 100644 index 000000000000..dec545d96d88 --- /dev/null +++ b/pkg/oci/image_resume_internal_test.go @@ -0,0 +1,151 @@ +package oci + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "time" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// droppingBlobRegistry emulates how quay.io serves layer blobs from S3/Akamai +// with a short-lived pre-signed URL: a full-blob GET on a slow connection is +// always cut off mid-transfer, so a client that restarts from byte zero can +// never complete the download. Only a client that resumes with a Range request +// (like docker pull does) receives the remaining bytes and can finish. +type droppingBlobRegistry struct { + inner http.Handler + + mu sync.Mutex + rangeRequests []int64 + fullRequests int +} + +// dropThreshold separates real layer blobs from small metadata blobs (image +// config), which are served untouched. +const dropThreshold = 1024 + +func (h *droppingBlobRegistry) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet || !strings.Contains(r.URL.Path, "/blobs/sha256:") { + h.inner.ServeHTTP(w, r) + return + } + + // Fetch the full blob from the inner registry (which does not speak + // Range) and apply the Range semantics here. + inner := r.Clone(r.Context()) + inner.Header.Del("Range") + rec := httptest.NewRecorder() + h.inner.ServeHTTP(rec, inner) + body := rec.Body.Bytes() + if rec.Code != http.StatusOK || len(body) <= dropThreshold { + for k, vv := range rec.Header() { + for _, v := range vv { + w.Header().Add(k, v) + } + } + w.WriteHeader(rec.Code) + _, _ = w.Write(body) + return + } + + if rh := r.Header.Get("Range"); rh != "" { + offset, err := strconv.ParseInt(strings.TrimSuffix(strings.TrimPrefix(rh, "bytes="), "-"), 10, 64) + if err != nil || offset < 0 || offset >= int64(len(body)) { + w.WriteHeader(http.StatusRequestedRangeNotSatisfiable) + return + } + h.mu.Lock() + h.rangeRequests = append(h.rangeRequests, offset) + h.mu.Unlock() + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", offset, len(body)-1, len(body))) + w.Header().Set("Content-Length", strconv.Itoa(len(body)-int(offset))) + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write(body[offset:]) + return + } + + h.mu.Lock() + h.fullRequests++ + h.mu.Unlock() + + // Announce the full size but deliver only half, then sever the + // connection, like a pre-signed URL expiring mid-download. + w.Header().Set("Content-Length", strconv.Itoa(len(body))) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(body[:len(body)/2]) + if f, ok := w.(http.Flusher); ok { + f.Flush() + } + panic(http.ErrAbortHandler) +} + +var _ = Describe("DownloadOCIImageTar resume", func() { + var ( + server *httptest.Server + reg *droppingBlobRegistry + tmpDir string + restoreWait func() + ) + + BeforeEach(func() { + reg = &droppingBlobRegistry{inner: registry.New()} + server = httptest.NewServer(reg) + + var err error + tmpDir, err = os.MkdirTemp("", "oci-resume-e2e-*") + Expect(err).NotTo(HaveOccurred()) + + prev := layerRetryBackoff + layerRetryBackoff = func(int) time.Duration { return 0 } + restoreWait = func() { layerRetryBackoff = prev } + }) + + AfterEach(func() { + restoreWait() + server.Close() + _ = os.RemoveAll(tmpDir) + }) + + It("completes the download by resuming interrupted layer transfers with Range requests", func() { + img, err := random.Image(4096, 1) + Expect(err).NotTo(HaveOccurred()) + + imageRef := strings.TrimPrefix(server.URL, "http://") + "/testrepo/backend:latest" + ref, err := name.ParseReference(imageRef) + Expect(err).NotTo(HaveOccurred()) + Expect(remote.Write(ref, img)).To(Succeed()) + + pulled, err := GetImage(imageRef, "", nil, nil) + Expect(err).NotTo(HaveOccurred()) + + tarPath := filepath.Join(tmpDir, "image.tar") + err = DownloadOCIImageTar(context.Background(), pulled, imageRef, tarPath, nil) + Expect(err).NotTo(HaveOccurred()) + + // The full-blob attempt was cut off, so success is only possible + // through at least one Range request picking up where it stopped. + reg.mu.Lock() + defer reg.mu.Unlock() + Expect(reg.rangeRequests).NotTo(BeEmpty()) + for _, off := range reg.rangeRequests { + Expect(off).To(BeNumerically(">", 0)) + } + + fi, err := os.Stat(tarPath) + Expect(err).NotTo(HaveOccurred()) + Expect(fi.Size()).To(BeNumerically(">", 0)) + }) +}) diff --git a/pkg/oci/layer_internal_test.go b/pkg/oci/layer_internal_test.go index faa8d5a45e6d..23ba4816057e 100644 --- a/pkg/oci/layer_internal_test.go +++ b/pkg/oci/layer_internal_test.go @@ -33,14 +33,18 @@ func (r *failingReader) Read(p []byte) (int, error) { // fakeLayer is a minimal v1.Layer whose Compressed() fails failUntil times with // err (after emitting a partial prefix) before finally returning data in full. +// The failing attempts emit prefix when set, or placeholder garbage otherwise. +// digest, when set, is what Digest() reports. type fakeLayer struct { data []byte + prefix []byte + digest v1.Hash failUntil int err error calls int } -func (f *fakeLayer) Digest() (v1.Hash, error) { return v1.Hash{}, nil } +func (f *fakeLayer) Digest() (v1.Hash, error) { return f.digest, nil } func (f *fakeLayer) DiffID() (v1.Hash, error) { return v1.Hash{}, nil } func (f *fakeLayer) Size() (int64, error) { return int64(len(f.data)), nil } func (f *fakeLayer) MediaType() (types.MediaType, error) { return types.DockerLayer, nil } @@ -51,7 +55,11 @@ func (f *fakeLayer) Uncompressed() (io.ReadCloser, error) { func (f *fakeLayer) Compressed() (io.ReadCloser, error) { f.calls++ if f.calls <= f.failUntil { - return io.NopCloser(&failingReader{prefix: []byte("partial-garbage"), err: f.err}), nil + prefix := f.prefix + if prefix == nil { + prefix = []byte("partial-garbage") + } + return io.NopCloser(&failingReader{prefix: prefix, err: f.err}), nil } return io.NopCloser(bytes.NewReader(f.data)), nil } @@ -86,7 +94,7 @@ var _ = Describe("downloadLayerToFile", func() { err: io.ErrUnexpectedEOF, } - err := downloadLayerToFile(context.Background(), layer, dst, nil) + err := downloadLayerToFile(context.Background(), layer, dst, nil, nil) Expect(err).NotTo(HaveOccurred()) Expect(layer.calls).To(Equal(3)) @@ -104,7 +112,7 @@ var _ = Describe("downloadLayerToFile", func() { err: errors.New("permission denied"), } - err := downloadLayerToFile(context.Background(), layer, dst, nil) + err := downloadLayerToFile(context.Background(), layer, dst, nil, nil) Expect(err).To(HaveOccurred()) Expect(layer.calls).To(Equal(1)) }) @@ -116,7 +124,7 @@ var _ = Describe("downloadLayerToFile", func() { err: io.ErrUnexpectedEOF, } - err := downloadLayerToFile(context.Background(), layer, dst, nil) + err := downloadLayerToFile(context.Background(), layer, dst, nil, nil) Expect(err).To(MatchError(io.ErrUnexpectedEOF)) Expect(layer.calls).To(Equal(layerDownloadRetries + 1)) }) diff --git a/pkg/oci/layer_resume_internal_test.go b/pkg/oci/layer_resume_internal_test.go new file mode 100644 index 000000000000..3f9e032d9ff0 --- /dev/null +++ b/pkg/oci/layer_resume_internal_test.go @@ -0,0 +1,191 @@ +package oci + +import ( + "bytes" + "context" + "io" + "os" + "time" + + v1 "github.com/google/go-containerregistry/pkg/v1" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// recordingOpener is a test blobRangeOpener that records the offsets it was +// asked to resume from and delegates the stream to open. +type recordingOpener struct { + offsets []int64 + open func(offset int64) (io.ReadCloser, int64, error) +} + +func (o *recordingOpener) opener() blobRangeOpener { + return func(_ context.Context, offset int64) (io.ReadCloser, int64, error) { + o.offsets = append(o.offsets, offset) + return o.open(offset) + } +} + +func sha256Of(data []byte) v1.Hash { + h, _, err := v1.SHA256(bytes.NewReader(data)) + Expect(err).NotTo(HaveOccurred()) + return h +} + +var _ = Describe("downloadLayerToFile resume", func() { + var ( + dst *os.File + data []byte + restoreWait func() + ) + + readDst := func() string { + got, err := os.ReadFile(dst.Name()) + Expect(err).NotTo(HaveOccurred()) + return string(got) + } + + BeforeEach(func() { + var err error + dst, err = os.CreateTemp("", "layer-resume-*.tar.gz") + Expect(err).NotTo(HaveOccurred()) + + data = []byte("0123456789abcdefghijklmnopqrstuvwxyzABCD") + + prev := layerRetryBackoff + layerRetryBackoff = func(int) time.Duration { return 0 } + restoreWait = func() { layerRetryBackoff = prev } + }) + + AfterEach(func() { + restoreWait() + _ = dst.Close() + _ = os.Remove(dst.Name()) + }) + + It("continues from the interruption offset instead of restarting", func() { + layer := &fakeLayer{ + data: data, + prefix: data[:15], + digest: sha256Of(data), + failUntil: 1, + err: io.ErrUnexpectedEOF, + } + rec := &recordingOpener{open: func(offset int64) (io.ReadCloser, int64, error) { + return io.NopCloser(bytes.NewReader(data[offset:])), offset, nil + }} + + err := downloadLayerToFile(context.Background(), layer, dst, nil, rec.opener()) + Expect(err).NotTo(HaveOccurred()) + Expect(readDst()).To(Equal(string(data))) + // The interrupted first attempt left 15 bytes; the resume must ask + // for exactly the rest, without a second full-stream attempt. + Expect(rec.offsets).To(Equal([]int64{15})) + Expect(layer.calls).To(Equal(1)) + }) + + It("restarts cleanly when the server ignores the Range request", func() { + layer := &fakeLayer{ + data: data, + prefix: data[:15], + digest: sha256Of(data), + failUntil: 1, + err: io.ErrUnexpectedEOF, + } + rec := &recordingOpener{open: func(int64) (io.ReadCloser, int64, error) { + // A 200 response: the whole blob from the first byte. + return io.NopCloser(bytes.NewReader(data)), 0, nil + }} + + err := downloadLayerToFile(context.Background(), layer, dst, nil, rec.opener()) + Expect(err).NotTo(HaveOccurred()) + // The partial bytes must have been discarded, not prepended. + Expect(readDst()).To(Equal(string(data))) + Expect(rec.offsets).To(HaveLen(1)) + Expect(layer.calls).To(Equal(1)) + }) + + It("discards a resumed download whose digest does not match", func() { + layer := &fakeLayer{ + data: data, + prefix: data[:15], + digest: sha256Of(data), + failUntil: 1, + err: io.ErrUnexpectedEOF, + } + rec := &recordingOpener{open: func(offset int64) (io.ReadCloser, int64, error) { + corrupt := bytes.Repeat([]byte("x"), len(data)-int(offset)) + return io.NopCloser(bytes.NewReader(corrupt)), offset, nil + }} + + err := downloadLayerToFile(context.Background(), layer, dst, nil, rec.opener()) + Expect(err).NotTo(HaveOccurred()) + // The spliced file failed verification, so the download must have + // started over through the verified layer reader and succeeded. + Expect(readDst()).To(Equal(string(data))) + Expect(rec.offsets).To(Equal([]int64{15})) + Expect(layer.calls).To(Equal(2)) + }) + + It("keeps retrying beyond the budget while each resume makes progress", func() { + const step = 5 + layer := &fakeLayer{ + data: data, + prefix: data[:step], + digest: sha256Of(data), + failUntil: 1, + err: io.ErrUnexpectedEOF, + } + rec := &recordingOpener{open: func(offset int64) (io.ReadCloser, int64, error) { + if offset+step >= int64(len(data)) { + return io.NopCloser(bytes.NewReader(data[offset:])), offset, nil + } + return io.NopCloser(&failingReader{prefix: data[offset : offset+step], err: io.ErrUnexpectedEOF}), offset, nil + }} + + err := downloadLayerToFile(context.Background(), layer, dst, nil, rec.opener()) + Expect(err).NotTo(HaveOccurred()) + Expect(readDst()).To(Equal(string(data))) + // 40 bytes delivered 5 at a time: 7 resumes, far more rounds than + // the retry budget allows for stalled attempts. + Expect(len(rec.offsets)).To(BeNumerically(">", layerDownloadRetries)) + }) + + It("gives up when resumes stop making progress", func(ctx SpecContext) { + layer := &fakeLayer{ + data: data, + prefix: data[:15], + digest: sha256Of(data), + failUntil: 1000, + err: io.ErrUnexpectedEOF, + } + rec := &recordingOpener{open: func(offset int64) (io.ReadCloser, int64, error) { + // Resume accepted but the connection dies before any byte. + return io.NopCloser(&failingReader{err: io.ErrUnexpectedEOF}), offset, nil + }} + + err := downloadLayerToFile(ctx, layer, dst, nil, rec.opener()) + Expect(err).To(MatchError(io.ErrUnexpectedEOF)) + Expect(len(rec.offsets)).To(Equal(layerDownloadRetries)) + }, NodeTimeout(10*time.Second)) + + It("terminates when the server ignores Range and keeps dropping mid-stream", func(ctx SpecContext) { + // Each round delivers some bytes from the start and dies: the file + // never gets further than before, so this must exhaust the budget + // rather than count the repeated partial bytes as progress. + layer := &fakeLayer{ + data: data, + prefix: data[:15], + digest: sha256Of(data), + failUntil: 1000, + err: io.ErrUnexpectedEOF, + } + rec := &recordingOpener{open: func(int64) (io.ReadCloser, int64, error) { + return io.NopCloser(&failingReader{prefix: data[:15], err: io.ErrUnexpectedEOF}), 0, nil + }} + + err := downloadLayerToFile(ctx, layer, dst, nil, rec.opener()) + Expect(err).To(MatchError(io.ErrUnexpectedEOF)) + Expect(len(rec.offsets)).To(Equal(layerDownloadRetries)) + }, NodeTimeout(10*time.Second)) +}) diff --git a/swagger/docs.go b/swagger/docs.go index b399f73e8ac3..16fa1de85ad2 100644 --- a/swagger/docs.go +++ b/swagger/docs.go @@ -4580,6 +4580,9 @@ const docTemplate = `{ "type": "object", "properties": { "config": {}, + "config_revision": { + "type": "string" + }, "details": { "type": "array", "items": { @@ -4595,6 +4598,9 @@ const docTemplate = `{ "message": { "type": "string" }, + "pending_cleanup": { + "type": "integer" + }, "success": { "type": "boolean" } @@ -4735,9 +4741,24 @@ const docTemplate = `{ "description": "e.g. \"llama-cpp\"; used by reconciler to replicate loads", "type": "string" }, + "cleanup_attempts": { + "type": "integer" + }, + "cleanup_error": { + "type": "string" + }, + "cleanup_next_retry_at": { + "type": "string" + }, + "config_revision": { + "type": "string" + }, "created_at": { "type": "string" }, + "effective_options_hash": { + "type": "string" + }, "id": { "type": "string" }, diff --git a/swagger/swagger.json b/swagger/swagger.json index b04ebca6d597..2be8aea429fc 100644 --- a/swagger/swagger.json +++ b/swagger/swagger.json @@ -4577,6 +4577,9 @@ "type": "object", "properties": { "config": {}, + "config_revision": { + "type": "string" + }, "details": { "type": "array", "items": { @@ -4592,6 +4595,9 @@ "message": { "type": "string" }, + "pending_cleanup": { + "type": "integer" + }, "success": { "type": "boolean" } @@ -4732,9 +4738,24 @@ "description": "e.g. \"llama-cpp\"; used by reconciler to replicate loads", "type": "string" }, + "cleanup_attempts": { + "type": "integer" + }, + "cleanup_error": { + "type": "string" + }, + "cleanup_next_retry_at": { + "type": "string" + }, + "config_revision": { + "type": "string" + }, "created_at": { "type": "string" }, + "effective_options_hash": { + "type": "string" + }, "id": { "type": "string" }, diff --git a/swagger/swagger.yaml b/swagger/swagger.yaml index f0e40bbe050f..58653861b6a4 100644 --- a/swagger/swagger.yaml +++ b/swagger/swagger.yaml @@ -414,6 +414,8 @@ definitions: localai.ModelResponse: properties: config: {} + config_revision: + type: string details: items: type: string @@ -424,6 +426,8 @@ definitions: type: string message: type: string + pending_cleanup: + type: integer success: type: boolean type: object @@ -518,8 +522,18 @@ definitions: backend_type: description: e.g. "llama-cpp"; used by reconciler to replicate loads type: string + cleanup_attempts: + type: integer + cleanup_error: + type: string + cleanup_next_retry_at: + type: string + config_revision: + type: string created_at: type: string + effective_options_hash: + type: string id: type: string in_flight: diff --git a/tests/e2e/distributed/distributed_full_flow_test.go b/tests/e2e/distributed/distributed_full_flow_test.go index 5eb9ff44281d..ad7f2669aaf0 100644 --- a/tests/e2e/distributed/distributed_full_flow_test.go +++ b/tests/e2e/distributed/distributed_full_flow_test.go @@ -260,6 +260,12 @@ var _ = Describe("Full Distributed Inference Flow", Label("Distributed"), func() data, _ := json.Marshal(reply) msg.Respond(data) }) + _, err := infra.NC.Conn().Subscribe("nodes.*.models.running", func(msg *nats.Msg) { + data, _ := json.Marshal(messaging.ModelsRunningReply{}) + _ = msg.Respond(data) + }) + Expect(err).NotTo(HaveOccurred()) + FlushNATS(infra.NC) return router } diff --git a/tests/e2e/distributed/router_tracking_test.go b/tests/e2e/distributed/router_tracking_test.go index 9691b31b02b7..75895a372d96 100644 --- a/tests/e2e/distributed/router_tracking_test.go +++ b/tests/e2e/distributed/router_tracking_test.go @@ -66,6 +66,12 @@ var _ = Describe("SmartRouter trackingKey", Label("Distributed"), func() { data, _ := json.Marshal(reply) msg.Respond(data) }) + _, err = infra.NC.Conn().Subscribe("nodes.*.models.running", func(msg *nats.Msg) { + data, _ := json.Marshal(messaging.ModelsRunningReply{}) + _ = msg.Respond(data) + }) + Expect(err).NotTo(HaveOccurred()) + FlushNATS(infra.NC) // Start a mock gRPC backend using the same helper as full flow tests llm := &trackingTestLLM{} diff --git a/website/data/stats.yaml b/website/data/stats.yaml index cee0db5e97e0..8bd8a0f9c08a 100644 --- a/website/data/stats.yaml +++ b/website/data/stats.yaml @@ -3,10 +3,10 @@ # The four GitHub fields are rewritten by .github/ci/refresh-site-counters.sh, # which runs weekly from .github/workflows/refresh-site-counters.yml. Editing # them by hand works but will be overwritten on the next run. -stars: 48067 -forks: 4320 -contributors: 225 -releases: 133 +stars: 48646 +forks: 4377 +contributors: 230 +releases: 136 # The GitHub API cannot answer for this one, so it is maintained by hand and # the refresh script carries it through untouched.