diff --git a/backend/cpp/ds4/Makefile b/backend/cpp/ds4/Makefile index bdf126122160..5e73c1328bbb 100644 --- a/backend/cpp/ds4/Makefile +++ b/backend/cpp/ds4/Makefile @@ -1,10 +1,10 @@ # ds4 backend Makefile. # -# Upstream pin lives below as DS4_VERSION?=84cc882352757baf628a1776badf7cc54d584e28 +# Upstream pin lives below as DS4_VERSION?=c1d4597a80e300b803dc642519718f2c999589da # (.github/bump_deps.sh) can find and update it - matches the # llama-cpp / ik-llama-cpp / turboquant convention. -DS4_VERSION?=84cc882352757baf628a1776badf7cc54d584e28 +DS4_VERSION?=c1d4597a80e300b803dc642519718f2c999589da DS4_REPO?=https://github.com/antirez/ds4 CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) diff --git a/core/application/startup.go b/core/application/startup.go index 66a8131620da..a5aec50908c7 100644 --- a/core/application/startup.go +++ b/core/application/startup.go @@ -373,6 +373,16 @@ func New(opts ...config.AppOption) (*Application, error) { cfgLoaderOpts := options.ToConfigLoaderOptions() modelRevisionLifecycle := modeladmin.NewDistributedModelRevisionLifecycle(distSvc.Registry, distSvc.ModelCleanup) gs.SetModelRevisionLifecycle(modelRevisionLifecycle) + // Bring the controller's stored revisions back in line with the + // configuration on disk. An inference request may only establish a + // revision, never replace one, so a model whose stored value had + // drifted stayed unroutable until someone deleted the row. + if err := modeladmin.ResyncModelConfigRevisions(options.Context, + application.ModelConfigLoader(), + modeladmin.NewRevisionStore(distSvc.Registry, modelRevisionLifecycle), + ); err != nil { + xlog.Warn("Failed to resync model config revisions", "error", err) + } gs.OnModelsChanged = func(evt messaging.CacheInvalidateEvent) { // ApplyRemoteChange honors the op: a "delete" prunes the element // (a reload-from-path is additive and cannot drop it), anything diff --git a/core/services/modeladmin/revision_resync.go b/core/services/modeladmin/revision_resync.go new file mode 100644 index 000000000000..4724f60e03b0 --- /dev/null +++ b/core/services/modeladmin/revision_resync.go @@ -0,0 +1,118 @@ +package modeladmin + +import ( + "context" + "errors" + "fmt" + + "github.com/mudler/xlog" + "gorm.io/gorm" + + "github.com/mudler/LocalAI/core/config" +) + +// ErrNoStoredRevision reports that the controller holds no revision for a +// model, which is the normal state for one that has never been served. +var ErrNoStoredRevision = gorm.ErrRecordNotFound + +// RevisionStore is the controller state this resync reads and corrects. +type RevisionStore interface { + GetModelConfigRevision(ctx context.Context, modelName string) (string, error) + ApplyConfigRevisions(ctx context.Context, transitions []ModelRevisionTransition) (int, error) +} + +// RevisionReader is the read half, satisfied by the node registry. +type RevisionReader interface { + GetModelConfigRevision(ctx context.Context, modelName string) (string, error) +} + +type revisionStore struct { + RevisionReader + lifecycle ModelRevisionLifecycle +} + +func (s revisionStore) ApplyConfigRevisions(ctx context.Context, t []ModelRevisionTransition) (int, error) { + return s.lifecycle.ApplyConfigRevisions(ctx, t) +} + +// NewRevisionStore pairs the registry that holds the stored revisions with the +// lifecycle that publishes new ones. Returns nil when either half is missing, +// which ResyncModelConfigRevisions treats as "nothing to reconcile". +func NewRevisionStore(reader RevisionReader, lifecycle ModelRevisionLifecycle) RevisionStore { + if reader == nil || lifecycle == nil { + return nil + } + return revisionStore{RevisionReader: reader, lifecycle: lifecycle} +} + +// ResyncModelConfigRevisions makes the controller's stored revision for each +// model agree with what this build computes from the configuration on disk. +// +// The stored revision is what every inference request is checked against, but +// nothing ever re-derived it from the persisted configuration: it moved only on +// an edit, a gallery install, or a peer's change broadcast. Any other way for +// the two to diverge left the model permanently unroutable, because an +// inference request may only establish a revision, never replace one. A +// configuration edited while this frontend was down, or a change in what the +// revision is computed over, both landed there, and the only recovery was +// deleting the row by hand. +// +// Running this at startup makes that self-correcting. Only a model whose stored +// revision disagrees is republished, so replicas of models that did not drift +// keep serving: republishing is not free, it quarantines every replica loaded +// under the old revision. +// +// A model with no stored revision is left alone. It has never been served, and +// inventing controller state for it here would quarantine nothing and describe +// a model that may never be requested. +func ResyncModelConfigRevisions(ctx context.Context, loader *config.ModelConfigLoader, store RevisionStore) error { + if loader == nil || store == nil { + return nil + } + + var transitions []ModelRevisionTransition + for _, cfg := range loader.GetAllModelsConfigs() { + want, err := config.ModelConfigRevision(&cfg) + if err != nil { + return fmt.Errorf("compute config revision for %q: %w", cfg.Name, err) + } + + stored, err := store.GetModelConfigRevision(ctx, cfg.Name) + if errors.Is(err, ErrNoStoredRevision) { + continue + } + if err != nil { + return fmt.Errorf("read stored config revision for %q: %w", cfg.Name, err) + } + if stored == want { + continue + } + + xlog.Warn("Stored model config revision disagrees with the configuration on disk, republishing", + "model", cfg.Name, "stored", shortRevision(stored), "computed", shortRevision(want)) + transitions = append(transitions, ModelRevisionTransition{ + ModelName: cfg.Name, ConfigRevision: want, Disabled: cfg.IsDisabled(), + }) + } + + if len(transitions) == 0 { + return nil + } + if _, err := store.ApplyConfigRevisions(ctx, transitions); err != nil { + return fmt.Errorf("republish model config revisions: %w", err) + } + xlog.Info("Republished model config revisions to match the configuration on disk", "models", len(transitions)) + return nil +} + +// shortRevision trims a revision for log output; the leading bytes identify it +// well enough to tell two apart. +func shortRevision(revision string) string { + if revision == "" { + return "(none)" + } + if len(revision) > 12 { + return revision[:12] + } + return revision +} diff --git a/core/services/modeladmin/revision_resync_test.go b/core/services/modeladmin/revision_resync_test.go new file mode 100644 index 000000000000..3979ccf75298 --- /dev/null +++ b/core/services/modeladmin/revision_resync_test.go @@ -0,0 +1,142 @@ +package modeladmin + +import ( + "context" + "errors" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/pkg/system" +) + +// stubRevisionStore stands in for the controller's stored revisions. +type stubRevisionStore struct { + stored map[string]string + getErr error + applied []ModelRevisionTransition + applyEr error +} + +func (s *stubRevisionStore) GetModelConfigRevision(_ context.Context, name string) (string, error) { + if s.getErr != nil { + return "", s.getErr + } + rev, ok := s.stored[name] + if !ok { + return "", ErrNoStoredRevision + } + return rev, nil +} + +func (s *stubRevisionStore) ApplyConfigRevisions(_ context.Context, t []ModelRevisionTransition) (int, error) { + s.applied = append(s.applied, t...) + return 0, s.applyEr +} + +// The controller pins a model's replicas to a stored revision and rejects any +// request carrying a different one. Nothing ever re-derived that stored value +// from the configuration on disk: it only moved on an edit, a gallery install +// or a peer's change event. So whenever the stored value stopped matching what +// this build computes for an unchanged file, every request for that model was +// rejected until an operator deleted the row by hand. +var _ = Describe("ResyncModelConfigRevisions", func() { + var ( + dir string + loader *config.ModelConfigLoader + store *stubRevisionStore + appConfig *config.ApplicationConfig + ) + + write := func(name, body string) { + Expect(os.WriteFile(filepath.Join(dir, name+".yaml"), []byte(body), 0o600)).To(Succeed()) + } + + revisionOf := func(name string) string { + cfg, ok := loader.GetModelConfig(name) + Expect(ok).To(BeTrue()) + rev, err := config.ModelConfigRevision(&cfg) + Expect(err).ToNot(HaveOccurred()) + return rev + } + + BeforeEach(func() { + dir = GinkgoT().TempDir() + appConfig = config.NewApplicationConfig() + appConfig.SystemState = &system.SystemState{Model: system.Model{ModelsPath: dir}} + loader = config.NewModelConfigLoader(dir) + store = &stubRevisionStore{stored: map[string]string{}} + }) + + load := func() { + Expect(loader.LoadModelConfigsFromPath(dir, appConfig.ToConfigLoaderOptions()...)).To(Succeed()) + } + + It("republishes the revision when the stored one no longer matches the config on disk", func() { + write("drifted", "name: drifted\nbackend: llama-cpp\ncontext_size: 4096\n") + load() + store.stored["drifted"] = "a-revision-from-an-earlier-build" + + Expect(ResyncModelConfigRevisions(context.Background(), loader, store)).To(Succeed()) + + Expect(store.applied).To(HaveLen(1)) + Expect(store.applied[0].ModelName).To(Equal("drifted")) + Expect(store.applied[0].ConfigRevision).To(Equal(revisionOf("drifted"))) + }) + + It("leaves a model alone when the stored revision already matches", func() { + write("agreed", "name: agreed\nbackend: llama-cpp\n") + load() + store.stored["agreed"] = revisionOf("agreed") + + Expect(ResyncModelConfigRevisions(context.Background(), loader, store)).To(Succeed()) + + Expect(store.applied).To(BeEmpty(), "republishing an unchanged revision would quarantine live replicas for nothing") + }) + + // A model nobody has served has no stored revision. Creating one here would + // invent controller state for a model that may never be requested; the first + // request establishes it. + It("does not create state for a model that has never been served", func() { + write("never-served", "name: never-served\nbackend: llama-cpp\n") + load() + + Expect(ResyncModelConfigRevisions(context.Background(), loader, store)).To(Succeed()) + + Expect(store.applied).To(BeEmpty()) + }) + + It("republishes only the models that actually drifted", func() { + write("drifted", "name: drifted\nbackend: llama-cpp\n") + write("agreed", "name: agreed\nbackend: llama-cpp\ncontext_size: 2048\n") + load() + store.stored["drifted"] = "stale" + store.stored["agreed"] = revisionOf("agreed") + + Expect(ResyncModelConfigRevisions(context.Background(), loader, store)).To(Succeed()) + + Expect(store.applied).To(HaveLen(1)) + Expect(store.applied[0].ModelName).To(Equal("drifted")) + }) + + It("reports a store failure instead of continuing silently", func() { + write("drifted", "name: drifted\nbackend: llama-cpp\n") + load() + store.stored["drifted"] = "stale" + store.applyEr = errors.New("database is down") + + Expect(ResyncModelConfigRevisions(context.Background(), loader, store)).ToNot(Succeed()) + }) + + It("skips a model whose stored revision cannot be read rather than guessing", func() { + write("unreadable", "name: unreadable\nbackend: llama-cpp\n") + load() + store.getErr = errors.New("connection reset") + + Expect(ResyncModelConfigRevisions(context.Background(), loader, store)).ToNot(Succeed()) + Expect(store.applied).To(BeEmpty()) + }) +}) diff --git a/core/services/nodes/interfaces.go b/core/services/nodes/interfaces.go index c204752de19b..93399bc145dc 100644 --- a/core/services/nodes/interfaces.go +++ b/core/services/nodes/interfaces.go @@ -60,6 +60,7 @@ type ModelRouter interface { GetNodeLabels(ctx context.Context, nodeID string) ([]NodeLabel, error) FindNodesWithModel(ctx context.Context, modelName string) ([]BackendNode, error) LoadedReplicaStats(ctx context.Context, modelName string, candidateNodeIDs []string) ([]ReplicaCandidate, error) + MarkUnhealthy(ctx context.Context, nodeID string) error LoadJobStore } diff --git a/core/services/nodes/model_router_test.go b/core/services/nodes/model_router_test.go index 43002006a1b0..13e36b12d40a 100644 --- a/core/services/nodes/model_router_test.go +++ b/core/services/nodes/model_router_test.go @@ -250,3 +250,7 @@ var _ = Describe("ModelRouterAdapter", func() { }) }) }) + +func (f *fakeModelRouterForSmartRouter) MarkUnhealthy(_ context.Context, _ string) error { + return nil +} diff --git a/core/services/nodes/reconciler.go b/core/services/nodes/reconciler.go index f7fefc8356c8..88ca28bbe2fb 100644 --- a/core/services/nodes/reconciler.go +++ b/core/services/nodes/reconciler.go @@ -278,8 +278,9 @@ func (rc *ReplicaReconciler) reconcileOnce(ctx context.Context) { // reconcileState runs the state-reconciliation passes: drain pending backend // ops for freshly-healthy nodes, reconcile registry rows against what workers -// report they are running, then port-probe whatever is left. All passes are -// best-effort: a failure on one node doesn't stop the rest. +// report they are running, port-probe whatever is left, then reclaim replica +// slots held by loads nobody is driving. All passes are best-effort: a failure +// on one node doesn't stop the rest. // // Order matters. The worker pass runs first and refreshes updated_at for every // model a worker vouches for, which takes those rows out of the port prober's @@ -292,6 +293,9 @@ func (rc *ReplicaReconciler) reconcileState(ctx context.Context) { rc.reconcileNodeProcesses(ctx) rc.probeLoadedModels(ctx) rc.sweepLeakedInFlight(ctx) + // Runs last: the passes above can move a row into a serving state, and a + // row that just became loaded is no longer this sweeper's business. + rc.reclaimAbandonedLoads(ctx) } // drainPendingBackendOps retries queued backend ops whose next_retry_at has diff --git a/core/services/nodes/reconciler_abandoned_load.go b/core/services/nodes/reconciler_abandoned_load.go new file mode 100644 index 000000000000..9b6b7b65430b --- /dev/null +++ b/core/services/nodes/reconciler_abandoned_load.go @@ -0,0 +1,98 @@ +package nodes + +import ( + "context" + "errors" + "time" + + "github.com/mudler/xlog" + "gorm.io/gorm" +) + +const ( + // abandonedLoadGrace is how long a replica row may sit in a pre-serving + // state before the sweeper will consider it at all. + // + // It exists to cover the window between creating the replica row and + // writing the load job that vouches for it. Without it a load could be + // reclaimed in the moment before its own job row exists. It is not the + // thing that protects a long transfer: the job heartbeat does that. + abandonedLoadGrace = 5 * time.Minute +) + +// preServingStates are the replica states that hold a slot without being able +// to serve a request. NextFreeReplicaIndex counts every state except +// "unloading", so a row parked in one of these occupies capacity while +// answering nothing. +var preServingStates = []string{"loading", "staging"} + +// reclaimAbandonedLoads removes replica rows whose load will never finish. +// +// The other reconciler passes and the router's eviction query all filter +// state = "loaded", and the per-model probe skips rows without an address, so +// nothing reclaimed a row that never got that far. On a node with one replica +// slot per model, a single interrupted transfer made the model unschedulable +// there until an operator intervened: scheduling saw no free slot, and eviction +// found nothing it was allowed to evict. +// +// A row is abandoned when no live load job vouches for it. Ownership is decided +// by the job's LastProgress heartbeat rather than elapsed time, because staging +// a large checkpoint legitimately runs for a long while without touching the +// replica row. That is the same signal job takeover already trusts, so a +// transfer this sweeper reclaims is one no replica is still driving. +func (rc *ReplicaReconciler) reclaimAbandonedLoads(ctx context.Context) { + if rc.db == nil { + return + } + + cutoff := time.Now().Add(-abandonedLoadGrace) + var stuck []NodeModel + if err := rc.db.WithContext(ctx). + Where("state IN ? AND updated_at < ?", preServingStates, cutoff). + Find(&stuck).Error; err != nil { + xlog.Warn("Reconciler: failed to list replicas stuck before serving", "error", err) + return + } + + now := time.Now() + for _, row := range stuck { + if rc.loadStillRunning(ctx, row.ModelName, now) { + continue + } + if err := rc.registry.RemoveNodeModel(ctx, row.NodeID, row.ModelName, row.ReplicaIndex); err != nil { + xlog.Warn("Reconciler: failed to reclaim abandoned load", + "node", row.NodeID, "model", row.ModelName, "replica", row.ReplicaIndex, + "state", row.State, "error", err) + continue + } + xlog.Warn("Reconciler: reclaimed a replica slot held by a load nobody is driving", + "node", row.NodeID, "model", row.ModelName, "replica", row.ReplicaIndex, "state", row.State) + } +} + +// loadStillRunning reports whether a load job is actively driving this model. +// +// A missing job means nobody is loading it. A failed job has already given up. +// An orphaned job stopped heartbeating, which is the condition another replica +// uses to take it over, so the transfer behind it is not progressing either. +// Any error reading the job is treated as "still running": leaving a slot held +// for one more pass costs a scheduling opportunity, while removing a row out +// from under a live transfer would restart a multi-gigabyte load. +func (rc *ReplicaReconciler) loadStillRunning(ctx context.Context, modelName string, now time.Time) bool { + job, err := rc.registry.GetLoadJob(ctx, modelName) + if errors.Is(err, gorm.ErrRecordNotFound) { + return false + } + if err != nil { + xlog.Warn("Reconciler: cannot read load job, leaving the replica slot held", + "model", modelName, "error", err) + return true + } + if job == nil { + return false + } + if job.State == LoadJobStateFailed { + return false + } + return !job.IsOrphaned(now) +} diff --git a/core/services/nodes/reconciler_abandoned_load_test.go b/core/services/nodes/reconciler_abandoned_load_test.go new file mode 100644 index 000000000000..6740b13aafa7 --- /dev/null +++ b/core/services/nodes/reconciler_abandoned_load_test.go @@ -0,0 +1,133 @@ +package nodes + +import ( + "context" + "runtime" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/gorm" + + "github.com/mudler/LocalAI/core/services/testutil" +) + +// A replica row in loading or staging holds its slot: NextFreeReplicaIndex +// counts every state except unloading. Nothing reclaimed such a row. Every +// reconciler sweep and the router's eviction query filter state = "loaded", and +// the per-model health probe skips rows with no address, which is exactly what a +// row that never finished loading has. So a worker that dropped out mid-transfer +// left a row that pinned the only replica slot on that node for that model, and +// the next request failed with "no replica slot ... all models busy". +// +// Elapsed time alone cannot decide this: staging a large checkpoint legitimately +// runs for tens of minutes. The load job's LastProgress heartbeat is the +// discriminator, the same signal job takeover already trusts. +var _ = Describe("ReplicaReconciler — abandoned load sweeper", func() { + var ( + db *gorm.DB + registry *NodeRegistry + node *BackendNode + rc *ReplicaReconciler + ) + + BeforeEach(func() { + if runtime.GOOS == "darwin" { + Skip("testcontainers requires Docker, not available on macOS CI") + } + db = testutil.SetupTestDB() + var err error + registry, err = NewNodeRegistry(db) + Expect(err).ToNot(HaveOccurred()) + node = &BackendNode{Name: "n1", NodeType: NodeTypeBackend, Address: "10.0.0.1:50051"} + Expect(registry.Register(context.Background(), node, true)).To(Succeed()) + rc = NewReplicaReconciler(ReplicaReconcilerOptions{Registry: registry, DB: db}) + }) + + // seedReplica creates a replica row in the given state, aged so it is past + // the sweeper's grace period unless stated otherwise. + seedReplica := func(model, state string, age time.Duration) { + Expect(db.Create(&NodeModel{ + ID: model + "-row", + NodeID: node.ID, + ModelName: model, + State: state, + UpdatedAt: time.Now().Add(-age), + }).Error).To(Succeed()) + } + + seedJob := func(model, state string, sinceProgress time.Duration) { + Expect(db.Create(&ModelLoadJob{ + TrackingKey: model, + State: state, + OwnerReplica: "someone", + LastProgress: time.Now().Add(-sinceProgress), + CreatedAt: time.Now().Add(-sinceProgress), + UpdatedAt: time.Now().Add(-sinceProgress), + }).Error).To(Succeed()) + } + + rowExists := func(model string) bool { + var count int64 + Expect(db.Model(&NodeModel{}).Where("model_name = ?", model).Count(&count).Error).To(Succeed()) + return count > 0 + } + + It("reclaims a staging row whose load job has stopped heartbeating", func() { + seedReplica("abandoned", "staging", time.Hour) + seedJob("abandoned", LoadJobStateStaging, 30*time.Minute) + + rc.reclaimAbandonedLoads(context.Background()) + + Expect(rowExists("abandoned")).To(BeFalse()) + }) + + It("reclaims a loading row that has no load job at all", func() { + seedReplica("orphan", "loading", time.Hour) + + rc.reclaimAbandonedLoads(context.Background()) + + Expect(rowExists("orphan")).To(BeFalse()) + }) + + It("keeps a long transfer whose job is still heartbeating", func() { + // The row itself is old, because staging does not touch it. Only the + // job proves the transfer is alive. + seedReplica("big-model", "staging", time.Hour) + seedJob("big-model", LoadJobStateStaging, time.Second) + + rc.reclaimAbandonedLoads(context.Background()) + + Expect(rowExists("big-model")).To(BeTrue(), "a live transfer must never be reclaimed") + }) + + It("leaves a freshly created row alone while its job row is still being written", func() { + seedReplica("just-started", "loading", time.Second) + + rc.reclaimAbandonedLoads(context.Background()) + + Expect(rowExists("just-started")).To(BeTrue()) + }) + + It("does not touch loaded replicas, which the other sweeps own", func() { + seedReplica("serving", "loaded", time.Hour) + + rc.reclaimAbandonedLoads(context.Background()) + + Expect(rowExists("serving")).To(BeTrue()) + }) + + It("frees the slot so the model can be scheduled on that node again", func() { + seedReplica("wedged", "staging", time.Hour) + seedJob("wedged", LoadJobStateFailed, time.Minute) + + _, err := registry.NextFreeReplicaIndex(context.Background(), node.ID, "wedged", 1) + Expect(err).To(MatchError(ErrNoFreeSlot), "precondition: the stuck row holds the only slot") + + rc.reclaimAbandonedLoads(context.Background()) + + idx, err := registry.NextFreeReplicaIndex(context.Background(), node.ID, "wedged", 1) + Expect(err).ToNot(HaveOccurred()) + Expect(idx).To(Equal(0)) + }) +}) diff --git a/core/services/nodes/registry.go b/core/services/nodes/registry.go index d6fd59662ff0..1e5d52db02f3 100644 --- a/core/services/nodes/registry.go +++ b/core/services/nodes/registry.go @@ -1159,11 +1159,28 @@ func requireCurrentRevision(tx *gorm.DB, modelName, revision string) error { return err } if state.ConfigRevision != revision { - return ErrStaleModelConfigRevision + // Name both sides. "stale model config revision" on its own says only + // that two hashes differ, which leaves an operator no way to tell an + // edited configuration from a revision that is not reproducible for one + // unchanged file. + return fmt.Errorf("%w (request carries %s, controller holds %s)", + ErrStaleModelConfigRevision, shortRevision(revision), shortRevision(state.ConfigRevision)) } return nil } +// shortRevision trims a revision for log and error output. The full value is a +// sha256 hex digest; the leading bytes identify it well enough to compare two. +func shortRevision(revision string) string { + if revision == "" { + return "(none)" + } + if len(revision) > 12 { + return revision[:12] + } + return revision +} + func validateRevisionWrite(modelName, revision string, revisionRequired bool) error { if modelName == "" { return fmt.Errorf("model name is required") diff --git a/core/services/nodes/revision_error_detail_test.go b/core/services/nodes/revision_error_detail_test.go new file mode 100644 index 000000000000..25be6866a5a0 --- /dev/null +++ b/core/services/nodes/revision_error_detail_test.go @@ -0,0 +1,24 @@ +package nodes + +import ( + "errors" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Stale revision error detail", func() { + It("keeps errors.Is matching so callers can still classify it", func() { + err := fmt.Errorf("%w (request carries %s, controller holds %s)", + ErrStaleModelConfigRevision, shortRevision("aaaabbbbccccdddd"), shortRevision("1111222233334444")) + Expect(errors.Is(err, ErrStaleModelConfigRevision)).To(BeTrue()) + Expect(err.Error()).To(ContainSubstring("stale model config revision")) + }) + + It("names both revisions so an operator can tell which side moved", func() { + Expect(shortRevision("aaaabbbbccccdddd")).To(Equal("aaaabbbbcccc")) + Expect(shortRevision("short")).To(Equal("short")) + Expect(shortRevision("")).To(Equal("(none)")) + }) +}) diff --git a/core/services/nodes/router.go b/core/services/nodes/router.go index 465b2d44ec9a..14b19092e5f7 100644 --- a/core/services/nodes/router.go +++ b/core/services/nodes/router.go @@ -1089,34 +1089,44 @@ func (r *SmartRouter) scheduleNewModel(ctx context.Context, backendType, modelID // If freeSlotNodes is empty (everyone full), candidateNodeIDs is whatever // it was — we'll fall through to eviction below. - var node *BackendNode - - if estimatedVRAM > 0 { - if candidateNodeIDs != nil { - node, err = r.registry.FindNodeWithVRAMFromSet(ctx, estimatedVRAM, candidateNodeIDs) - } else { - node, err = r.registry.FindNodeWithVRAM(ctx, estimatedVRAM) - } - if err != nil { - xlog.Warn("No nodes with enough VRAM, falling back to standard scheduling", - "required_vram", vram.FormatBytes(estimatedVRAM), "error", err) + // Node choice is wrapped in a liveness check: a node's stored status comes + // from its HTTP heartbeat, which is a different channel from the bus that + // carries the install. A worker that has died stops answering on the bus at + // once but stays healthy in the database until its heartbeat ages out, so + // without this the scheduler could commit to a node it cannot reach. + selectNode := func() *BackendNode { + var candidate *BackendNode + var selErr error + if estimatedVRAM > 0 { + if candidateNodeIDs != nil { + candidate, selErr = r.registry.FindNodeWithVRAMFromSet(ctx, estimatedVRAM, candidateNodeIDs) + } else { + candidate, selErr = r.registry.FindNodeWithVRAM(ctx, estimatedVRAM) + } + if selErr != nil { + xlog.Warn("No nodes with enough VRAM, falling back to standard scheduling", + "required_vram", vram.FormatBytes(estimatedVRAM), "error", selErr) + } } - } - if node == nil { - if candidateNodeIDs != nil { - node, err = r.registry.FindIdleNodeFromSet(ctx, candidateNodeIDs) - if err != nil { - node, err = r.registry.FindLeastLoadedNodeFromSet(ctx, candidateNodeIDs) - } - } else { - node, err = r.registry.FindIdleNode(ctx) - if err != nil { - node, err = r.registry.FindLeastLoadedNode(ctx) + if candidate == nil { + if candidateNodeIDs != nil { + candidate, selErr = r.registry.FindIdleNodeFromSet(ctx, candidateNodeIDs) + if selErr != nil { + candidate, _ = r.registry.FindLeastLoadedNodeFromSet(ctx, candidateNodeIDs) + } + } else { + candidate, selErr = r.registry.FindIdleNode(ctx) + if selErr != nil { + candidate, _ = r.registry.FindLeastLoadedNode(ctx) + } } } + return candidate } + node := r.pickReachableNode(ctx, selectNode) + // 4. Preemptive eviction: if no suitable node found, evict the LRU model with zero in-flight if node == nil { evictedNode, evictErr := r.evictLRUAndFreeNode(ctx) diff --git a/core/services/nodes/router_liveness.go b/core/services/nodes/router_liveness.go new file mode 100644 index 000000000000..88646162fde2 --- /dev/null +++ b/core/services/nodes/router_liveness.go @@ -0,0 +1,60 @@ +package nodes + +import ( + "context" + "errors" + + "github.com/mudler/xlog" + "github.com/nats-io/nats.go" +) + +// maxNodeLivenessRetries bounds how many unreachable nodes a single scheduling +// attempt discards before giving up. Each discarded node is marked unhealthy, +// so the bound only has to cover one burst of dead workers rather than the +// whole fleet. +const maxNodeLivenessRetries = 3 + +// nodeAnswersOnBus reports whether a node still has a live subscription. +// +// Only nats.ErrNoResponders means "absent". Any other outcome, a timeout or a +// transport hiccup, leaves the node eligible: wrongly excluding a node that is +// merely slow costs real capacity, while the install that follows already +// reports its own failure. When no command sender is configured there is no bus +// to consult and every node is treated as reachable, which preserves the +// behaviour of deployments that do not run one. +func (r *SmartRouter) nodeAnswersOnBus(node *BackendNode) bool { + if r.unloader == nil || node == nil { + return true + } + err := r.unloader.PingNode(node.ID) + return !errors.Is(err, nats.ErrNoResponders) +} + +// pickReachableNode calls selectNode until it yields a node that still answers +// on the bus, and returns nil when it cannot find one. +// +// A node that does not answer is marked unhealthy before the next attempt. That +// both removes it from the next selection, which queries only healthy nodes, +// and tells every other scheduler in the cluster what this one just learned, so +// the discovery is not repeated one failed request at a time. +func (r *SmartRouter) pickReachableNode(ctx context.Context, selectNode func() *BackendNode) *BackendNode { + for range maxNodeLivenessRetries { + node := selectNode() + if node == nil { + return nil + } + if r.nodeAnswersOnBus(node) { + return node + } + xlog.Warn("Scheduled node is not answering on the bus, marking unhealthy and re-scheduling", + "node", node.Name, "nodeID", node.ID) + if err := r.registry.MarkUnhealthy(ctx, node.ID); err != nil { + // Without the demotion the next selection would hand back the same + // node, so stop rather than spin. + xlog.Warn("Failed to mark unreachable node unhealthy", + "node", node.Name, "nodeID", node.ID, "error", err) + return nil + } + } + return nil +} diff --git a/core/services/nodes/router_nats_liveness_test.go b/core/services/nodes/router_nats_liveness_test.go new file mode 100644 index 000000000000..ec4820c9f0a8 --- /dev/null +++ b/core/services/nodes/router_nats_liveness_test.go @@ -0,0 +1,118 @@ +package nodes + +import ( + "context" + "errors" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// A node's stored status comes from its HTTP heartbeat, but work is dispatched +// over NATS. A worker that dies stops answering on the bus immediately and +// keeps its healthy status until the heartbeat ages out, so the scheduler could +// commit to a node it could not reach. The request then failed outright with +// "no responders available" rather than moving to a node that was actually up. +var _ = Describe("Scheduling past a node that left the bus", func() { + var ( + reg *fakeModelRouter + fake *fakeUnloader + router *SmartRouter + ) + + newNode := func(id string) *BackendNode { + return &BackendNode{ID: id, Name: id, Address: id + ":50051"} + } + + // selectorReturning hands back each node in turn, mimicking a scheduler + // that re-picks after the previous choice was demoted. + selectorReturning := func(nodes ...*BackendNode) func() *BackendNode { + i := 0 + return func() *BackendNode { + if i >= len(nodes) { + return nil + } + n := nodes[i] + i++ + return n + } + } + + BeforeEach(func() { + reg = &fakeModelRouter{} + fake = &fakeUnloader{deadNodes: map[string]bool{}} + router = NewSmartRouter(reg, SmartRouterOptions{Unloader: fake}) + }) + + It("passes over a node that no longer answers and takes one that does", func() { + dead, alive := newNode("dead-node"), newNode("alive-node") + fake.deadNodes["dead-node"] = true + + picked := router.pickReachableNode(context.Background(), selectorReturning(dead, alive)) + + Expect(picked).ToNot(BeNil()) + Expect(picked.ID).To(Equal("alive-node")) + Expect(fake.pingCalls).To(Equal([]string{"dead-node", "alive-node"})) + }) + + It("demotes the absent node so other schedulers stop choosing it", func() { + dead, alive := newNode("dead-node"), newNode("alive-node") + fake.deadNodes["dead-node"] = true + + router.pickReachableNode(context.Background(), selectorReturning(dead, alive)) + + Expect(reg.markedUnhealthy).To(Equal([]string{"dead-node"})) + }) + + It("takes the first node when it answers, without probing further", func() { + first, second := newNode("first"), newNode("second") + + picked := router.pickReachableNode(context.Background(), selectorReturning(first, second)) + + Expect(picked.ID).To(Equal("first")) + Expect(fake.pingCalls).To(Equal([]string{"first"})) + }) + + It("gives up rather than spinning when every node is gone", func() { + a, b, c, d := newNode("a"), newNode("b"), newNode("c"), newNode("d") + for _, id := range []string{"a", "b", "c", "d"} { + fake.deadNodes[id] = true + } + + picked := router.pickReachableNode(context.Background(), selectorReturning(a, b, c, d)) + + Expect(picked).To(BeNil()) + Expect(len(fake.pingCalls)).To(BeNumerically("<=", maxNodeLivenessRetries)) + }) + + It("stops when the demotion itself fails, so it cannot loop on one node", func() { + dead := newNode("dead-node") + fake.deadNodes["dead-node"] = true + reg.markUnhealthyErr = errors.New("database is down") + + picked := router.pickReachableNode(context.Background(), selectorReturning(dead, dead, dead)) + + Expect(picked).To(BeNil()) + Expect(fake.pingCalls).To(Equal([]string{"dead-node"})) + }) + + // Only a no-responders answer proves absence. Excluding a node that is + // merely slow would cost real capacity. + It("keeps a node that answers slowly or errors for another reason", func() { + slow := newNode("slow-node") + fake.pingErr = errors.New("timeout waiting for reply") + + picked := router.pickReachableNode(context.Background(), selectorReturning(slow)) + + Expect(picked).ToNot(BeNil()) + Expect(picked.ID).To(Equal("slow-node")) + Expect(reg.markedUnhealthy).To(BeEmpty()) + }) + + It("treats every node as reachable when no command sender is configured", func() { + plain := NewSmartRouter(reg, SmartRouterOptions{}) + node := newNode("only-node") + + Expect(plain.pickReachableNode(context.Background(), selectorReturning(node))).To(Equal(node)) + }) +}) diff --git a/core/services/nodes/router_test.go b/core/services/nodes/router_test.go index 96db9b93fcb9..0f0938335778 100644 --- a/core/services/nodes/router_test.go +++ b/core/services/nodes/router_test.go @@ -17,6 +17,7 @@ import ( "github.com/mudler/LocalAI/pkg/distributedhdr" grpc "github.com/mudler/LocalAI/pkg/grpc" pb "github.com/mudler/LocalAI/pkg/grpc/proto" + "github.com/nats-io/nats.go" ggrpc "google.golang.org/grpc" "google.golang.org/protobuf/proto" "gorm.io/gorm" @@ -61,6 +62,10 @@ func (f *fakeFileStager) ListRemoteDir(_ context.Context, _, _ string) ([]string // fakeModelRouter implements ModelRouter with configurable return values. type fakeModelRouter struct { + // markedUnhealthy records nodes demoted by the scheduler's liveness check. + markedUnhealthy []string + markUnhealthyErr error + fakeLoadJobStore // FindAndLockNodeWithModel returns @@ -474,7 +479,15 @@ type fakeUnloader struct { stopCalls []string // "nodeID:model" stopErr error unloadCalls []string - unloadErr error + + // deadNodes names the nodes PingNode reports as absent from the bus, and + // pingCalls records every node it was asked about, in order. + deadNodes map[string]bool + pingCalls []string + // pingErr is returned for nodes not in deadNodes, so a spec can model a + // node that is reachable but answering badly. + pingErr error + unloadErr error } // installCall captures the args we care about when asserting that the @@ -532,6 +545,22 @@ func (f *fakeUnloader) UnloadModelOnNode(nodeID, modelName string) error { return f.unloadErr } +func (f *fakeModelRouter) MarkUnhealthy(_ context.Context, nodeID string) error { + f.markedUnhealthy = append(f.markedUnhealthy, nodeID) + return f.markUnhealthyErr +} + +func (f *fakeUnloader) PingNode(nodeID string) error { + f.mu.Lock() + f.pingCalls = append(f.pingCalls, nodeID) + dead := f.deadNodes[nodeID] + f.mu.Unlock() + if dead { + return nats.ErrNoResponders + } + return f.pingErr +} + // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- diff --git a/core/services/nodes/unloader.go b/core/services/nodes/unloader.go index 8d47d71a622d..3b1cd15b8955 100644 --- a/core/services/nodes/unloader.go +++ b/core/services/nodes/unloader.go @@ -36,6 +36,10 @@ type NodeCommandSender interface { ListBackends(nodeID string) (*messaging.BackendListReply, error) StopBackend(nodeID, backend string) error UnloadModelOnNode(nodeID, modelName string) error + // PingNode reports whether the node is still subscribed on the bus. It + // returns nats.ErrNoResponders when nothing answers for the node, which is + // the only condition callers may read as "this node cannot be given work". + PingNode(nodeID string) error } // RemoteUnloaderAdapter implements NodeCommandSender and model.RemoteModelUnloader @@ -360,6 +364,27 @@ func (a *RemoteUnloaderAdapter) ListBackends(nodeID string) (*messaging.BackendL return messaging.RequestJSON[messaging.BackendListRequest, messaging.BackendListReply](a.nats, subject, messaging.BackendListRequest{}, 30*time.Second) } +// PingNode checks that a worker still has a live subscription on the bus. +// +// A node's status in the database comes from its HTTP heartbeat, which is a +// separate channel from NATS. A worker that has died stops answering on NATS +// at once but keeps its healthy status until the heartbeat ages out, so the +// scheduler could pick a node that could not be given work and the request +// failed with "no responders available". +// +// It reuses the models.running subject rather than a dedicated ping subject on +// purpose: a new subject would go unanswered by any worker that has not been +// upgraded yet, and this check would then report every one of them as dead. +// The worker answers out of its in-memory process table, so a live node +// replies immediately, and NATS reports no-responders without waiting out the +// timeout. +func (a *RemoteUnloaderAdapter) PingNode(nodeID string) error { + subject := messaging.SubjectNodeModelsRunning(nodeID) + _, err := messaging.RequestJSON[messaging.ModelsRunningRequest, messaging.ModelsRunningReply]( + a.nats, subject, messaging.ModelsRunningRequest{}, 5*time.Second) + return err +} + // ListRunningModels asks a worker node which model backend processes it // currently has running, via NATS request-reply. // diff --git a/core/services/worker/ephemeral_cleanup.go b/core/services/worker/ephemeral_cleanup.go new file mode 100644 index 000000000000..4a8e92da9677 --- /dev/null +++ b/core/services/worker/ephemeral_cleanup.go @@ -0,0 +1,112 @@ +package worker + +import ( + "context" + "os" + "path/filepath" + "time" + + "github.com/mudler/xlog" +) + +const ( + // defaultEphemeralStagingTTL bounds how long a staged request input can + // outlive the request that needed it. Inference reads these files while the + // request runs, so the window has to cover a slow multimodal request; it + // does not have to cover anything longer. + defaultEphemeralStagingTTL = 6 * time.Hour + // defaultEphemeralStagingSweep is how often the worker sweeps. + defaultEphemeralStagingSweep = 30 * time.Minute +) + +// StartEphemeralStagingCleanup sweeps the worker's own staging directory for +// per-request input files left behind by finished requests. +// +// The frontend already expires ephemeral keys from object storage +// (services/storage.StartEphemeralCleanup), but a worker receives these files +// over the file-transfer server and writes them to its local disk, where +// nothing expired them. They accumulated for as long as the worker lived and +// eventually filled the volume, at which point every backend start failed +// because the process manager could no longer create a state directory. +func StartEphemeralStagingCleanup(ctx context.Context, stagingDir string, ttl, interval time.Duration) { + if stagingDir == "" { + return + } + if ttl <= 0 { + ttl = defaultEphemeralStagingTTL + } + if interval <= 0 { + interval = defaultEphemeralStagingSweep + } + + go func() { + ticker := time.NewTicker(interval) + defer ticker.Stop() + // Sweep once at startup: a worker that crashed with staged files leaves + // them behind, and waiting a full interval to reclaim that space is the + // case that hurts on a volume that is already close to full. + CleanEphemeralStaging(stagingDir, ttl) + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + CleanEphemeralStaging(stagingDir, ttl) + } + } + }() + + xlog.Info("Ephemeral staging cleanup started", "dir", stagingDir, "ttl", ttl, "interval", interval) +} + +// CleanEphemeralStaging removes staged per-request directories older than ttl. +// It only ever descends into /ephemeral, so staged model weights, +// which live alongside it and are not scratch, are never considered. +func CleanEphemeralStaging(stagingDir string, ttl time.Duration) { + root := filepath.Join(stagingDir, "ephemeral") + categories, err := os.ReadDir(root) + if err != nil { + // A worker that has never served a file-bearing request has no + // ephemeral directory at all. That is the normal case, not a fault. + if !os.IsNotExist(err) { + xlog.Warn("Ephemeral staging cleanup: cannot read staging root", "dir", root, "error", err) + } + return + } + + cutoff := time.Now().Add(-ttl) + removed := 0 + for _, category := range categories { + if !category.IsDir() { + continue + } + categoryDir := filepath.Join(root, category.Name()) + entries, err := os.ReadDir(categoryDir) + if err != nil { + xlog.Warn("Ephemeral staging cleanup: cannot read category", "dir", categoryDir, "error", err) + continue + } + for _, entry := range entries { + path := filepath.Join(categoryDir, entry.Name()) + info, err := entry.Info() + if err != nil { + xlog.Warn("Ephemeral staging cleanup: cannot stat entry", "path", path, "error", err) + continue + } + // A request rewrites nothing after staging, so the entry's own + // modification time is when its request was served. + if !info.ModTime().Before(cutoff) { + continue + } + if err := os.RemoveAll(path); err != nil { + xlog.Warn("Ephemeral staging cleanup: cannot remove", "path", path, "error", err) + continue + } + removed++ + } + } + + if removed > 0 { + xlog.Info("Ephemeral staging cleanup removed stale request files", "count", removed, "dir", root) + } +} diff --git a/core/services/worker/ephemeral_cleanup_test.go b/core/services/worker/ephemeral_cleanup_test.go new file mode 100644 index 000000000000..3542f1afcffa --- /dev/null +++ b/core/services/worker/ephemeral_cleanup_test.go @@ -0,0 +1,58 @@ +package worker + +import ( + "os" + "path/filepath" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Worker ephemeral staging cleanup", func() { + var stagingDir string + + // mkEphemeral creates one staged request directory holding a file, and + // backdates both so the sweeper sees it as `age` old. + mkEphemeral := func(id string, age time.Duration) string { + dir := filepath.Join(stagingDir, "ephemeral", "inputs", id) + Expect(os.MkdirAll(dir, 0o750)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(dir, "payload.bin"), []byte("x"), 0o600)).To(Succeed()) + stamp := time.Now().Add(-age) + Expect(os.Chtimes(filepath.Join(dir, "payload.bin"), stamp, stamp)).To(Succeed()) + Expect(os.Chtimes(dir, stamp, stamp)).To(Succeed()) + return dir + } + + BeforeEach(func() { stagingDir = GinkgoT().TempDir() }) + + It("removes staged request directories older than the TTL", func() { + old := mkEphemeral("aaaa1111", 48*time.Hour) + CleanEphemeralStaging(stagingDir, time.Hour) + Expect(old).ToNot(BeAnExistingFile()) + }) + + It("keeps directories a running request may still be reading", func() { + fresh := mkEphemeral("bbbb2222", 5*time.Minute) + CleanEphemeralStaging(stagingDir, time.Hour) + Expect(fresh).To(BeAnExistingFile()) + }) + + It("leaves staged models and everything outside ephemeral alone", func() { + modelDir := filepath.Join(stagingDir, "models", "some-model") + Expect(os.MkdirAll(modelDir, 0o750)).To(Succeed()) + weights := filepath.Join(modelDir, "weights.gguf") + Expect(os.WriteFile(weights, []byte("w"), 0o600)).To(Succeed()) + stamp := time.Now().Add(-90 * 24 * time.Hour) + Expect(os.Chtimes(weights, stamp, stamp)).To(Succeed()) + Expect(os.Chtimes(modelDir, stamp, stamp)).To(Succeed()) + + CleanEphemeralStaging(stagingDir, time.Hour) + + Expect(weights).To(BeAnExistingFile(), "a staged model is not ephemeral scratch") + }) + + It("does nothing when no ephemeral directory exists", func() { + Expect(func() { CleanEphemeralStaging(stagingDir, time.Hour) }).ToNot(Panic()) + }) +}) diff --git a/core/services/worker/worker.go b/core/services/worker/worker.go index 2c48c14ea6e5..6434c3cd6b69 100644 --- a/core/services/worker/worker.go +++ b/core/services/worker/worker.go @@ -159,6 +159,10 @@ func Run(ctx *cliContext.Context, cfg *Config) error { return fmt.Errorf("starting HTTP file transfer server: %w", err) } + // Per-request input files land in stagingDir over that server and nothing + // used to remove them, so a long-lived worker filled its own disk. + StartEphemeralStagingCleanup(shutdownCtx, stagingDir, 0, 0) + // Connect to NATS xlog.Info("Connecting to NATS", "url", sanitize.URL(cfg.NatsURL)) natsClient, err := connectNats() diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index 3d1e42a85a43..70d63684cb8c 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -1020,11 +1020,30 @@ Notes: - Upgrade the worker when it does not support the exact model-stop request. - Stop and restart the stale backend only as an operational recovery action. LocalAI keeps it non-routable while durable cleanup is pending. +**A model cannot be scheduled on a node that looks free (`no replica slot ... all models busy, cannot evict`):** +- A replica row in `staging` or `loading` holds its slot: slot allocation counts every state except `unloading`. If a worker drops out mid-transfer, that row never reaches `loaded`, and eviction only ever considers `loaded` replicas, so on a node with one replica slot per model the model became unschedulable there. +- The reconciler now reclaims a replica row stuck before serving when no load job is still driving it, and the freed slot is immediately reusable. +- Liveness is decided by the load job's progress heartbeat, not by elapsed time. Staging a large checkpoint legitimately runs for a long time without touching the replica row, so a transfer that is still progressing is never reclaimed however long it takes. +- `Reconciler: reclaimed a replica slot held by a load nobody is driving` names each row reclaimed this way. + +**A request fails with `nats: no responders available for request`:** +- The chosen worker was not subscribed on the bus when the frontend tried to install the backend on it. A node's status comes from its HTTP heartbeat, which is a separate channel: a worker that stops stays `healthy` until that heartbeat ages out. +- The scheduler now checks that a node still answers on the bus before it commits to it, marks one that does not as unhealthy, and picks another. A request should therefore see this only when no reachable node is left. +- Only a no-responders answer counts as absent. A worker that answers slowly stays eligible, because excluding it would cost capacity that is really there. +- Check the worker process is running and its NATS connection is up. `Scheduled node is not answering on the bus` in the frontend log names each node demoted this way. + +**A worker fills its own disk over time:** +- A request that carries a file (an image, an audio clip, a video) stages that file to the worker under `/../staging/ephemeral/`. The worker deletes these 6 hours after the request that needed them, and sweeps every 30 minutes plus once at startup, so a worker that crashed mid-request still reclaims the space. +- Releases before this sweep existed kept every staged input for the lifetime of the worker. Delete `/../staging/ephemeral/` on an affected worker once, as the user the worker runs as; the sweep keeps it bounded from then on. +- Staged **model** files are not touched by this. They live beside the ephemeral directory and are not per-request scratch. +- A worker whose volume is genuinely full reports `creating backend process state directory under ...: no space left on device` when a backend starts. + **Requests fail with `stale model config revision` although nobody edited the model:** - A model's stored revision must describe its persisted configuration. Releases before this fix also hashed the per-request prediction parameters, so the first request after a restart pinned the revision to its own `temperature`, `top_p`, `stop` and similar values. Every later request that sent different values was then rejected. - Upgrade the frontend replicas first. After the upgrade the revision is stamped when the configuration is loaded, so it no longer depends on the request body. -- The stored revision does not heal on its own, because the recorded value belongs to no persisted configuration. Clear it once per affected model so the next request establishes the correct revision: `DELETE FROM model_config_states WHERE model_name = '';` -- Saving any edit for the model through the API or the WebUI has the same effect, because an edit publishes the current revision. +- Each frontend now reconciles the stored revisions against the configuration on disk at startup, and republishes any that disagree, so a drifted revision heals on the next restart. Only models that actually drifted are republished, because republishing quarantines the replicas loaded under the old revision. +- A model that has never been served has no stored revision and is left alone; its first request establishes one. +- On a release without that reconciliation, clear the row once per affected model so the next request establishes the correct revision: `DELETE FROM model_config_states WHERE model_name = '';` Saving any edit through the API or the WebUI has the same effect. **Port conflicts on workers:** - Each model gets its own gRPC process on an incrementing port (50051, 50052, ...) diff --git a/gallery/index.yaml b/gallery/index.yaml index 4f0fb21920c9..c9a240db2178 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -50,8 +50,8 @@ use_tokenizer_template: true files: - filename: llama-cpp/models/Huihui-Qwen3.8-27B-abliterated-bf16/Huihui-Qwen3.8-27B-abliterated-bf16.gguf - sha256: a64a5e5464d7d0ea7ffcbc937cf28f8a7bc9b0a6e87be6034e6b854418d5abd5 uri: https://huggingface.co/huihui-ai/Huihui-Qwen3.8-27B-abliterated-GGUF/resolve/main/Huihui-Qwen3.8-27B-abliterated-bf16.gguf + sha256: b880f2042df16a9a493800c83f1ee16b7cd46e8ca695f193655940a1949e9097 - filename: llama-cpp/mmproj/Huihui-Qwen3.8-27B-abliterated-bf16/mmproj-model-bf16.gguf sha256: c9a09064683620bea3d3bfed5d4462e1a97a7d2fff7e5045d6862a0a85eeb5b5 uri: https://huggingface.co/huihui-ai/Huihui-Qwen3.8-27B-abliterated-GGUF/resolve/main/mmproj-model-bf16.gguf @@ -1180,7 +1180,7 @@ files: - filename: llama-cpp/models/nemotron-3.5-lightning-30b-a3b/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-Q8_0.gguf uri: huggingface://ggml-org/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-Q8_0.gguf - sha256: d4cc4c9fffaa356db8b49cfaba9233609cfb29b4dd89d827665210dc1cc8dbbb + sha256: b0e25ce2d301930e706d549a59d18c5969dcec664fdf7466435287b03fefda36 - &muse-glimmer-30b name: "muse-glimmer-30b" variants: @@ -1386,6 +1386,47 @@ - filename: llama-cpp/models/muse-glimmer-30b/dflash-kquant.gguf uri: huggingface://meta-models/Muse-Glimmer-30B-GGUF/dflash-kquant.gguf sha256: 27d9a805fa29b943cfb6ad4843367cd4eaaaf06bd452d8cc3e00a2cd18a677bc +- name: "homura-30b-q4" + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/hyrelabs/Homura-30B-GGUF + - https://huggingface.co/darkc0de/Muse-Glimmer-30B-heretic + description: | + Homura 30B is an English, agent-focused fine-tune of Muse Glimmer 30B. + It targets autonomous tool use and direct instruction following. This + entry uses the publisher's 16.9 GB Q4_K_M GGUF and supports a 131K-token + context window. + license: "apache-2.0" + tags: + - llm + - gguf + - cpu + - gpu + - agent + - tools + - long-context + - uncensored + last_checked: "2026-08-23" + overrides: + backend: llama-cpp + context_size: 131072 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + parameters: + model: llama-cpp/models/homura-30b/Homura-30B-Q4_K_M.gguf + temperature: 0.2 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/homura-30b/Homura-30B-Q4_K_M.gguf + uri: huggingface://hyrelabs/Homura-30B-GGUF/Homura-30B-Q4_K_M.gguf + sha256: fd4dc394b193ac1cc0f4bb9fb44bff3d6eebd34df62246fc753104c0cc98cc51 - &qwen3-5-9b-defiant-fable name: "qwen3.5-9b-defiant-fable-mtp" variants: @@ -1483,6 +1524,164 @@ - filename: llama-cpp/mmproj/qwen3.5-9b-defiant-fable/mmproj-BF16.gguf uri: huggingface://DavidAU/Qwen3.5-9B-The-Defiant-Fable-Uncensored-Heretic-NEO-IMATRIX-MAX-MTP-GGUF/mmproj-BF16.gguf sha256: 853698ce7aa6c7ba732478bad280240969ddf7b0fcbf93900046f63903a83383 +- &qwen3-8-2b-distill + name: "qwen3.8-2b-distill-q4" + variants: + - model: qwen3.8-2b-distill-q8 + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/empero-ai/Qwen3.8-2B-Distill + - https://huggingface.co/empero-ai/Qwen3.8-2B-Distill-GGUF + description: | + Qwen3.8 2B Distill is an Apache-2.0, text-only Qwen3.5 2B fine-tune + distilled from Qwen3.8 2.4T A95B reasoning traces. It targets compact + reasoning, coding, instruction following, and function calling with a 262K + native context window. This entry uses the balanced Q4_K_M GGUF + quantization; the Q8_0 variant offers higher fidelity. + license: apache-2.0 + icon: https://qianwen-res.oss-cn-beijing.aliyuncs.com/logo_qwen.jpg + tags: + - llm + - gguf + - cpu + - gpu + - qwen3.5 + - reasoning + - coding + - tool-use + last_checked: "2026-08-23" + overrides: + backend: llama-cpp + context_size: 32768 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + - reasoning_format:deepseek + parameters: + model: llama-cpp/models/qwen3.8-2b-distill/Qwen3.8-2B-Q4_K_M.gguf + temperature: 0.6 + top_k: 20 + top_p: 0.95 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/qwen3.8-2b-distill/Qwen3.8-2B-Q4_K_M.gguf + uri: huggingface://empero-ai/Qwen3.8-2B-Distill-GGUF/Qwen3.8-2B-Q4_K_M.gguf + sha256: 4aa0fb13c431514262f259d420ecc95a8714df58ac2a2384514e20b93983f0ff +- !!merge <<: *qwen3-8-2b-distill + name: "qwen3.8-2b-distill-q8" + variants: [] + description: | + Qwen3.8 2B Distill in the higher-fidelity Q8_0 GGUF format. This text-only + Qwen3.5 2B fine-tune targets reasoning, coding, instruction following, and + function calling with a 262K native context window. + overrides: + backend: llama-cpp + context_size: 32768 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + - reasoning_format:deepseek + parameters: + model: llama-cpp/models/qwen3.8-2b-distill/Qwen3.8-2B-Q8_0.gguf + temperature: 0.6 + top_k: 20 + top_p: 0.95 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/qwen3.8-2b-distill/Qwen3.8-2B-Q8_0.gguf + uri: huggingface://empero-ai/Qwen3.8-2B-Distill-GGUF/Qwen3.8-2B-Q8_0.gguf + sha256: 866773b0d68f09a1db9733555e92daff85b617f9a2e601773dff494c5ca2bbf2 +- &qwen3-8-4b-distill + name: "qwen3.8-4b-distill-q4" + variants: + - model: qwen3.8-4b-distill-q8 + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/empero-ai/Qwen3.8-4B-Distill + - https://huggingface.co/empero-ai/Qwen3.8-4B-Distill-GGUF + description: | + Qwen3.8 4B Distill is an Apache-2.0, text-only Qwen3.5 4B fine-tune + distilled from Qwen3.8 2.4T A95B reasoning traces. It targets reasoning, + coding, instruction following, and function calling with a 262K native + context window. This entry uses the balanced Q4_K_M GGUF quantization; the + Q8_0 variant offers higher fidelity. + license: apache-2.0 + icon: https://qianwen-res.oss-cn-beijing.aliyuncs.com/logo_qwen.jpg + tags: + - llm + - gguf + - cpu + - gpu + - qwen3.5 + - reasoning + - coding + - tool-use + last_checked: "2026-08-23" + overrides: + backend: llama-cpp + context_size: 32768 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + - reasoning_format:deepseek + parameters: + model: llama-cpp/models/qwen3.8-4b-distill/Qwen3.8-4B-Q4_K_M.gguf + temperature: 0.6 + top_k: 20 + top_p: 0.95 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/qwen3.8-4b-distill/Qwen3.8-4B-Q4_K_M.gguf + uri: huggingface://empero-ai/Qwen3.8-4B-Distill-GGUF/Qwen3.8-4B-Q4_K_M.gguf + sha256: dec96e8cf2e11b613bb46513dec485377f9ca5a351e71712ee0e244f287c6790 +- !!merge <<: *qwen3-8-4b-distill + name: "qwen3.8-4b-distill-q8" + variants: [] + description: | + Qwen3.8 4B Distill in the higher-fidelity Q8_0 GGUF format. This text-only + Qwen3.5 4B fine-tune targets reasoning, coding, instruction following, and + function calling with a 262K native context window. + overrides: + backend: llama-cpp + context_size: 32768 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + - reasoning_format:deepseek + parameters: + model: llama-cpp/models/qwen3.8-4b-distill/Qwen3.8-4B-Q8_0.gguf + temperature: 0.6 + top_k: 20 + top_p: 0.95 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/qwen3.8-4b-distill/Qwen3.8-4B-Q8_0.gguf + uri: huggingface://empero-ai/Qwen3.8-4B-Distill-GGUF/Qwen3.8-4B-Q8_0.gguf + sha256: 770b780d6754a4954d1caf395c9239eaeb394f15c7a7ea34039883377c93c9c3 - name: "btl-4-compact" url: "github:mudler/LocalAI/gallery/virtual.yaml@master" urls: @@ -4865,7 +5064,9 @@ - gpu icon: https://cdn-uploads.huggingface.co/production/uploads/61b8e2ba285851687028d395/2b08LKpev0DNEk6DlnWkY.png variants: + - model: lfm2.5-2.6b-dspark - model: lfm2.5-2.6b-q8 + - model: lfm2.5-2.6b-q8-dspark overrides: backend: llama-cpp context_size: 131072 @@ -4921,6 +5122,105 @@ - filename: llama-cpp/models/LFM2.5-2.6B-GGUF/LFM2.5-2.6B-Q8_0.gguf uri: https://huggingface.co/LiquidAI/LFM2.5-2.6B-GGUF/resolve/main/LFM2.5-2.6B-Q8_0.gguf sha256: 1e22128dfa128bdfb684da167e74e072d0a056baa7d06d9f280291e2839b0fc9 +- !!merge <<: *lfm2-5-2-6b + name: "lfm2.5-2.6b-dspark" + description: | + LFM2.5-2.6B with LiquidAI's DSpark speculative drafter. This build pairs + the Q4_K_M target with the compact Q4_K_M draft sidecar for lower-memory + hosts. DSpark proposes blocks of tokens that the target model verifies, + which preserves the target model's output while accelerating generation. + tags: + - llm + - gguf + - reasoning + - cpu + - gpu + - dspark + variants: null + urls: + - https://huggingface.co/LiquidAI/LFM2.5-2.6B + - https://huggingface.co/LiquidAI/LFM2.5-2.6B-GGUF + - https://huggingface.co/LiquidAI/LFM2.5-2.6B-DSpark-GGUF + overrides: + backend: llama-cpp + context_size: 131072 + draft_model: llama-cpp/models/LFM2.5-2.6B-DSpark-GGUF/LFM2.5-2.6B-DSpark-Q4_K_M.gguf + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - completion + options: + - use_jinja:true + - spec_type:draft-dspark + - spec_n_max:10 + - spec_n_min:0 + parameters: + model: llama-cpp/models/LFM2.5-2.6B-GGUF/LFM2.5-2.6B-Q4_K_M.gguf + repeat_penalty: 1.1 + temperature: 0.1 + top_k: 50 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/LFM2.5-2.6B-GGUF/LFM2.5-2.6B-Q4_K_M.gguf + uri: https://huggingface.co/LiquidAI/LFM2.5-2.6B-GGUF/resolve/main/LFM2.5-2.6B-Q4_K_M.gguf + sha256: 02a8b7e17487d326e46d68ce0ba24211e1b80a14c4cd0597fa73c1cd697f52ed + - filename: llama-cpp/models/LFM2.5-2.6B-DSpark-GGUF/LFM2.5-2.6B-DSpark-Q4_K_M.gguf + uri: https://huggingface.co/LiquidAI/LFM2.5-2.6B-DSpark-GGUF/resolve/main/LFM2.5-2.6B-DSpark-Q4_K_M.gguf + sha256: 63786b768e43562591a934625a66539eab39beb09e6e3d38e17509e34b79a3cd +- !!merge <<: *lfm2-5-2-6b + name: "lfm2.5-2.6b-q8-dspark" + description: | + LFM2.5-2.6B with LiquidAI's DSpark speculative drafter. This build pairs + the higher-quality Q8_0 target with the recommended F16 draft sidecar for + the best acceptance length. DSpark proposes blocks of tokens that the + target model verifies, which preserves the target model's output while + accelerating generation. + tags: + - llm + - gguf + - reasoning + - cpu + - gpu + - dspark + variants: null + urls: + - https://huggingface.co/LiquidAI/LFM2.5-2.6B + - https://huggingface.co/LiquidAI/LFM2.5-2.6B-GGUF + - https://huggingface.co/LiquidAI/LFM2.5-2.6B-DSpark-GGUF + overrides: + backend: llama-cpp + context_size: 131072 + draft_model: llama-cpp/models/LFM2.5-2.6B-DSpark-GGUF/LFM2.5-2.6B-DSpark-F16.gguf + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - completion + options: + - use_jinja:true + - spec_type:draft-dspark + - spec_n_max:10 + - spec_n_min:0 + parameters: + model: llama-cpp/models/LFM2.5-2.6B-GGUF/LFM2.5-2.6B-Q8_0.gguf + repeat_penalty: 1.1 + temperature: 0.1 + top_k: 50 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/LFM2.5-2.6B-GGUF/LFM2.5-2.6B-Q8_0.gguf + uri: https://huggingface.co/LiquidAI/LFM2.5-2.6B-GGUF/resolve/main/LFM2.5-2.6B-Q8_0.gguf + sha256: 1e22128dfa128bdfb684da167e74e072d0a056baa7d06d9f280291e2839b0fc9 + - filename: llama-cpp/models/LFM2.5-2.6B-DSpark-GGUF/LFM2.5-2.6B-DSpark-F16.gguf + uri: https://huggingface.co/LiquidAI/LFM2.5-2.6B-DSpark-GGUF/resolve/main/LFM2.5-2.6B-DSpark-F16.gguf + sha256: e198962c08903f3ba29f0ce6bf8e17f5e60bf85ec2f8673e1e2aab03508937e5 - &bigbang-v1 name: "bigbang-v1-q4-k-m" variants: @@ -8807,6 +9107,94 @@ - filename: llama-cpp/mmproj/tencent_UI-Mate-9B-GGUF/mmproj-tencent_UI-Mate-9B-f16.gguf sha256: 5a8380c4637dddceed9dbc28fffcdfa8601909c0ece9fe218fbd6888ec5d2c16 uri: huggingface://bartowski/tencent_UI-Mate-9B-GGUF/mmproj-tencent_UI-Mate-9B-f16.gguf +- &ui-mate-27b + name: ui-mate-27b + url: github:mudler/LocalAI/gallery/virtual.yaml@master + variants: + - model: ui-mate-27b-q8 + urls: + - https://huggingface.co/tencent/UI-Mate-27B + - https://huggingface.co/bartowski/tencent_UI-Mate-27B-GGUF + description: | + UI-Mate-27B is Tencent's 27B-parameter multimodal computer-use agent, + fine-tuned from Qwen3.6-27B. It accepts task instructions, screenshots, + and interaction history, then emits reasoning and structured mouse and + keyboard actions for long-running desktop tasks. The model requires an + external runtime to execute its actions and should run with human + confirmation for sensitive operations. This entry uses the recommended + Q4_K_M GGUF quantization. + license: apache-2.0 + tags: + - ui-mate + - qwen + - qwen3.6 + - 27b + - llm + - gguf + - quantized + - chat + - vision + - multimodal + - agent + - computer-use + - gpu + - cpu + last_checked: "2026-08-22" + overrides: + backend: llama-cpp + function: + grammar: + disable: true + known_usecases: + - chat + - vision + mmproj: llama-cpp/mmproj/tencent_UI-Mate-27B-GGUF/mmproj-tencent_UI-Mate-27B-f16.gguf + options: + - use_jinja:true + parameters: + model: llama-cpp/models/tencent_UI-Mate-27B-GGUF/tencent_UI-Mate-27B-Q4_K_M.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/tencent_UI-Mate-27B-GGUF/tencent_UI-Mate-27B-Q4_K_M.gguf + sha256: 05ee4cca09f10de19e828e51497cc8b4c9eabffe2a177a54d1188ff869669f20 + uri: huggingface://bartowski/tencent_UI-Mate-27B-GGUF/tencent_UI-Mate-27B-Q4_K_M.gguf + - filename: llama-cpp/mmproj/tencent_UI-Mate-27B-GGUF/mmproj-tencent_UI-Mate-27B-f16.gguf + sha256: 991376d8e954dda92b454358306fb2b4bbd51d522b12977cdfeb3eebbb904fbb + uri: huggingface://bartowski/tencent_UI-Mate-27B-GGUF/mmproj-tencent_UI-Mate-27B-f16.gguf +- !!merge <<: *ui-mate-27b + name: ui-mate-27b-q8 + variants: [] + description: | + UI-Mate-27B is Tencent's 27B-parameter multimodal computer-use agent, + fine-tuned from Qwen3.6-27B. It accepts task instructions, screenshots, + and interaction history, then emits reasoning and structured mouse and + keyboard actions for long-running desktop tasks. The model requires an + external runtime to execute its actions and should run with human + confirmation for sensitive operations. This entry uses the higher-quality + Q8_0 GGUF quantization. + overrides: + backend: llama-cpp + function: + grammar: + disable: true + known_usecases: + - chat + - vision + mmproj: llama-cpp/mmproj/tencent_UI-Mate-27B-GGUF/mmproj-tencent_UI-Mate-27B-f16.gguf + options: + - use_jinja:true + parameters: + model: llama-cpp/models/tencent_UI-Mate-27B-GGUF/tencent_UI-Mate-27B-Q8_0.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/tencent_UI-Mate-27B-GGUF/tencent_UI-Mate-27B-Q8_0.gguf + sha256: 3626d336fb902e91f2e7d241d95c9410c74094d5f1739c76ef896140878bcd68 + uri: huggingface://bartowski/tencent_UI-Mate-27B-GGUF/tencent_UI-Mate-27B-Q8_0.gguf + - filename: llama-cpp/mmproj/tencent_UI-Mate-27B-GGUF/mmproj-tencent_UI-Mate-27B-f16.gguf + sha256: 991376d8e954dda92b454358306fb2b4bbd51d522b12977cdfeb3eebbb904fbb + uri: huggingface://bartowski/tencent_UI-Mate-27B-GGUF/mmproj-tencent_UI-Mate-27B-f16.gguf - &fara1-5-4b name: fara1.5-4b url: github:mudler/LocalAI/gallery/virtual.yaml@master @@ -18275,11 +18663,11 @@ model: SmolVLM2-256M-Video-Instruct-Q8_0.gguf files: - filename: SmolVLM2-256M-Video-Instruct-Q8_0.gguf - sha256: af7ce9951a2f46c4f6e5def253e5b896ca5e417010e7a9949fdc9e5175c27767 uri: huggingface://ggml-org/SmolVLM2-256M-Video-Instruct-GGUF/SmolVLM2-256M-Video-Instruct-Q8_0.gguf + sha256: 1202d1c54493bddff5b0ecbc36fcb7520ff720b9fa3d7224aeb293581f90529a - filename: mmproj-SmolVLM2-256M-Video-Instruct-Q8_0.gguf - sha256: d34913a588464ff7215f086193e0426a4f045eaba74456ee5e2667d8ed6798b1 uri: huggingface://ggml-org/SmolVLM2-256M-Video-Instruct-GGUF/mmproj-SmolVLM2-256M-Video-Instruct-Q8_0.gguf + sha256: 05d5751132244a6ebd64cba9b34898c0d874b2cb78159d758e1d4da3aad91581 - name: qwen3-30b-a3b url: github:mudler/LocalAI/gallery/qwen3.yaml@master urls: diff --git a/go.mod b/go.mod index 8e8084cec68a..5418b351de5d 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( github.com/modelcontextprotocol/go-sdk v1.5.0 github.com/mudler/cogito v0.11.1-0.20260721122412-6eece18a6bb6 github.com/mudler/edgevpn v0.34.0 - github.com/mudler/go-processmanager v0.1.2-0.20260720195933-3d64f5c974fc + github.com/mudler/go-processmanager v0.1.2-0.20260823202314-dfa0ed852db6 github.com/mudler/memory v0.0.0-20260406210934-424c1ecf2cf8 github.com/mudler/nib v0.6.0 github.com/mudler/xlog v0.0.6 diff --git a/go.sum b/go.sum index 7bcbec8e6fd3..6169840bd308 100644 --- a/go.sum +++ b/go.sum @@ -1027,6 +1027,8 @@ github.com/mudler/go-piper v0.0.0-20241023091659-2494246fd9fc h1:RxwneJl1VgvikiX github.com/mudler/go-piper v0.0.0-20241023091659-2494246fd9fc/go.mod h1:O7SwdSWMilAWhBZMK9N9Y/oBDyMMzshE3ju8Xkexwig= github.com/mudler/go-processmanager v0.1.2-0.20260720195933-3d64f5c974fc h1:NEFmd7+JoImN5dZI81/vcBRjtMg+GfEa7nEjQ029hd0= github.com/mudler/go-processmanager v0.1.2-0.20260720195933-3d64f5c974fc/go.mod h1:h6kmHUZeafr+k5hRYpGLMzJFH4hItHffgpRo2QIkP+o= +github.com/mudler/go-processmanager v0.1.2-0.20260823202314-dfa0ed852db6 h1:/nFm1Ttf8g1BnWtEth986JR34pCh9rzae5A2vKBZosc= +github.com/mudler/go-processmanager v0.1.2-0.20260823202314-dfa0ed852db6/go.mod h1:h6kmHUZeafr+k5hRYpGLMzJFH4hItHffgpRo2QIkP+o= github.com/mudler/localrecall v0.6.3 h1:uXOrP9JmetzxgVKzSrawviyBHZfAcvPBBIrvVUdZjDA= github.com/mudler/localrecall v0.6.3/go.mod h1:28k5n19raUrkuwXkacdNsBlj8yuSnGhpT16tu+2+4dU= github.com/mudler/memory v0.0.0-20260406210934-424c1ecf2cf8 h1:Ry8RiWy8fZ6Ff4E7dPmjRsBrnHOnPeOOj2LhCgyjQu0= diff --git a/pkg/model/process.go b/pkg/model/process.go index 4fd6041e4508..fb1ed004a1d1 100644 --- a/pkg/model/process.go +++ b/pkg/model/process.go @@ -231,6 +231,16 @@ func (ml *ModelLoader) StartProcess(grpcProcess, id string, serverAddress string return ml.startProcess(grpcProcess, id, serverAddress, args...) } +// newProcessStateDir creates the directory a backend process uses for its pid, +// state and log files, and reports why when it cannot. +func newProcessStateDir() (string, error) { + dir, err := os.MkdirTemp(os.TempDir(), "go-processmanager") + if err != nil { + return "", fmt.Errorf("creating backend process state directory under %s: %w", os.TempDir(), err) + } + return dir, nil +} + func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string, args ...string) (*process.Process, error) { // Make sure the process is executable // Check first if it has executable permissions @@ -261,8 +271,19 @@ func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string // and the GPU would silently fall back to CPU). No-op for other backends. env = append(env, vulkanICDEnv(workDir)...) + // Resolve the state directory here rather than through + // process.WithTemporaryStateDir(). process.New applies its options but + // discards the error they return, so a temp directory that cannot be + // created leaves StateDir empty and every later option unapplied. Run() + // then reported "mkdir : no such file or directory" with no path, hiding + // the real cause (a full volume, or a TMPDIR that no longer resolves). + stateDir, err := newProcessStateDir() + if err != nil { + return nil, err + } + grpcControlProcess := process.New( - process.WithTemporaryStateDir(), + process.WithStateDir(stateDir), process.WithName(filepath.Base(grpcProcess)), process.WithArgs(append(args, []string{"--addr", serverAddress}...)...), process.WithEnvironment(env...), diff --git a/pkg/model/process_statedir_test.go b/pkg/model/process_statedir_test.go new file mode 100644 index 000000000000..207c2991d991 --- /dev/null +++ b/pkg/model/process_statedir_test.go @@ -0,0 +1,38 @@ +package model + +import ( + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Backend process state directory", func() { + It("reports why the state directory could not be created", func() { + // A worker whose volume is full, or whose TMPDIR no longer resolves, + // cannot get a state directory. go-processmanager's New() drops the + // option error, leaving StateDir empty, and Run() then failed with + // "mkdir : no such file or directory" naming no path at all. Resolving + // the directory here keeps the real cause attached. + GinkgoT().Setenv("TMPDIR", filepath.Join(GinkgoT().TempDir(), "does-not-exist")) + + dir, err := newProcessStateDir() + Expect(err).To(HaveOccurred()) + Expect(dir).To(BeEmpty()) + Expect(err.Error()).To(ContainSubstring("backend process state directory")) + Expect(err.Error()).To(ContainSubstring("does-not-exist"), + "the error must name the directory it could not create") + }) + + It("returns a usable directory when the temp location works", func() { + GinkgoT().Setenv("TMPDIR", GinkgoT().TempDir()) + + dir, err := newProcessStateDir() + Expect(err).ToNot(HaveOccurred()) + Expect(dir).ToNot(BeEmpty()) + info, statErr := os.Stat(dir) + Expect(statErr).ToNot(HaveOccurred()) + Expect(info.IsDir()).To(BeTrue()) + }) +})