From 4f42daee8fa9c1cad2f725cef252e5d5855a1b97 Mon Sep 17 00:00:00 2001 From: Rex Raphael Date: Sat, 15 Aug 2026 16:08:51 -0500 Subject: [PATCH 1/8] feat(job,worker): record what each attempt actually consumed exec.Usage was populated by every rung and then discarded inside the terminal closure, so nothing accumulated. A resource estimator cannot be built from data that was never kept, and the history it needs cannot be backfilled. Usage pairs the prediction with the measurement on one row: Resources is what the enqueuing process computed, PeakRSS and the rest are what the attempt really used, and InputBytes is the feature the two correlate through. Failed attempts are recorded too -- a job that OOMs is the strongest evidence about how much memory it needed. UsageRecorder is an optional store capability rather than part of the composite Store, following the WakeNotifier precedent. Usage is telemetry, not correctness: a backend without it runs jobs identically and simply keeps no history, which is what lets backends gain the capability one at a time. Recording happens before handleFailure, which increments RetryCount on its way to scheduling the retry, and through a context.WithoutCancel so a timed-out attempt still leaves its measurements behind. --- id/id.go | 10 ++ job/jobtest/usage.go | 303 +++++++++++++++++++++++++++++++++++++ job/usage.go | 99 ++++++++++++ store/memory/store.go | 16 +- store/memory/usage.go | 87 +++++++++++ store/memory/usage_test.go | 13 ++ worker/runner.go | 23 ++- worker/usage.go | 102 +++++++++++++ worker/usage_test.go | 230 ++++++++++++++++++++++++++++ 9 files changed, 874 insertions(+), 9 deletions(-) create mode 100644 job/jobtest/usage.go create mode 100644 job/usage.go create mode 100644 store/memory/usage.go create mode 100644 store/memory/usage_test.go create mode 100644 worker/usage.go create mode 100644 worker/usage_test.go diff --git a/id/id.go b/id/id.go index 804ec32..0095c78 100644 --- a/id/id.go +++ b/id/id.go @@ -33,6 +33,7 @@ const ( PrefixEvent Prefix = "evt" PrefixWorker Prefix = "wkr" PrefixArtifact Prefix = "art" + PrefixUsage Prefix = "use" ) // ID is the primary identifier type for all Dispatch entities. @@ -140,6 +141,9 @@ type WorkerID = ID // ArtifactID is a type-safe identifier for artifacts (prefix: "art"). type ArtifactID = ID +// UsageID is a type-safe identifier for usage records (prefix: "use"). +type UsageID = ID + // AnyID is a type alias that accepts any valid prefix. type AnyID = ID @@ -174,6 +178,9 @@ func NewWorkerID() ID { return New(PrefixWorker) } // NewArtifactID generates a new unique artifact ID. func NewArtifactID() ID { return New(PrefixArtifact) } +// NewUsageID generates a new unique usage record ID. +func NewUsageID() ID { return New(PrefixUsage) } + // ────────────────────────────────────────────────── // Convenience parsers // ────────────────────────────────────────────────── @@ -205,6 +212,9 @@ func ParseWorkerID(s string) (ID, error) { return ParseWithPrefix(s, PrefixWorke // ParseArtifactID parses a string and validates the "art" prefix. func ParseArtifactID(s string) (ID, error) { return ParseWithPrefix(s, PrefixArtifact) } +// ParseUsageID parses a string and validates the "use" prefix. +func ParseUsageID(s string) (ID, error) { return ParseWithPrefix(s, PrefixUsage) } + // ParseAny parses a string into an ID without type checking the prefix. func ParseAny(s string) (ID, error) { return Parse(s) } diff --git a/job/jobtest/usage.go b/job/jobtest/usage.go new file mode 100644 index 0000000..33849d4 --- /dev/null +++ b/job/jobtest/usage.go @@ -0,0 +1,303 @@ +// Package jobtest provides shared conformance suites for the optional +// job store capabilities. +package jobtest + +import ( + "context" + "testing" + "time" + + "github.com/xraph/dispatch/id" + "github.com/xraph/dispatch/job" + "github.com/xraph/dispatch/resource" +) + +// RunUsageRecorderSuite exercises the job.UsageRecorder contract. +// newRecorder must return a fresh, empty recorder on every call. +func RunUsageRecorderSuite(t *testing.T, newRecorder func() job.UsageRecorder) { + t.Helper() + + tests := []struct { + name string + fn func(*testing.T, job.UsageRecorder) + }{ + {"RecordAndList", testRecordAndList}, + {"RecordsFailedAttempts", testRecordsFailedAttempts}, + {"PreservesPredictionAndActual", testPreservesPredictionAndActual}, + {"FilterByName", testFilterByName}, + {"FilterBySince", testFilterBySince}, + {"NewestFirst", testNewestFirst}, + {"Paging", testPaging}, + {"Purge", testPurge}, + {"PurgeRespectsLimit", testPurgeRespectsLimit}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.fn(t, newRecorder()) + }) + } +} + +func newUsage(name string, at time.Time) *job.Usage { + return &job.Usage{ + ID: id.NewUsageID(), + JobID: id.NewJobID(), + Name: name, + Queue: "default", + Status: job.StateCompleted, + WallTime: time.Second, + RecordedAt: at, + } +} + +func testRecordAndList(t *testing.T, r job.UsageRecorder) { + ctx := context.Background() + u := newUsage("tessellate", time.Now().UTC()) + + if err := r.RecordJobUsage(ctx, u); err != nil { + t.Fatalf("RecordJobUsage: %v", err) + } + + got, err := r.ListJobUsage(ctx, job.UsageListOpts{}) + if err != nil { + t.Fatalf("ListJobUsage: %v", err) + } + + if len(got) != 1 { + t.Fatalf("got %d records, want 1", len(got)) + } + + if got[0].Name != "tessellate" || got[0].JobID != u.JobID { + t.Fatalf("round trip mismatch: %+v", got[0]) + } + + if got[0].WallTime != time.Second { + t.Fatalf("WallTime = %v, want 1s", got[0].WallTime) + } +} + +// testRecordsFailedAttempts pins the case that matters most for sizing: +// a job that died is the strongest evidence about what it needed. +func testRecordsFailedAttempts(t *testing.T, r job.UsageRecorder) { + ctx := context.Background() + + u := newUsage("tessellate", time.Now().UTC()) + u.Status = job.StateFailed + u.PeakRSS = 8 << 30 + + if err := r.RecordJobUsage(ctx, u); err != nil { + t.Fatalf("RecordJobUsage: %v", err) + } + + got, err := r.ListJobUsage(ctx, job.UsageListOpts{}) + if err != nil { + t.Fatalf("ListJobUsage: %v", err) + } + + if len(got) != 1 { + t.Fatalf("a failed attempt was not recorded") + } + + if got[0].Status != job.StateFailed { + t.Fatalf("Status = %q, want failed", got[0].Status) + } + + if got[0].PeakRSS != 8<<30 { + t.Fatalf("PeakRSS = %d, want %d", got[0].PeakRSS, int64(8)<<30) + } +} + +// testPreservesPredictionAndActual is the whole point of the record: an +// estimate with no measurement beside it cannot be calibrated. +func testPreservesPredictionAndActual(t *testing.T, r job.UsageRecorder) { + ctx := context.Background() + + u := newUsage("render", time.Now().UTC()) + u.InputBytes = 2 << 30 + u.Resources = resource.MemoryBytes(4 << 30) + u.PeakRSS = 3 << 30 + u.CPUTime = 90 * time.Second + u.DiskWritten = 512 << 20 + u.Executor = "subprocess" + + if err := r.RecordJobUsage(ctx, u); err != nil { + t.Fatalf("RecordJobUsage: %v", err) + } + + got, err := r.ListJobUsage(ctx, job.UsageListOpts{}) + if err != nil { + t.Fatalf("ListJobUsage: %v", err) + } + + rec := got[0] + + if rec.InputBytes != 2<<30 { + t.Fatalf("InputBytes = %d, want %d", rec.InputBytes, int64(2)<<30) + } + + if rec.Resources.IsZero() { + t.Fatal("the predicted resource set was not preserved") + } + + if rec.PeakRSS != 3<<30 || rec.CPUTime != 90*time.Second || rec.DiskWritten != 512<<20 { + t.Fatalf("measurements not preserved: %+v", rec) + } + + if rec.Executor != "subprocess" { + t.Fatalf("Executor = %q, want subprocess", rec.Executor) + } +} + +func testFilterByName(t *testing.T, r job.UsageRecorder) { + ctx := context.Background() + now := time.Now().UTC() + + for _, name := range []string{"a", "a", "b"} { + if err := r.RecordJobUsage(ctx, newUsage(name, now)); err != nil { + t.Fatalf("RecordJobUsage: %v", err) + } + } + + got, err := r.ListJobUsage(ctx, job.UsageListOpts{Name: "a"}) + if err != nil { + t.Fatalf("ListJobUsage: %v", err) + } + + if len(got) != 2 { + t.Fatalf("got %d records for name a, want 2", len(got)) + } +} + +func testFilterBySince(t *testing.T, r job.UsageRecorder) { + ctx := context.Background() + now := time.Now().UTC() + + if err := r.RecordJobUsage(ctx, newUsage("a", now.Add(-48*time.Hour))); err != nil { + t.Fatalf("RecordJobUsage old: %v", err) + } + + if err := r.RecordJobUsage(ctx, newUsage("a", now)); err != nil { + t.Fatalf("RecordJobUsage new: %v", err) + } + + got, err := r.ListJobUsage(ctx, job.UsageListOpts{Since: now.Add(-time.Hour)}) + if err != nil { + t.Fatalf("ListJobUsage: %v", err) + } + + if len(got) != 1 { + t.Fatalf("got %d records since cutoff, want 1", len(got)) + } +} + +func testNewestFirst(t *testing.T, r job.UsageRecorder) { + ctx := context.Background() + now := time.Now().UTC() + + old := newUsage("a", now.Add(-time.Hour)) + recent := newUsage("a", now) + + for _, u := range []*job.Usage{old, recent} { + if err := r.RecordJobUsage(ctx, u); err != nil { + t.Fatalf("RecordJobUsage: %v", err) + } + } + + got, err := r.ListJobUsage(ctx, job.UsageListOpts{}) + if err != nil { + t.Fatalf("ListJobUsage: %v", err) + } + + if len(got) != 2 { + t.Fatalf("got %d records, want 2", len(got)) + } + + if got[0].ID != recent.ID { + t.Fatal("ListJobUsage did not return the most recent record first") + } +} + +func testPaging(t *testing.T, r job.UsageRecorder) { + ctx := context.Background() + now := time.Now().UTC() + + for i := range 5 { + if err := r.RecordJobUsage(ctx, newUsage("a", now.Add(-time.Duration(i)*time.Minute))); err != nil { + t.Fatalf("RecordJobUsage %d: %v", i, err) + } + } + + first, err := r.ListJobUsage(ctx, job.UsageListOpts{Limit: 2}) + if err != nil { + t.Fatalf("ListJobUsage: %v", err) + } + + if len(first) != 2 { + t.Fatalf("limit 2 returned %d", len(first)) + } + + second, err := r.ListJobUsage(ctx, job.UsageListOpts{Limit: 2, Offset: 2}) + if err != nil { + t.Fatalf("ListJobUsage offset: %v", err) + } + + if len(second) != 2 { + t.Fatalf("limit 2 offset 2 returned %d", len(second)) + } + + if first[0].ID == second[0].ID { + t.Fatal("offset did not advance the page") + } +} + +func testPurge(t *testing.T, r job.UsageRecorder) { + ctx := context.Background() + now := time.Now().UTC() + + if err := r.RecordJobUsage(ctx, newUsage("a", now.Add(-48*time.Hour))); err != nil { + t.Fatalf("RecordJobUsage old: %v", err) + } + + if err := r.RecordJobUsage(ctx, newUsage("a", now)); err != nil { + t.Fatalf("RecordJobUsage new: %v", err) + } + + removed, err := r.PurgeJobUsage(ctx, now.Add(-time.Hour), 0) + if err != nil { + t.Fatalf("PurgeJobUsage: %v", err) + } + + if removed != 1 { + t.Fatalf("PurgeJobUsage removed %d, want 1", removed) + } + + got, err := r.ListJobUsage(ctx, job.UsageListOpts{}) + if err != nil { + t.Fatalf("ListJobUsage: %v", err) + } + + if len(got) != 1 { + t.Fatalf("%d records survived the purge, want 1", len(got)) + } +} + +func testPurgeRespectsLimit(t *testing.T, r job.UsageRecorder) { + ctx := context.Background() + now := time.Now().UTC() + + for range 5 { + if err := r.RecordJobUsage(ctx, newUsage("a", now.Add(-48*time.Hour))); err != nil { + t.Fatalf("RecordJobUsage: %v", err) + } + } + + removed, err := r.PurgeJobUsage(ctx, now, 2) + if err != nil { + t.Fatalf("PurgeJobUsage: %v", err) + } + + if removed != 2 { + t.Fatalf("PurgeJobUsage with limit 2 removed %d", removed) + } +} diff --git a/job/usage.go b/job/usage.go new file mode 100644 index 0000000..bb2d03a --- /dev/null +++ b/job/usage.go @@ -0,0 +1,99 @@ +package job + +import ( + "context" + "time" + + "github.com/xraph/dispatch/id" + "github.com/xraph/dispatch/resource" +) + +// Usage is what one attempt actually consumed, recorded against what was +// predicted for it. +// +// The pairing is the point. Resources is the estimate the enqueuing +// process computed; WallTime, CPUTime, PeakRSS, and DiskWritten are what +// the attempt really used. Without both on the same row there is no way +// to tell a good estimate from a lucky one, and no way to improve either. +// +// InputBytes is recorded separately because it is the feature an +// estimator regresses on: for the workloads this exists to serve, a job's +// footprint is mostly a function of how big its input was. +type Usage struct { + ID id.UsageID `json:"id"` + JobID id.JobID `json:"job_id"` + Name string `json:"name"` + Queue string `json:"queue"` + Attempt int `json:"attempt"` + + // Status is the terminal state this attempt reached. Failed attempts + // are recorded too: a job that OOMs is the single most informative + // data point about how much memory it needed. + Status State `json:"status"` + + // InputBytes totals the artifacts bound to the job, or zero when it + // declared none. + InputBytes int64 `json:"input_bytes"` + + // Resources is what was predicted, empty when the resource model is + // off. + Resources resource.Set `json:"resources,omitempty"` + + // Measured consumption. Every field is zero when the rung that ran + // the attempt could not account it: the in-process rung knows only + // wall time, because a handler sharing the worker's address space has + // no separable RSS. + WallTime time.Duration `json:"wall_time"` + CPUTime time.Duration `json:"cpu_time,omitempty"` + PeakRSS int64 `json:"peak_rss,omitempty"` + DiskWritten int64 `json:"disk_written,omitempty"` + + // Executor names the rung that ran the attempt, so measurements taken + // under different isolation are not silently pooled. + Executor string `json:"executor,omitempty"` + + ScopeAppID string `json:"scope_app_id,omitempty"` + ScopeOrgID string `json:"scope_org_id,omitempty"` + WorkerID id.WorkerID `json:"worker_id,omitempty"` + RecordedAt time.Time `json:"recorded_at"` +} + +// UsageListOpts filters a usage query. +type UsageListOpts struct { + // Name filters to one job definition. Empty means all. + Name string + // Since bounds the query to attempts recorded at or after this time. + Since time.Time + // Limit caps the rows returned. Zero means no limit. + Limit int + // Offset skips rows, for paging. + Offset int +} + +// UsageRecorder is an optional store capability: backends that can retain +// per-attempt measurements implement it, and the worker records through +// it when they do. +// +// It is deliberately not part of the composite Store. Usage is telemetry, +// not correctness — a backend without it runs jobs exactly the same way, +// it just accumulates no history to size them from later. Making it +// optional is what lets a backend gain the capability without every other +// backend having to gain it at the same moment. +// +// Callers must treat every method as best effort. A failure to record +// what a job consumed must never fail the job that consumed it. +type UsageRecorder interface { + // RecordJobUsage persists one attempt's measurements. + RecordJobUsage(ctx context.Context, u *Usage) error + + // ListJobUsage returns recorded attempts, most recent first. + ListJobUsage(ctx context.Context, opts UsageListOpts) ([]*Usage, error) + + // PurgeJobUsage deletes records older than before, up to limit rows, + // and returns how many it removed. + // + // This table grows with every attempt the fleet makes, so something + // has to bound it. Retention is the operator's call, which is why the + // cutoff is a parameter rather than a policy baked in here. + PurgeJobUsage(ctx context.Context, before time.Time, limit int) (int64, error) +} diff --git a/store/memory/store.go b/store/memory/store.go index e726218..8f15f59 100644 --- a/store/memory/store.go +++ b/store/memory/store.go @@ -21,13 +21,14 @@ import ( // Ensure Store implements store.Store at compile time. // We can't import store here (import cycle), so we verify each subsystem. var ( - _ job.Store = (*Store)(nil) - _ workflow.Store = (*Store)(nil) - _ cron.Store = (*Store)(nil) - _ dlq.Store = (*Store)(nil) - _ event.Store = (*Store)(nil) - _ cluster.Store = (*Store)(nil) - _ artifact.Store = (*Store)(nil) + _ job.Store = (*Store)(nil) + _ workflow.Store = (*Store)(nil) + _ cron.Store = (*Store)(nil) + _ dlq.Store = (*Store)(nil) + _ event.Store = (*Store)(nil) + _ cluster.Store = (*Store)(nil) + _ artifact.Store = (*Store)(nil) + _ job.UsageRecorder = (*Store)(nil) ) // Store is a fully in-memory implementation of store.Store. @@ -43,6 +44,7 @@ type Store struct { events map[string]*event.Event workers map[string]*cluster.Worker + jobUsage []*job.Usage artifacts map[string]*artifact.Artifact artifactLinks []*artifact.Link diff --git a/store/memory/usage.go b/store/memory/usage.go new file mode 100644 index 0000000..3e41875 --- /dev/null +++ b/store/memory/usage.go @@ -0,0 +1,87 @@ +package memory + +import ( + "context" + "sort" + "time" + + "github.com/xraph/dispatch/job" +) + +// RecordJobUsage persists one attempt's measurements. +func (s *Store) RecordJobUsage(_ context.Context, u *job.Usage) error { + s.mu.Lock() + defer s.mu.Unlock() + + rec := *u + s.jobUsage = append(s.jobUsage, &rec) + + return nil +} + +// ListJobUsage returns recorded attempts, most recent first. +func (s *Store) ListJobUsage(_ context.Context, opts job.UsageListOpts) ([]*job.Usage, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + out := make([]*job.Usage, 0, len(s.jobUsage)) + + for _, u := range s.jobUsage { + if opts.Name != "" && u.Name != opts.Name { + continue + } + + if !opts.Since.IsZero() && u.RecordedAt.Before(opts.Since) { + continue + } + + rec := *u + out = append(out, &rec) + } + + sort.Slice(out, func(i, j int) bool { + if out[i].RecordedAt.Equal(out[j].RecordedAt) { + return out[i].ID.String() > out[j].ID.String() + } + + return out[i].RecordedAt.After(out[j].RecordedAt) + }) + + if opts.Offset > 0 { + if opts.Offset >= len(out) { + return nil, nil + } + + out = out[opts.Offset:] + } + + if opts.Limit > 0 && opts.Limit < len(out) { + out = out[:opts.Limit] + } + + return out, nil +} + +// PurgeJobUsage deletes records older than before, up to limit rows. +func (s *Store) PurgeJobUsage(_ context.Context, before time.Time, limit int) (int64, error) { + s.mu.Lock() + defer s.mu.Unlock() + + kept := s.jobUsage[:0] + + var removed int64 + + for _, u := range s.jobUsage { + if u.RecordedAt.Before(before) && (limit <= 0 || removed < int64(limit)) { + removed++ + + continue + } + + kept = append(kept, u) + } + + s.jobUsage = kept + + return removed, nil +} diff --git a/store/memory/usage_test.go b/store/memory/usage_test.go new file mode 100644 index 0000000..9b69459 --- /dev/null +++ b/store/memory/usage_test.go @@ -0,0 +1,13 @@ +package memory_test + +import ( + "testing" + + "github.com/xraph/dispatch/job" + "github.com/xraph/dispatch/job/jobtest" + "github.com/xraph/dispatch/store/memory" +) + +func TestUsageRecorderConformance(t *testing.T) { + jobtest.RunUsageRecorderSuite(t, func() job.UsageRecorder { return memory.New() }) +} diff --git a/worker/runner.go b/worker/runner.go index e852e84..5d0371d 100644 --- a/worker/runner.go +++ b/worker/runner.go @@ -244,7 +244,9 @@ func (r *Runner) Close() error { // all — the current lease holder's own write must stand untouched — // and only emits JobFailed so extensions still observe the loss. func (r *Runner) Execute(ctx context.Context, j *job.Job) error { - terminal, err := r.terminalFor(j) + at := &attemptUsage{} + + terminal, err := r.terminalFor(j, at) if err != nil { // terminalFor fails before the middleware chain — and therefore // r.mw — ever runs, so returning err bare here, as this used to, @@ -271,9 +273,16 @@ func (r *Runner) Execute(ctx context.Context, j *job.Job) error { j.UpdatedAt = now if execErr != nil { + // Recorded before handleFailure, which mutates RetryCount on its + // way to scheduling the next attempt: the row must describe the + // attempt that just ran, not the one about to. + r.recordUsage(ctx, j, at, job.StateFailed, elapsed, j.WorkerID) + return r.handleFailure(ctx, j, execErr, now) } + r.recordUsage(ctx, j, at, job.StateCompleted, elapsed, j.WorkerID) + return r.handleSuccess(ctx, j, now, elapsed) } @@ -283,7 +292,7 @@ func (r *Runner) Execute(ctx context.Context, j *job.Job) error { // timeout, and artifact staging — wraps this closure, which is precisely // why staging keeps running in the worker process and an out-of-process // handler receives a directory rather than storage credentials. -func (r *Runner) terminalFor(j *job.Job) (middleware.Handler, error) { +func (r *Runner) terminalFor(j *job.Job, at *attemptUsage) (middleware.Handler, error) { if r.executors == nil { handler, ok := r.registry.Get(j.Name) if !ok { @@ -356,6 +365,16 @@ func (r *Runner) terminalFor(j *job.Job) (middleware.Handler, error) { return &exec.Error{Status: exec.StatusLaunchFailed, Msg: runErr.Error()} } + // The Result is the only place an attempt's measurements exist, + // and it does not survive this closure. Copy them out before + // collapsing it to an error, and before committing outputs: an + // attempt whose commit fails still consumed what it consumed. + if at != nil && res != nil { + at.usage = res.Usage + at.executor = executor.Name() + at.measured = true + } + // Commit what the sandbox actually left on disk before reporting // the attempt as done. This runs ahead of the lease-fenced terminal // write Execute makes afterward (see abandonLostLease), not gated diff --git a/worker/usage.go b/worker/usage.go new file mode 100644 index 0000000..0cb3d52 --- /dev/null +++ b/worker/usage.go @@ -0,0 +1,102 @@ +package worker + +import ( + "context" + "time" + + log "github.com/xraph/go-utils/log" + + "github.com/xraph/dispatch/artifact/staging" + "github.com/xraph/dispatch/exec" + "github.com/xraph/dispatch/id" + "github.com/xraph/dispatch/job" +) + +// attemptUsage carries an attempt's measurements out of the terminal +// closure, which is the only place the executor's Result is visible. +// +// The closure runs beneath the whole middleware chain, so it cannot +// return anything but an error without changing that chain's signature. +// A pointer threaded down and read back after is the narrow way to get +// the numbers out. +type attemptUsage struct { + usage exec.Usage + executor string + measured bool +} + +// recordUsage persists what an attempt consumed, beside what was +// predicted for it. +// +// It is best effort by contract. Telemetry that can fail a job is worse +// than no telemetry, so every failure here is logged at debug and +// swallowed: the job has already reached its terminal state by the time +// this runs, and nothing downstream depends on the row existing. +func (r *Runner) recordUsage( + ctx context.Context, + j *job.Job, + at *attemptUsage, + status job.State, + elapsed time.Duration, + workerID id.WorkerID, +) { + recorder, ok := r.store.(job.UsageRecorder) + if !ok { + // The backend has no usage table. Running jobs is unaffected; + // there is simply no history to size them from later. + return + } + + u := &job.Usage{ + ID: id.NewUsageID(), + JobID: j.ID, + Name: j.Name, + Queue: j.Queue, + Attempt: j.RetryCount, + Status: status, + InputBytes: boundInputBytes(j), + Resources: j.Resources, + WallTime: elapsed, + ScopeAppID: j.ScopeAppID, + ScopeOrgID: j.ScopeOrgID, + WorkerID: workerID, + RecordedAt: time.Now().UTC(), + } + + if at != nil && at.measured { + u.Executor = at.executor + u.CPUTime = at.usage.CPUTime + u.PeakRSS = at.usage.PeakRSS + u.DiskWritten = at.usage.DiskWritten + + // The rung's own wall time is the better figure when it has one: + // it excludes the staging and middleware that ran around it. + if at.usage.WallTime > 0 { + u.WallTime = at.usage.WallTime + } + } + + // A cancelled job context must not stop the record being written — + // that is exactly the attempt worth knowing about. The store's own + // timeout still bounds this. + if err := recorder.RecordJobUsage(context.WithoutCancel(ctx), u); err != nil { + r.logger.Debug("dispatch/worker: could not record job usage", + log.String("job_id", j.ID.String()), + log.String("error", err.Error()), + ) + } +} + +// boundInputBytes totals the artifacts bound to a job. +// +// This is the feature a later estimator regresses on, so it is recorded +// even when the resource model is off: the correlation between input size +// and footprint is the thing worth having history of. +func boundInputBytes(j *job.Job) int64 { + bindings, err := staging.GetBindings(j) + if err != nil { + return 0 + } + + return staging.TotalBoundSize(bindings) +} diff --git a/worker/usage_test.go b/worker/usage_test.go new file mode 100644 index 0000000..ec6f62a --- /dev/null +++ b/worker/usage_test.go @@ -0,0 +1,230 @@ +package worker_test + +import ( + "context" + "errors" + "testing" + "time" + + log "github.com/xraph/go-utils/log" + + "github.com/xraph/dispatch/backoff" + "github.com/xraph/dispatch/ext" + "github.com/xraph/dispatch/id" + "github.com/xraph/dispatch/job" + "github.com/xraph/dispatch/store/memory" + "github.com/xraph/dispatch/worker" +) + +// newUsageRunner builds a Runner over a memory store, which implements +// job.UsageRecorder. +func newUsageRunner(t *testing.T, reg *job.Registry) (*worker.Runner, *memory.Store) { + t.Helper() + + s := memory.New() + + r := worker.NewRunner( + reg, + ext.NewRegistry(log.NewNoopLogger()), + s, + nil, + backoff.NewExponential(time.Millisecond, time.Second), + nil, + log.NewNoopLogger(), + ) + + return r, s +} + +func enqueued(t *testing.T, s *memory.Store, name string) *job.Job { + t.Helper() + + j := &job.Job{ + ID: id.NewJobID(), + Name: name, + Queue: "default", + State: job.StateRunning, + MaxRetries: 3, + RunAt: time.Now().UTC(), + } + + if err := s.EnqueueJob(context.Background(), j); err != nil { + t.Fatalf("EnqueueJob: %v", err) + } + + return j +} + +// TestUsageRecordedOnSuccess is the loop this feature exists to close: +// an attempt runs, and what it consumed survives it. +func TestUsageRecordedOnSuccess(t *testing.T) { + ctx := context.Background() + + reg := job.NewRegistry() + job.RegisterDefinition(reg, job.NewDefinition("ok", + func(context.Context, struct{}) error { + time.Sleep(2 * time.Millisecond) + + return nil + })) + + r, s := newUsageRunner(t, reg) + j := enqueued(t, s, "ok") + + if err := r.Execute(ctx, j); err != nil { + t.Fatalf("Execute: %v", err) + } + + got, err := s.ListJobUsage(ctx, job.UsageListOpts{}) + if err != nil { + t.Fatalf("ListJobUsage: %v", err) + } + + if len(got) != 1 { + t.Fatalf("got %d usage records, want 1", len(got)) + } + + rec := got[0] + + if rec.JobID != j.ID || rec.Name != "ok" { + t.Fatalf("record does not identify the job: %+v", rec) + } + + if rec.Status != job.StateCompleted { + t.Fatalf("Status = %q, want completed", rec.Status) + } + + if rec.WallTime <= 0 { + t.Fatal("WallTime was not measured") + } +} + +// TestUsageRecordedOnFailure matters more than the success case: a job +// that died is the strongest evidence about what it needed. +func TestUsageRecordedOnFailure(t *testing.T) { + ctx := context.Background() + + reg := job.NewRegistry() + job.RegisterDefinition(reg, job.NewDefinition("boom", + func(context.Context, struct{}) error { + return errors.New("exploded") + })) + + r, s := newUsageRunner(t, reg) + j := enqueued(t, s, "boom") + + // Execute returns the handler error on a retryable failure. + _ = r.Execute(ctx, j) + + got, err := s.ListJobUsage(ctx, job.UsageListOpts{}) + if err != nil { + t.Fatalf("ListJobUsage: %v", err) + } + + if len(got) != 1 { + t.Fatalf("got %d usage records for a failed attempt, want 1", len(got)) + } + + if got[0].Status != job.StateFailed { + t.Fatalf("Status = %q, want failed", got[0].Status) + } +} + +// TestUsageRecordsAttemptNotNextAttempt guards the ordering bug this is +// easy to introduce: handleFailure increments RetryCount on its way to +// scheduling the retry, so recording after it would misattribute every +// measurement by one attempt. +func TestUsageRecordsAttemptNotNextAttempt(t *testing.T) { + ctx := context.Background() + + reg := job.NewRegistry() + job.RegisterDefinition(reg, job.NewDefinition("boom", + func(context.Context, struct{}) error { + return errors.New("exploded") + })) + + r, s := newUsageRunner(t, reg) + j := enqueued(t, s, "boom") + j.RetryCount = 2 + + _ = r.Execute(ctx, j) + + got, err := s.ListJobUsage(ctx, job.UsageListOpts{}) + if err != nil { + t.Fatalf("ListJobUsage: %v", err) + } + + if len(got) != 1 { + t.Fatalf("got %d records, want 1", len(got)) + } + + if got[0].Attempt != 2 { + t.Fatalf("Attempt = %d, want 2 (the attempt that ran, not the next one)", got[0].Attempt) + } +} + +// TestUsageRecordingSurvivesCancelledContext pins the case that would +// otherwise lose exactly the attempts worth studying: a job cancelled by +// timeout still has to leave its measurements behind. +func TestUsageRecordingSurvivesCancelledContext(t *testing.T) { + reg := job.NewRegistry() + job.RegisterDefinition(reg, job.NewDefinition("slow", + func(ctx context.Context, _ struct{}) error { + <-ctx.Done() + + return ctx.Err() + })) + + r, s := newUsageRunner(t, reg) + j := enqueued(t, s, "slow") + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + + _ = r.Execute(ctx, j) + + got, err := s.ListJobUsage(context.Background(), job.UsageListOpts{}) + if err != nil { + t.Fatalf("ListJobUsage: %v", err) + } + + if len(got) != 1 { + t.Fatalf("a cancelled attempt recorded %d rows, want 1", len(got)) + } +} + +// TestNoUsageStoreIsNotAnError proves the capability is genuinely +// optional: a backend without it runs jobs unchanged. +func TestNoUsageStoreIsNotAnError(t *testing.T) { + ctx := context.Background() + + reg := job.NewRegistry() + job.RegisterDefinition(reg, job.NewDefinition("ok", + func(context.Context, struct{}) error { return nil })) + + s := memory.New() + + // storeWithoutUsage hides the memory store's UsageRecorder methods. + r := worker.NewRunner( + reg, + ext.NewRegistry(log.NewNoopLogger()), + storeWithoutUsage{s}, + nil, + backoff.NewExponential(time.Millisecond, time.Second), + nil, + log.NewNoopLogger(), + ) + + j := enqueued(t, s, "ok") + + if err := r.Execute(ctx, j); err != nil { + t.Fatalf("Execute against a store without usage support: %v", err) + } +} + +// storeWithoutUsage embeds the job.Store interface, so it satisfies +// job.Store by promotion while deliberately not satisfying +// job.UsageRecorder — whatever the concrete value beneath it can do. +type storeWithoutUsage struct { + job.Store +} From 6bdf0d0b6d1e882c21af71f5b24e7f3e993e5ab6 Mon Sep 17 00:00:00 2001 From: Rex Raphael Date: Sat, 15 Aug 2026 16:12:28 -0500 Subject: [PATCH 2/8] feat(postgres): persist job usage records The production backend is the one that actually needs to accumulate history, so it implements UsageRecorder first. The predicted resource set is stored as JSONB rather than scalar columns: unlike the job row nothing queries it, the table is read in bulk by whatever computes estimates, and a blob keeps custom resource keys intact without a column per key. Two indexes, one per reader: (name, recorded_at DESC) for an estimator reading one definition over a window, and recorded_at alone for the retention sweep. Purge selects before deleting because Postgres will not take a LIMIT on DELETE, which is also what lets a sweep over a table this size work in batches instead of locking it whole. --- store/postgres/migrations.go | 49 ++++++++ store/postgres/store.go | 17 +-- store/postgres/usage.go | 211 +++++++++++++++++++++++++++++++++++ store/postgres/usage_test.go | 19 ++++ 4 files changed, 288 insertions(+), 8 deletions(-) create mode 100644 store/postgres/usage.go create mode 100644 store/postgres/usage_test.go diff --git a/store/postgres/migrations.go b/store/postgres/migrations.go index 6165ee2..59a149c 100644 --- a/store/postgres/migrations.go +++ b/store/postgres/migrations.go @@ -689,6 +689,55 @@ func init() { DROP COLUMN IF EXISTS primary_input_hash`) }, }, + + // Create the job usage table. + &migrate.Migration{ + Name: "create_job_usage_table", + Version: "20261007120000", + Up: func(ctx context.Context, exec migrate.Executor) error { + if _, err := exec.Exec(ctx, ` + CREATE TABLE IF NOT EXISTS dispatch_job_usage ( + id TEXT PRIMARY KEY, + job_id TEXT NOT NULL, + name TEXT NOT NULL, + queue TEXT NOT NULL DEFAULT 'default', + attempt INTEGER NOT NULL DEFAULT 0, + status TEXT NOT NULL, + input_bytes BIGINT NOT NULL DEFAULT 0, + resources JSONB, + wall_time_ns BIGINT NOT NULL DEFAULT 0, + cpu_time_ns BIGINT NOT NULL DEFAULT 0, + peak_rss BIGINT NOT NULL DEFAULT 0, + disk_written BIGINT NOT NULL DEFAULT 0, + executor TEXT, + scope_app_id TEXT, + scope_org_id TEXT, + worker_id TEXT, + recorded_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + )`); err != nil { + return err + } + + // The estimator reads by definition over a window; the + // retention sweep reads by age alone. + if _, err := exec.Exec(ctx, ` + CREATE INDEX IF NOT EXISTS idx_dispatch_job_usage_name_time + ON dispatch_job_usage (name, recorded_at DESC)`); err != nil { + return err + } + + _, err := exec.Exec(ctx, ` + CREATE INDEX IF NOT EXISTS idx_dispatch_job_usage_recorded + ON dispatch_job_usage (recorded_at)`) + + return err + }, + Down: func(ctx context.Context, exec migrate.Executor) error { + _, err := exec.Exec(ctx, `DROP TABLE IF EXISTS dispatch_job_usage`) + + return err + }, + }, ) } diff --git a/store/postgres/store.go b/store/postgres/store.go index 15e74fc..bdf7e93 100644 --- a/store/postgres/store.go +++ b/store/postgres/store.go @@ -22,14 +22,15 @@ import ( // Ensure Store implements all subsystem interfaces at compile time. var ( - _ job.Store = (*Store)(nil) - _ job.LeaseStore = (*Store)(nil) - _ workflow.Store = (*Store)(nil) - _ cron.Store = (*Store)(nil) - _ dlq.Store = (*Store)(nil) - _ event.Store = (*Store)(nil) - _ cluster.Store = (*Store)(nil) - _ artifact.Store = (*Store)(nil) + _ job.Store = (*Store)(nil) + _ job.LeaseStore = (*Store)(nil) + _ workflow.Store = (*Store)(nil) + _ cron.Store = (*Store)(nil) + _ dlq.Store = (*Store)(nil) + _ event.Store = (*Store)(nil) + _ cluster.Store = (*Store)(nil) + _ artifact.Store = (*Store)(nil) + _ job.UsageRecorder = (*Store)(nil) ) // Store is a grove ORM implementation of store.Store using PostgreSQL dialect. diff --git a/store/postgres/usage.go b/store/postgres/usage.go new file mode 100644 index 0000000..14ffc3b --- /dev/null +++ b/store/postgres/usage.go @@ -0,0 +1,211 @@ +package postgres + +import ( + "context" + "fmt" + "time" + + "github.com/xraph/grove" + + "github.com/xraph/dispatch/id" + "github.com/xraph/dispatch/job" + "github.com/xraph/dispatch/resource" +) + +// jobUsageModel is one attempt's measurements. +// +// The predicted resource set is stored as JSON rather than scalar columns +// because, unlike the job row, nothing queries it: this table is read in +// bulk by whatever computes estimates, and a blob keeps custom resource +// keys intact without a column per key. +type jobUsageModel struct { + grove.BaseModel `grove:"table:dispatch_job_usage"` + + ID string `grove:"id,pk"` + JobID string `grove:"job_id,notnull"` + Name string `grove:"name,notnull"` + Queue string `grove:"queue,notnull"` + Attempt int `grove:"attempt,notnull,default:0"` + Status string `grove:"status,notnull"` + InputBytes int64 `grove:"input_bytes,notnull,default:0"` + Resources []byte `grove:"resources,type:jsonb"` + WallTimeNS int64 `grove:"wall_time_ns,notnull,default:0"` + CPUTimeNS int64 `grove:"cpu_time_ns,notnull,default:0"` + PeakRSS int64 `grove:"peak_rss,notnull,default:0"` + DiskWritten int64 `grove:"disk_written,notnull,default:0"` + Executor string `grove:"executor"` + ScopeAppID string `grove:"scope_app_id"` + ScopeOrgID string `grove:"scope_org_id"` + WorkerID string `grove:"worker_id"` + RecordedAt time.Time `grove:"recorded_at,notnull,default:current_timestamp"` +} + +func toUsageModel(u *job.Usage) (*jobUsageModel, error) { + res, err := resource.EncodeSet(u.Resources) + if err != nil { + return nil, fmt.Errorf("dispatch/postgres: encode usage resources: %w", err) + } + + return &jobUsageModel{ + ID: u.ID.String(), + JobID: u.JobID.String(), + Name: u.Name, + Queue: u.Queue, + Attempt: u.Attempt, + Status: string(u.Status), + InputBytes: u.InputBytes, + Resources: res, + WallTimeNS: int64(u.WallTime), + CPUTimeNS: int64(u.CPUTime), + PeakRSS: u.PeakRSS, + DiskWritten: u.DiskWritten, + Executor: u.Executor, + ScopeAppID: u.ScopeAppID, + ScopeOrgID: u.ScopeOrgID, + WorkerID: u.WorkerID.String(), + RecordedAt: u.RecordedAt, + }, nil +} + +func fromUsageModel(m *jobUsageModel) (*job.Usage, error) { + uid, err := id.ParseUsageID(m.ID) + if err != nil { + return nil, err + } + + jid, err := id.ParseJobID(m.JobID) + if err != nil { + return nil, err + } + + res, err := resource.DecodeSet(m.Resources) + if err != nil { + return nil, fmt.Errorf("dispatch/postgres: decode usage resources: %w", err) + } + + u := &job.Usage{ + ID: uid, + JobID: jid, + Name: m.Name, + Queue: m.Queue, + Attempt: m.Attempt, + Status: job.State(m.Status), + InputBytes: m.InputBytes, + Resources: res, + WallTime: time.Duration(m.WallTimeNS), + CPUTime: time.Duration(m.CPUTimeNS), + PeakRSS: m.PeakRSS, + DiskWritten: m.DiskWritten, + Executor: m.Executor, + ScopeAppID: m.ScopeAppID, + ScopeOrgID: m.ScopeOrgID, + RecordedAt: m.RecordedAt, + } + + // A worker id is absent on any attempt recorded outside a pool. + if m.WorkerID != "" { + wid, werr := id.ParseWorkerID(m.WorkerID) + if werr != nil { + return nil, werr + } + + u.WorkerID = wid + } + + return u, nil +} + +// RecordJobUsage persists one attempt's measurements. +func (s *Store) RecordJobUsage(ctx context.Context, u *job.Usage) error { + m, err := toUsageModel(u) + if err != nil { + return err + } + + if _, err := s.pgdb.NewInsert(m).Exec(ctx); err != nil { + return fmt.Errorf("dispatch/postgres: record job usage: %w", err) + } + + return nil +} + +// ListJobUsage returns recorded attempts, most recent first. +func (s *Store) ListJobUsage(ctx context.Context, opts job.UsageListOpts) ([]*job.Usage, error) { + var models []jobUsageModel + + q := s.pgdb.NewSelect(&models) + + if opts.Name != "" { + q = q.Where("name = ?", opts.Name) + } + + if !opts.Since.IsZero() { + q = q.Where("recorded_at >= ?", opts.Since) + } + + q = q.OrderExpr("recorded_at DESC, id DESC") + + if opts.Limit > 0 { + q = q.Limit(opts.Limit) + } + + if opts.Offset > 0 { + q = q.Offset(opts.Offset) + } + + if err := q.Scan(ctx); err != nil { + return nil, fmt.Errorf("dispatch/postgres: list job usage: %w", err) + } + + out := make([]*job.Usage, 0, len(models)) + + for i := range models { + u, cerr := fromUsageModel(&models[i]) + if cerr != nil { + return nil, cerr + } + + out = append(out, u) + } + + return out, nil +} + +// PurgeJobUsage deletes records older than before, up to limit rows. +// +// Postgres will not take a LIMIT on DELETE, so the rows are selected +// first. That is also what makes the bound meaningful: a retention sweep +// over a table this size has to be able to work in batches rather than +// locking the whole thing in one statement. +func (s *Store) PurgeJobUsage(ctx context.Context, before time.Time, limit int) (int64, error) { + query := ` + DELETE FROM dispatch_job_usage + WHERE id IN ( + SELECT id FROM dispatch_job_usage + WHERE recorded_at < $1 + ORDER BY recorded_at ASC` + + args := []any{before} + + if limit > 0 { + query += ` + LIMIT $2` + + args = append(args, limit) + } + + query += ` + )` + + res, err := s.pgdb.Exec(ctx, query, args...) + if err != nil { + return 0, fmt.Errorf("dispatch/postgres: purge job usage: %w", err) + } + + n, err := res.RowsAffected() + if err != nil { + return 0, nil //nolint:nilerr // the rows are gone; the count is advisory + } + + return n, nil +} diff --git a/store/postgres/usage_test.go b/store/postgres/usage_test.go new file mode 100644 index 0000000..bf176dc --- /dev/null +++ b/store/postgres/usage_test.go @@ -0,0 +1,19 @@ +//go:build integration + +package postgres_test + +import ( + "testing" + + "github.com/xraph/dispatch/job" + "github.com/xraph/dispatch/job/jobtest" +) + +// TestUsageRecorderConformance runs the shared job.UsageRecorder suite +// against Postgres. Each subtest gets its own container because the +// suite asserts absolute row counts. +func TestUsageRecorderConformance(t *testing.T) { + jobtest.RunUsageRecorderSuite(t, func() job.UsageRecorder { + return setupTestStore(t) + }) +} From d302d6e7a7e437bf7e1504d19a25bb088426fdb0 Mon Sep 17 00:00:00 2001 From: Rex Raphael Date: Mon, 17 Aug 2026 10:23:12 -0500 Subject: [PATCH 3/8] feat(sqlite,mongo,redis): persist job usage records Completes the capability across every backend, so a fleet on any store accumulates the history an estimator needs rather than only the Postgres ones. Each backend expresses the same contract in its own terms. SQLite selects victims before deleting because it is usually built without DELETE ... LIMIT, which also keeps the batching identical to Postgres rather than dialect-dependent. Mongo deletes in one statement when unbounded and selects first when limited, since DeleteMany takes no limit. Redis has no query engine at all, so the ordering and both filters come from sorted sets written on record: one global, scored by time, for the newest-first read and the retention sweep, and one per definition for the read an estimator actually makes. Durations are stored as integer nanoseconds everywhere. Neither BSON nor SQLite has a duration type, and a consistent representation keeps the numbers comparable across backends. --- store/mongo/store.go | 24 ++- store/mongo/usage.go | 231 ++++++++++++++++++++++++++++ store/mongo/usage_test.go | 47 ++++++ store/redis/keys.go | 13 ++ store/redis/keys_internal_test.go | 16 ++ store/redis/store.go | 17 +- store/redis/usage.go | 248 ++++++++++++++++++++++++++++++ store/redis/usage_test.go | 18 +++ store/sqlite/migrations.go | 49 ++++++ store/sqlite/store.go | 17 +- store/sqlite/usage.go | 214 ++++++++++++++++++++++++++ store/sqlite/usage_test.go | 16 ++ 12 files changed, 886 insertions(+), 24 deletions(-) create mode 100644 store/mongo/usage.go create mode 100644 store/mongo/usage_test.go create mode 100644 store/redis/usage.go create mode 100644 store/redis/usage_test.go create mode 100644 store/sqlite/usage.go create mode 100644 store/sqlite/usage_test.go diff --git a/store/mongo/store.go b/store/mongo/store.go index 7a7c67b..5644e28 100644 --- a/store/mongo/store.go +++ b/store/mongo/store.go @@ -36,18 +36,20 @@ const ( colWorkers = "dispatch_workers" colArtifacts = "dispatch_artifacts" colArtifactLinks = "dispatch_artifact_links" + colJobUsage = "dispatch_job_usage" ) // Ensure Store implements all subsystem interfaces at compile time. var ( - _ job.Store = (*Store)(nil) - _ workflow.Store = (*Store)(nil) - _ cron.Store = (*Store)(nil) - _ dlq.Store = (*Store)(nil) - _ event.Store = (*Store)(nil) - _ cluster.Store = (*Store)(nil) - _ artifact.Store = (*Store)(nil) - _ job.LeaseStore = (*Store)(nil) + _ job.Store = (*Store)(nil) + _ workflow.Store = (*Store)(nil) + _ cron.Store = (*Store)(nil) + _ dlq.Store = (*Store)(nil) + _ event.Store = (*Store)(nil) + _ cluster.Store = (*Store)(nil) + _ artifact.Store = (*Store)(nil) + _ job.UsageRecorder = (*Store)(nil) + _ job.LeaseStore = (*Store)(nil) ) // Store is a grove ORM implementation of store.Store using MongoDB driver. @@ -152,6 +154,12 @@ func isDuplicateKey(err error) bool { // migrationIndexes returns the index definitions for all dispatch collections. func migrationIndexes() map[string][]mongod.IndexModel { return map[string][]mongod.IndexModel{ + colJobUsage: { + // The estimator reads one definition over a window; the + // retention sweep reads by age alone. + {Keys: bson.D{{Key: "name", Value: 1}, {Key: "recorded_at", Value: -1}}}, + {Keys: bson.D{{Key: "recorded_at", Value: 1}}}, + }, colArtifacts: { // Partial unique index on the storage coordinates: only live // rows collide, so a purged key becomes reusable. diff --git a/store/mongo/usage.go b/store/mongo/usage.go new file mode 100644 index 0000000..2820d8e --- /dev/null +++ b/store/mongo/usage.go @@ -0,0 +1,231 @@ +package mongo + +import ( + "context" + "fmt" + "time" + + "go.mongodb.org/mongo-driver/v2/bson" + "go.mongodb.org/mongo-driver/v2/mongo/options" + + "github.com/xraph/grove" + + "github.com/xraph/dispatch/id" + "github.com/xraph/dispatch/job" + "github.com/xraph/dispatch/resource" +) + +// jobUsageModel is one attempt's measurements. +// +// Durations are integer nanoseconds so the numbers survive a round trip +// unchanged: BSON has no duration type, and storing them as a document +// would make the aggregation an estimator wants harder for nothing. +type jobUsageModel struct { + grove.BaseModel `grove:"table:dispatch_job_usage"` + + ID string `grove:"id,pk" bson:"_id"` + JobID string `bson:"job_id"` + Name string `bson:"name"` + Queue string `bson:"queue"` + Attempt int `bson:"attempt"` + Status string `bson:"status"` + InputBytes int64 `bson:"input_bytes"` + Resources []byte `bson:"resources,omitempty"` + WallTimeNS int64 `bson:"wall_time_ns"` + CPUTimeNS int64 `bson:"cpu_time_ns"` + PeakRSS int64 `bson:"peak_rss"` + DiskWritten int64 `bson:"disk_written"` + Executor string `bson:"executor,omitempty"` + ScopeAppID string `bson:"scope_app_id,omitempty"` + ScopeOrgID string `bson:"scope_org_id,omitempty"` + WorkerID string `bson:"worker_id,omitempty"` + RecordedAt time.Time `bson:"recorded_at"` +} + +func toUsageModel(u *job.Usage) (*jobUsageModel, error) { + res, err := resource.EncodeSet(u.Resources) + if err != nil { + return nil, fmt.Errorf("dispatch/mongo: encode usage resources: %w", err) + } + + return &jobUsageModel{ + ID: u.ID.String(), + JobID: u.JobID.String(), + Name: u.Name, + Queue: u.Queue, + Attempt: u.Attempt, + Status: string(u.Status), + InputBytes: u.InputBytes, + Resources: res, + WallTimeNS: int64(u.WallTime), + CPUTimeNS: int64(u.CPUTime), + PeakRSS: u.PeakRSS, + DiskWritten: u.DiskWritten, + Executor: u.Executor, + ScopeAppID: u.ScopeAppID, + ScopeOrgID: u.ScopeOrgID, + WorkerID: u.WorkerID.String(), + RecordedAt: u.RecordedAt, + }, nil +} + +func fromUsageModel(m *jobUsageModel) (*job.Usage, error) { + uid, err := id.ParseUsageID(m.ID) + if err != nil { + return nil, err + } + + jid, err := id.ParseJobID(m.JobID) + if err != nil { + return nil, err + } + + res, err := resource.DecodeSet(m.Resources) + if err != nil { + return nil, fmt.Errorf("dispatch/mongo: decode usage resources: %w", err) + } + + u := &job.Usage{ + ID: uid, + JobID: jid, + Name: m.Name, + Queue: m.Queue, + Attempt: m.Attempt, + Status: job.State(m.Status), + InputBytes: m.InputBytes, + Resources: res, + WallTime: time.Duration(m.WallTimeNS), + CPUTime: time.Duration(m.CPUTimeNS), + PeakRSS: m.PeakRSS, + DiskWritten: m.DiskWritten, + Executor: m.Executor, + ScopeAppID: m.ScopeAppID, + ScopeOrgID: m.ScopeOrgID, + RecordedAt: m.RecordedAt, + } + + if m.WorkerID != "" { + wid, werr := id.ParseWorkerID(m.WorkerID) + if werr != nil { + return nil, werr + } + + u.WorkerID = wid + } + + return u, nil +} + +// RecordJobUsage persists one attempt's measurements. +func (s *Store) RecordJobUsage(ctx context.Context, u *job.Usage) error { + m, err := toUsageModel(u) + if err != nil { + return err + } + + if _, err := s.mdb.Collection(colJobUsage).InsertOne(ctx, m); err != nil { + return fmt.Errorf("dispatch/mongo: record job usage: %w", err) + } + + return nil +} + +// ListJobUsage returns recorded attempts, most recent first. +func (s *Store) ListJobUsage(ctx context.Context, opts job.UsageListOpts) ([]*job.Usage, error) { + filter := bson.M{} + + if opts.Name != "" { + filter["name"] = opts.Name + } + + if !opts.Since.IsZero() { + filter["recorded_at"] = bson.M{"$gte": opts.Since} + } + + find := options.Find(). + SetSort(bson.D{{Key: "recorded_at", Value: -1}, {Key: "_id", Value: -1}}) + + if opts.Limit > 0 { + find = find.SetLimit(int64(opts.Limit)) + } + + if opts.Offset > 0 { + find = find.SetSkip(int64(opts.Offset)) + } + + cur, err := s.mdb.Collection(colJobUsage).Find(ctx, filter, find) + if err != nil { + return nil, fmt.Errorf("dispatch/mongo: list job usage: %w", err) + } + + var models []jobUsageModel + if err := cur.All(ctx, &models); err != nil { + return nil, fmt.Errorf("dispatch/mongo: decode job usage: %w", err) + } + + out := make([]*job.Usage, 0, len(models)) + + for i := range models { + u, cerr := fromUsageModel(&models[i]) + if cerr != nil { + return nil, cerr + } + + out = append(out, u) + } + + return out, nil +} + +// PurgeJobUsage deletes records older than before, up to limit rows. +// +// DeleteMany takes no limit, so a bounded purge selects its victims +// first. An unbounded one deletes in a single statement, which is what +// the caller asked for by passing zero. +func (s *Store) PurgeJobUsage(ctx context.Context, before time.Time, limit int) (int64, error) { + col := s.mdb.Collection(colJobUsage) + filter := bson.M{"recorded_at": bson.M{"$lt": before}} + + if limit <= 0 { + res, err := col.DeleteMany(ctx, filter) + if err != nil { + return 0, fmt.Errorf("dispatch/mongo: purge job usage: %w", err) + } + + return res.DeletedCount, nil + } + + find := options.Find(). + SetSort(bson.D{{Key: "recorded_at", Value: 1}}). + SetLimit(int64(limit)). + SetProjection(bson.M{"_id": 1}) + + cur, err := col.Find(ctx, filter, find) + if err != nil { + return 0, fmt.Errorf("dispatch/mongo: purge job usage: %w", err) + } + + var victims []struct { + ID string `bson:"_id"` + } + + if aerr := cur.All(ctx, &victims); aerr != nil { + return 0, fmt.Errorf("dispatch/mongo: purge job usage: %w", aerr) + } + + if len(victims) == 0 { + return 0, nil + } + + ids := make([]string, 0, len(victims)) + for _, v := range victims { + ids = append(ids, v.ID) + } + + res, err := col.DeleteMany(ctx, bson.M{"_id": bson.M{"$in": ids}}) + if err != nil { + return 0, fmt.Errorf("dispatch/mongo: purge job usage: %w", err) + } + + return res.DeletedCount, nil +} diff --git a/store/mongo/usage_test.go b/store/mongo/usage_test.go new file mode 100644 index 0000000..8f62cc3 --- /dev/null +++ b/store/mongo/usage_test.go @@ -0,0 +1,47 @@ +package mongo_test + +import ( + "context" + "testing" + + "go.mongodb.org/mongo-driver/v2/bson" + mongod "go.mongodb.org/mongo-driver/v2/mongo" + "go.mongodb.org/mongo-driver/v2/mongo/options" + + "github.com/xraph/dispatch/job" + "github.com/xraph/dispatch/job/jobtest" +) + +// TestUsageRecorderConformance runs the shared job.UsageRecorder suite +// against MongoDB. One container serves every subtest; the collection is +// emptied between them because the suite asserts absolute counts. +func TestUsageRecorderConformance(t *testing.T) { + ctx := context.Background() + uri := startMongo(t) + store := openStore(t, uri) + + if err := store.Migrate(ctx); err != nil { + t.Fatalf("migrate: %v", err) + } + + client, err := mongod.Connect(options.Client().ApplyURI(uri)) + if err != nil { + t.Fatalf("connect raw mongo client: %v", err) + } + + t.Cleanup(func() { + if derr := client.Disconnect(ctx); derr != nil { + t.Errorf("disconnect: %v", derr) + } + }) + + col := client.Database(testDBName).Collection("dispatch_job_usage") + + jobtest.RunUsageRecorderSuite(t, func() job.UsageRecorder { + if _, derr := col.DeleteMany(ctx, bson.M{}); derr != nil { + t.Fatalf("clear usage collection: %v", derr) + } + + return store + }) +} diff --git a/store/redis/keys.go b/store/redis/keys.go index c643765..163d5e4 100644 --- a/store/redis/keys.go +++ b/store/redis/keys.go @@ -128,3 +128,16 @@ func (k keys) artifactLinks(artifactID string) string { func (k keys) ownerLinks(kind, ownerID string) string { return k.full(fmt.Sprintf("artifact_owner_links:%s:%s", kind, ownerID)) } + +// ── Job usage keys ── + +// usage returns the key for a usage record. +func (k keys) usage(id string) string { return k.full("usage:" + id) } + +// usageIndex is the Sorted Set of every usage record scored by record +// time. It drives the newest-first read and the retention sweep. +func (k keys) usageIndex() string { return k.full("usage_index") } + +// usageName is the per-definition Sorted Set, which is the read an +// estimator actually makes: one job name over a time window. +func (k keys) usageName(name string) string { return k.full("usage_name:" + name) } diff --git a/store/redis/keys_internal_test.go b/store/redis/keys_internal_test.go index 890240e..5e0b7df 100644 --- a/store/redis/keys_internal_test.go +++ b/store/redis/keys_internal_test.go @@ -21,3 +21,19 @@ func TestKeys_full(t *testing.T) { }) } } + +// Usage keys arrived after WithKeyPrefix and must not slip back to the +// unprefixed namespace: a shared Redis would then mix two tenants' usage +// history into one estimator read. +func TestKeys_usageCarriesPrefix(t *testing.T) { + k := newKeys("ws_acme:") + for got, want := range map[string]string{ + k.usage("u1"): "ws_acme:dispatch:usage:u1", + k.usageIndex(): "ws_acme:dispatch:usage_index", + k.usageName("resize"): "ws_acme:dispatch:usage_name:resize", + } { + if got != want { + t.Fatalf("usage key = %q, want %q", got, want) + } + } +} diff --git a/store/redis/store.go b/store/redis/store.go index 6b59f7f..f6629d1 100644 --- a/store/redis/store.go +++ b/store/redis/store.go @@ -25,14 +25,15 @@ import ( // Compile-time interface checks. var ( - _ job.Store = (*Store)(nil) - _ job.LeaseStore = (*Store)(nil) - _ workflow.Store = (*Store)(nil) - _ cron.Store = (*Store)(nil) - _ dlq.Store = (*Store)(nil) - _ event.Store = (*Store)(nil) - _ cluster.Store = (*Store)(nil) - _ artifact.Store = (*Store)(nil) + _ job.Store = (*Store)(nil) + _ job.LeaseStore = (*Store)(nil) + _ workflow.Store = (*Store)(nil) + _ cron.Store = (*Store)(nil) + _ dlq.Store = (*Store)(nil) + _ event.Store = (*Store)(nil) + _ cluster.Store = (*Store)(nil) + _ artifact.Store = (*Store)(nil) + _ job.UsageRecorder = (*Store)(nil) ) // Option configures the Store. diff --git a/store/redis/usage.go b/store/redis/usage.go new file mode 100644 index 0000000..140a7e3 --- /dev/null +++ b/store/redis/usage.go @@ -0,0 +1,248 @@ +package redis + +import ( + "context" + "fmt" + "strconv" + "time" + + goredis "github.com/redis/go-redis/v9" + + "github.com/xraph/dispatch/id" + "github.com/xraph/dispatch/job" + "github.com/xraph/dispatch/resource" +) + +// usageEntity is the JSON shape stored under a usage key. +type usageEntity struct { + ID string `json:"id"` + JobID string `json:"job_id"` + Name string `json:"name"` + Queue string `json:"queue"` + Attempt int `json:"attempt"` + Status string `json:"status"` + InputBytes int64 `json:"input_bytes"` + Resources []byte `json:"resources,omitempty"` + WallTimeNS int64 `json:"wall_time_ns"` + CPUTimeNS int64 `json:"cpu_time_ns"` + PeakRSS int64 `json:"peak_rss"` + DiskWritten int64 `json:"disk_written"` + Executor string `json:"executor,omitempty"` + ScopeAppID string `json:"scope_app_id,omitempty"` + ScopeOrgID string `json:"scope_org_id,omitempty"` + WorkerID string `json:"worker_id,omitempty"` + RecordedAt time.Time `json:"recorded_at"` +} + +func toUsageEntity(u *job.Usage) (*usageEntity, error) { + res, err := resource.EncodeSet(u.Resources) + if err != nil { + return nil, fmt.Errorf("dispatch/redis: encode usage resources: %w", err) + } + + return &usageEntity{ + ID: u.ID.String(), + JobID: u.JobID.String(), + Name: u.Name, + Queue: u.Queue, + Attempt: u.Attempt, + Status: string(u.Status), + InputBytes: u.InputBytes, + Resources: res, + WallTimeNS: int64(u.WallTime), + CPUTimeNS: int64(u.CPUTime), + PeakRSS: u.PeakRSS, + DiskWritten: u.DiskWritten, + Executor: u.Executor, + ScopeAppID: u.ScopeAppID, + ScopeOrgID: u.ScopeOrgID, + WorkerID: u.WorkerID.String(), + RecordedAt: u.RecordedAt, + }, nil +} + +func fromUsageEntity(e *usageEntity) (*job.Usage, error) { + uid, err := id.ParseUsageID(e.ID) + if err != nil { + return nil, err + } + + jid, err := id.ParseJobID(e.JobID) + if err != nil { + return nil, err + } + + res, err := resource.DecodeSet(e.Resources) + if err != nil { + return nil, fmt.Errorf("dispatch/redis: decode usage resources: %w", err) + } + + u := &job.Usage{ + ID: uid, + JobID: jid, + Name: e.Name, + Queue: e.Queue, + Attempt: e.Attempt, + Status: job.State(e.Status), + InputBytes: e.InputBytes, + Resources: res, + WallTime: time.Duration(e.WallTimeNS), + CPUTime: time.Duration(e.CPUTimeNS), + PeakRSS: e.PeakRSS, + DiskWritten: e.DiskWritten, + Executor: e.Executor, + ScopeAppID: e.ScopeAppID, + ScopeOrgID: e.ScopeOrgID, + RecordedAt: e.RecordedAt, + } + + if e.WorkerID != "" { + wid, werr := id.ParseWorkerID(e.WorkerID) + if werr != nil { + return nil, werr + } + + u.WorkerID = wid + } + + return u, nil +} + +// RecordJobUsage persists one attempt's measurements. +// +// Redis has no query engine, so the ordering and the two filters the +// contract requires have to come from indexes written here: a sorted set +// scored by record time for the global view and the retention sweep, and +// one per definition for the read an estimator actually makes. +func (s *Store) RecordJobUsage(ctx context.Context, u *job.Usage) error { + e, err := toUsageEntity(u) + if err != nil { + return err + } + + key := u.ID.String() + + if serr := s.setEntity(ctx, s.keys.usage(key), e); serr != nil { + return fmt.Errorf("dispatch/redis: record job usage: %w", serr) + } + + score := float64(u.RecordedAt.UnixNano()) + member := goredis.Z{Score: score, Member: key} + + pipe := s.rdb.TxPipeline() + pipe.ZAdd(ctx, s.keys.usageIndex(), member) + pipe.ZAdd(ctx, s.keys.usageName(u.Name), member) + + if _, perr := pipe.Exec(ctx); perr != nil { + return fmt.Errorf("dispatch/redis: index job usage: %w", perr) + } + + return nil +} + +// ListJobUsage returns recorded attempts, most recent first. +func (s *Store) ListJobUsage(ctx context.Context, opts job.UsageListOpts) ([]*job.Usage, error) { + index := s.keys.usageIndex() + if opts.Name != "" { + index = s.keys.usageName(opts.Name) + } + + minScore := "-inf" + if !opts.Since.IsZero() { + minScore = strconv.FormatInt(opts.Since.UnixNano(), 10) + } + + // Newest first, which ZRevRangeByScore gives directly. + ids, err := s.rdb.ZRevRangeByScore(ctx, index, &goredis.ZRangeBy{ + Min: minScore, + Max: "+inf", + }).Result() + if err != nil { + return nil, fmt.Errorf("dispatch/redis: list job usage: %w", err) + } + + if opts.Offset > 0 { + if opts.Offset >= len(ids) { + return nil, nil + } + + ids = ids[opts.Offset:] + } + + if opts.Limit > 0 && opts.Limit < len(ids) { + ids = ids[:opts.Limit] + } + + out := make([]*job.Usage, 0, len(ids)) + + for _, got := range ids { + var e usageEntity + + if gerr := s.getEntity(ctx, s.keys.usage(got), &e); gerr != nil { + if isNotFound(gerr) { + // The index outlived the record. Nothing reads usage for + // correctness, so skip rather than fail the whole query. + continue + } + + return nil, fmt.Errorf("dispatch/redis: get job usage: %w", gerr) + } + + u, cerr := fromUsageEntity(&e) + if cerr != nil { + return nil, cerr + } + + out = append(out, u) + } + + return out, nil +} + +// PurgeJobUsage deletes records older than before, up to limit rows. +func (s *Store) PurgeJobUsage(ctx context.Context, before time.Time, limit int) (int64, error) { + ids, err := s.rdb.ZRangeByScore(ctx, s.keys.usageIndex(), &goredis.ZRangeBy{ + Min: "-inf", + Max: strconv.FormatInt(before.UnixNano(), 10), + }).Result() + if err != nil { + return 0, fmt.Errorf("dispatch/redis: purge job usage: %w", err) + } + + if limit > 0 && limit < len(ids) { + ids = ids[:limit] + } + + if len(ids) == 0 { + return 0, nil + } + + var removed int64 + + for _, got := range ids { + // The per-definition index is keyed by name, which lives on the + // record, so the record has to be read before it is dropped. + name := "" + + var e usageEntity + if gerr := s.getEntity(ctx, s.keys.usage(got), &e); gerr == nil { + name = e.Name + } + + pipe := s.rdb.TxPipeline() + pipe.Del(ctx, s.keys.usage(got)) + pipe.ZRem(ctx, s.keys.usageIndex(), got) + + if name != "" { + pipe.ZRem(ctx, s.keys.usageName(name), got) + } + + if _, perr := pipe.Exec(ctx); perr != nil { + return removed, fmt.Errorf("dispatch/redis: purge job usage: %w", perr) + } + + removed++ + } + + return removed, nil +} diff --git a/store/redis/usage_test.go b/store/redis/usage_test.go new file mode 100644 index 0000000..81f7574 --- /dev/null +++ b/store/redis/usage_test.go @@ -0,0 +1,18 @@ +//go:build integration + +package redis_test + +import ( + "testing" + + "github.com/xraph/dispatch/job" + "github.com/xraph/dispatch/job/jobtest" +) + +// TestUsageRecorderConformance runs the shared job.UsageRecorder suite +// against Redis. +func TestUsageRecorderConformance(t *testing.T) { + jobtest.RunUsageRecorderSuite(t, func() job.UsageRecorder { + return setupTestStore(t) + }) +} diff --git a/store/sqlite/migrations.go b/store/sqlite/migrations.go index 6459061..7e9eae7 100644 --- a/store/sqlite/migrations.go +++ b/store/sqlite/migrations.go @@ -614,6 +614,55 @@ func init() { return nil }, }, + + // Create the job usage table. + &migrate.Migration{ + Name: "create_job_usage_table", + Version: "20261007120000", + Up: func(ctx context.Context, exec migrate.Executor) error { + if _, err := exec.Exec(ctx, ` + CREATE TABLE IF NOT EXISTS dispatch_job_usage ( + id TEXT PRIMARY KEY, + job_id TEXT NOT NULL, + name TEXT NOT NULL, + queue TEXT NOT NULL DEFAULT 'default', + attempt INTEGER NOT NULL DEFAULT 0, + status TEXT NOT NULL, + input_bytes INTEGER NOT NULL DEFAULT 0, + resources TEXT, + wall_time_ns INTEGER NOT NULL DEFAULT 0, + cpu_time_ns INTEGER NOT NULL DEFAULT 0, + peak_rss INTEGER NOT NULL DEFAULT 0, + disk_written INTEGER NOT NULL DEFAULT 0, + executor TEXT, + scope_app_id TEXT, + scope_org_id TEXT, + worker_id TEXT, + recorded_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) + )`); err != nil { + return err + } + + // The estimator reads by definition over a window; the + // retention sweep reads by age alone. + if _, err := exec.Exec(ctx, ` + CREATE INDEX IF NOT EXISTS idx_dispatch_job_usage_name_time + ON dispatch_job_usage (name, recorded_at DESC)`); err != nil { + return err + } + + _, err := exec.Exec(ctx, ` + CREATE INDEX IF NOT EXISTS idx_dispatch_job_usage_recorded + ON dispatch_job_usage (recorded_at)`) + + return err + }, + Down: func(ctx context.Context, exec migrate.Executor) error { + _, err := exec.Exec(ctx, `DROP TABLE IF EXISTS dispatch_job_usage`) + + return err + }, + }, ) } diff --git a/store/sqlite/store.go b/store/sqlite/store.go index 97b45aa..7e412ed 100644 --- a/store/sqlite/store.go +++ b/store/sqlite/store.go @@ -25,14 +25,15 @@ import ( // Ensure Store implements all subsystem interfaces at compile time. var ( - _ job.Store = (*Store)(nil) - _ workflow.Store = (*Store)(nil) - _ cron.Store = (*Store)(nil) - _ dlq.Store = (*Store)(nil) - _ event.Store = (*Store)(nil) - _ cluster.Store = (*Store)(nil) - _ artifact.Store = (*Store)(nil) - _ job.LeaseStore = (*Store)(nil) + _ job.Store = (*Store)(nil) + _ workflow.Store = (*Store)(nil) + _ cron.Store = (*Store)(nil) + _ dlq.Store = (*Store)(nil) + _ event.Store = (*Store)(nil) + _ cluster.Store = (*Store)(nil) + _ artifact.Store = (*Store)(nil) + _ job.UsageRecorder = (*Store)(nil) + _ job.LeaseStore = (*Store)(nil) ) // Store is a grove ORM implementation of store.Store using SQLite dialect. diff --git a/store/sqlite/usage.go b/store/sqlite/usage.go new file mode 100644 index 0000000..be426aa --- /dev/null +++ b/store/sqlite/usage.go @@ -0,0 +1,214 @@ +package sqlite + +import ( + "context" + "fmt" + "time" + + "github.com/xraph/grove" + + "github.com/xraph/dispatch/id" + "github.com/xraph/dispatch/job" + "github.com/xraph/dispatch/resource" +) + +// jobUsageModel is one attempt's measurements. +// +// Durations are stored as integer nanoseconds rather than a SQLite +// interval type, which does not exist. The predicted resource set is a +// JSON string for the same reason it is JSON in Postgres: nothing +// queries it column-wise, and a blob keeps custom resource keys. +type jobUsageModel struct { + grove.BaseModel `grove:"table:dispatch_job_usage"` + + ID string `grove:"id,pk"` + JobID string `grove:"job_id,notnull"` + Name string `grove:"name,notnull"` + Queue string `grove:"queue,notnull"` + Attempt int `grove:"attempt,notnull,default:0"` + Status string `grove:"status,notnull"` + InputBytes int64 `grove:"input_bytes,notnull,default:0"` + Resources *string `grove:"resources"` + WallTimeNS int64 `grove:"wall_time_ns,notnull,default:0"` + CPUTimeNS int64 `grove:"cpu_time_ns,notnull,default:0"` + PeakRSS int64 `grove:"peak_rss,notnull,default:0"` + DiskWritten int64 `grove:"disk_written,notnull,default:0"` + Executor string `grove:"executor"` + ScopeAppID string `grove:"scope_app_id"` + ScopeOrgID string `grove:"scope_org_id"` + WorkerID string `grove:"worker_id"` + RecordedAt time.Time `grove:"recorded_at,notnull"` +} + +func toUsageModel(u *job.Usage) (*jobUsageModel, error) { + res, err := resource.EncodeSetString(u.Resources) + if err != nil { + return nil, fmt.Errorf("dispatch/sqlite: encode usage resources: %w", err) + } + + return &jobUsageModel{ + ID: u.ID.String(), + JobID: u.JobID.String(), + Name: u.Name, + Queue: u.Queue, + Attempt: u.Attempt, + Status: string(u.Status), + InputBytes: u.InputBytes, + Resources: res, + WallTimeNS: int64(u.WallTime), + CPUTimeNS: int64(u.CPUTime), + PeakRSS: u.PeakRSS, + DiskWritten: u.DiskWritten, + Executor: u.Executor, + ScopeAppID: u.ScopeAppID, + ScopeOrgID: u.ScopeOrgID, + WorkerID: u.WorkerID.String(), + RecordedAt: u.RecordedAt, + }, nil +} + +func fromUsageModel(m *jobUsageModel) (*job.Usage, error) { + uid, err := id.ParseUsageID(m.ID) + if err != nil { + return nil, err + } + + jid, err := id.ParseJobID(m.JobID) + if err != nil { + return nil, err + } + + res, err := resource.DecodeSetString(m.Resources) + if err != nil { + return nil, fmt.Errorf("dispatch/sqlite: decode usage resources: %w", err) + } + + u := &job.Usage{ + ID: uid, + JobID: jid, + Name: m.Name, + Queue: m.Queue, + Attempt: m.Attempt, + Status: job.State(m.Status), + InputBytes: m.InputBytes, + Resources: res, + WallTime: time.Duration(m.WallTimeNS), + CPUTime: time.Duration(m.CPUTimeNS), + PeakRSS: m.PeakRSS, + DiskWritten: m.DiskWritten, + Executor: m.Executor, + ScopeAppID: m.ScopeAppID, + ScopeOrgID: m.ScopeOrgID, + RecordedAt: m.RecordedAt, + } + + if m.WorkerID != "" { + wid, werr := id.ParseWorkerID(m.WorkerID) + if werr != nil { + return nil, werr + } + + u.WorkerID = wid + } + + return u, nil +} + +// RecordJobUsage persists one attempt's measurements. +func (s *Store) RecordJobUsage(ctx context.Context, u *job.Usage) error { + m, err := toUsageModel(u) + if err != nil { + return err + } + + if _, err := s.sdb.NewInsert(m).Exec(ctx); err != nil { + return fmt.Errorf("dispatch/sqlite: record job usage: %w", err) + } + + return nil +} + +// ListJobUsage returns recorded attempts, most recent first. +func (s *Store) ListJobUsage(ctx context.Context, opts job.UsageListOpts) ([]*job.Usage, error) { + var models []jobUsageModel + + q := s.sdb.NewSelect(&models) + + if opts.Name != "" { + q = q.Where("name = ?", opts.Name) + } + + if !opts.Since.IsZero() { + q = q.Where("recorded_at >= ?", opts.Since) + } + + q = q.OrderExpr("recorded_at DESC, id DESC") + + if opts.Limit > 0 { + q = q.Limit(opts.Limit) + } + + if opts.Offset > 0 { + q = q.Offset(opts.Offset) + } + + if err := q.Scan(ctx); err != nil { + return nil, fmt.Errorf("dispatch/sqlite: list job usage: %w", err) + } + + out := make([]*job.Usage, 0, len(models)) + + for i := range models { + u, cerr := fromUsageModel(&models[i]) + if cerr != nil { + return nil, cerr + } + + out = append(out, u) + } + + return out, nil +} + +// PurgeJobUsage deletes records older than before, up to limit rows. +// +// SQLite is usually built without DELETE ... LIMIT, so the victims are +// selected first and deleted by id. That also keeps the batching +// behaviour identical to Postgres rather than dialect-dependent. +func (s *Store) PurgeJobUsage(ctx context.Context, before time.Time, limit int) (int64, error) { + query := ` + SELECT id FROM dispatch_job_usage + WHERE recorded_at < ? + ORDER BY recorded_at ASC` + + args := []any{before} + + if limit > 0 { + query += ` + LIMIT ?` + + args = append(args, limit) + } + + ids, err := s.selectIDs(ctx, query, args...) + if err != nil { + return 0, fmt.Errorf("dispatch/sqlite: purge job usage: %w", err) + } + + if len(ids) == 0 { + return 0, nil + } + + del := `DELETE FROM dispatch_job_usage WHERE id IN (` + placeholders(len(ids)) + `)` + + delArgs := make([]any, 0, len(ids)) + for _, got := range ids { + delArgs = append(delArgs, got) + } + + if _, err := s.sdb.Exec(ctx, del, delArgs...); err != nil { + return 0, fmt.Errorf("dispatch/sqlite: purge job usage: %w", err) + } + + return int64(len(ids)), nil +} diff --git a/store/sqlite/usage_test.go b/store/sqlite/usage_test.go new file mode 100644 index 0000000..bdea4a7 --- /dev/null +++ b/store/sqlite/usage_test.go @@ -0,0 +1,16 @@ +package sqlite_test + +import ( + "testing" + + "github.com/xraph/dispatch/job" + "github.com/xraph/dispatch/job/jobtest" +) + +// TestUsageRecorderConformance runs the shared job.UsageRecorder suite +// against SQLite. +func TestUsageRecorderConformance(t *testing.T) { + jobtest.RunUsageRecorderSuite(t, func() job.UsageRecorder { + return openSqliteStore(t) + }) +} From 527f7d5a160481b05ac89d45fc4fa29e25e3dcef Mon Sep 17 00:00:00 2001 From: Rex Raphael Date: Mon, 17 Aug 2026 11:27:40 -0500 Subject: [PATCH 4/8] refactor(redis): route cron, lease, wake, and usage through grove kv First slice of removing the go-redis coupling, now that grove kv has sorted sets, sets, hashes, and a Store-level publish/subscribe. Where a TxPipeline collapsed several index writes into one round trip, the calls are now sequential and the comment says what a crash between them costs. In every case converted here the answer is a dangling index entry that the next read skips, not a half-written record: the entity is authoritative and the indexes are derived. Two capabilities keep go-redis in the package for now. Lease renewal and reclamation run Lua for their compare-and-set, and event streams use XADD/XRANGE; grove kv has no scripting or stream interface, so those are untouched rather than reimplemented less safely. --- store/redis/cron.go | 43 ++++++++++++++++++++-------------- store/redis/lease.go | 8 ++++--- store/redis/usage.go | 55 ++++++++++++++++++++++---------------------- store/redis/wake.go | 44 ++++++++--------------------------- 4 files changed, 69 insertions(+), 81 deletions(-) diff --git a/store/redis/cron.go b/store/redis/cron.go index 1f2c388..310cedc 100644 --- a/store/redis/cron.go +++ b/store/redis/cron.go @@ -83,11 +83,12 @@ func (s *Store) RegisterCron(ctx context.Context, entry *cron.Entry) error { key := s.keys.cron(eID) // Check for duplicate name. - existing, err := s.rdb.HGet(ctx, s.keys.cronNames(), entry.Name).Result() - if err != nil && !isRedisNil(err) { + existing, err := s.kv.HGet(ctx, s.keys.cronNames(), entry.Name) + if err != nil && !isNotFound(err) { return fmt.Errorf("dispatch/redis: register cron check name: %w", err) } - if existing != "" { + + if len(existing) > 0 { return dispatch.ErrDuplicateCron } @@ -96,11 +97,11 @@ func (s *Store) RegisterCron(ctx context.Context, entry *cron.Entry) error { return fmt.Errorf("dispatch/redis: register cron set: %w", setErr) } - pipe := s.rdb.TxPipeline() - pipe.SAdd(ctx, s.keys.cronIDs(), eID) - pipe.HSet(ctx, s.keys.cronNames(), entry.Name, eID) - _, err = pipe.Exec(ctx) - if err != nil { + if _, err = s.kv.SAdd(ctx, s.keys.cronIDs(), eID); err != nil { + return fmt.Errorf("dispatch/redis: register cron indexes: %w", err) + } + + if _, err = s.kv.HSet(ctx, s.keys.cronNames(), map[string][]byte{entry.Name: []byte(eID)}); err != nil { return fmt.Errorf("dispatch/redis: register cron indexes: %w", err) } return nil @@ -120,7 +121,7 @@ func (s *Store) GetCron(ctx context.Context, entryID id.CronID) (*cron.Entry, er // ListCrons returns all cron entries. func (s *Store) ListCrons(ctx context.Context) ([]*cron.Entry, error) { - ids, err := s.rdb.SMembers(ctx, s.keys.cronIDs()).Result() + ids, err := s.kv.SMembers(ctx, s.keys.cronIDs()) if err != nil { return nil, fmt.Errorf("dispatch/redis: list crons: %w", err) } @@ -244,15 +245,23 @@ func (s *Store) DeleteCron(ctx context.Context, entryID id.CronID) error { return fmt.Errorf("dispatch/redis: delete cron get: %w", err) } - pipe := s.rdb.TxPipeline() - pipe.Del(ctx, key) - pipe.SRem(ctx, s.keys.cronIDs(), eID) - if e.Name != "" { - pipe.HDel(ctx, s.keys.cronNames(), e.Name) - } - _, err := pipe.Exec(ctx) - if err != nil { + // The entity, the id index, and the name index are three writes with + // no transaction between them. Nothing reads a cron entry through the + // indexes alone, so a crash mid-delete leaves a dangling index entry + // that the next list skips rather than a half-deleted entry. + if err := s.kv.Delete(ctx, key); err != nil { return fmt.Errorf("dispatch/redis: delete cron: %w", err) } + + if _, err := s.kv.SRem(ctx, s.keys.cronIDs(), eID); err != nil { + return fmt.Errorf("dispatch/redis: delete cron index: %w", err) + } + + if e.Name != "" { + if _, err := s.kv.HDel(ctx, s.keys.cronNames(), e.Name); err != nil { + return fmt.Errorf("dispatch/redis: delete cron name index: %w", err) + } + } + return nil } diff --git a/store/redis/lease.go b/store/redis/lease.go index d0fee6b..07f006d 100644 --- a/store/redis/lease.go +++ b/store/redis/lease.go @@ -9,6 +9,8 @@ import ( goredis "github.com/redis/go-redis/v9" + kvdriver "github.com/xraph/grove/kv/driver" + "github.com/xraph/dispatch" "github.com/xraph/dispatch/id" "github.com/xraph/dispatch/job" @@ -270,7 +272,7 @@ func (s *Store) ReclaimExpiredLeases(ctx context.Context, limit int) ([]*job.Job t := now() - ids, err := s.rdb.SMembers(ctx, s.keys.jobIDs()).Result() + ids, err := s.kv.SMembers(ctx, s.keys.jobIDs()) if err != nil { return nil, fmt.Errorf("dispatch/redis: reclaim smembers: %w", err) } @@ -329,8 +331,8 @@ func (s *Store) ReclaimExpiredLeases(ctx context.Context, limit int) ([]*job.Job // even for a job that was enqueued straight into running (as the // conformance suite's RunningJob helper does) and was therefore // never popped in the first place. - zErr := s.rdb.ZAdd(ctx, s.keys.queue(after.Queue), - goredis.Z{Score: jobScore(after.Priority, after.RunAt), Member: jID}).Err() + _, zErr := s.kv.ZAdd(ctx, s.keys.queue(after.Queue), + kvdriver.ScoredMember{Score: jobScore(after.Priority, after.RunAt), Member: jID}) if zErr != nil { return nil, fmt.Errorf("dispatch/redis: reclaim requeue: %w", zErr) } diff --git a/store/redis/usage.go b/store/redis/usage.go index 140a7e3..e4a7e0b 100644 --- a/store/redis/usage.go +++ b/store/redis/usage.go @@ -3,10 +3,9 @@ package redis import ( "context" "fmt" - "strconv" "time" - goredis "github.com/redis/go-redis/v9" + kvdriver "github.com/xraph/grove/kv/driver" "github.com/xraph/dispatch/id" "github.com/xraph/dispatch/job" @@ -126,15 +125,18 @@ func (s *Store) RecordJobUsage(ctx context.Context, u *job.Usage) error { return fmt.Errorf("dispatch/redis: record job usage: %w", serr) } - score := float64(u.RecordedAt.UnixNano()) - member := goredis.Z{Score: score, Member: key} + member := kvdriver.ScoredMember{Score: float64(u.RecordedAt.UnixNano()), Member: key} - pipe := s.rdb.TxPipeline() - pipe.ZAdd(ctx, s.keys.usageIndex(), member) - pipe.ZAdd(ctx, s.keys.usageName(u.Name), member) + // The global index and the per-definition one are two writes. A crash + // between them leaves a record reachable through one and not the + // other, which costs a row in an estimate and nothing else -- usage is + // telemetry, so neither index is authoritative. + if _, err := s.kv.ZAdd(ctx, s.keys.usageIndex(), member); err != nil { + return fmt.Errorf("dispatch/redis: index job usage: %w", err) + } - if _, perr := pipe.Exec(ctx); perr != nil { - return fmt.Errorf("dispatch/redis: index job usage: %w", perr) + if _, err := s.kv.ZAdd(ctx, s.keys.usageName(u.Name), member); err != nil { + return fmt.Errorf("dispatch/redis: index job usage by name: %w", err) } return nil @@ -147,16 +149,13 @@ func (s *Store) ListJobUsage(ctx context.Context, opts job.UsageListOpts) ([]*jo index = s.keys.usageName(opts.Name) } - minScore := "-inf" + // Newest first, which ZRevRangeByScore gives directly. + spec := kvdriver.RangeSpec{Reverse: true} if !opts.Since.IsZero() { - minScore = strconv.FormatInt(opts.Since.UnixNano(), 10) + spec.Min, spec.HasMin = float64(opts.Since.UnixNano()), true } - // Newest first, which ZRevRangeByScore gives directly. - ids, err := s.rdb.ZRevRangeByScore(ctx, index, &goredis.ZRangeBy{ - Min: minScore, - Max: "+inf", - }).Result() + ids, err := s.kv.ZRange(ctx, index, spec) if err != nil { return nil, fmt.Errorf("dispatch/redis: list job usage: %w", err) } @@ -201,10 +200,10 @@ func (s *Store) ListJobUsage(ctx context.Context, opts job.UsageListOpts) ([]*jo // PurgeJobUsage deletes records older than before, up to limit rows. func (s *Store) PurgeJobUsage(ctx context.Context, before time.Time, limit int) (int64, error) { - ids, err := s.rdb.ZRangeByScore(ctx, s.keys.usageIndex(), &goredis.ZRangeBy{ - Min: "-inf", - Max: strconv.FormatInt(before.UnixNano(), 10), - }).Result() + ids, err := s.kv.ZRange(ctx, s.keys.usageIndex(), kvdriver.RangeSpec{ + Max: float64(before.UnixNano()), + HasMax: true, + }) if err != nil { return 0, fmt.Errorf("dispatch/redis: purge job usage: %w", err) } @@ -229,16 +228,18 @@ func (s *Store) PurgeJobUsage(ctx context.Context, before time.Time, limit int) name = e.Name } - pipe := s.rdb.TxPipeline() - pipe.Del(ctx, s.keys.usage(got)) - pipe.ZRem(ctx, s.keys.usageIndex(), got) + if derr := s.kv.Delete(ctx, s.keys.usage(got)); derr != nil { + return removed, fmt.Errorf("dispatch/redis: purge job usage: %w", derr) + } - if name != "" { - pipe.ZRem(ctx, s.keys.usageName(name), got) + if _, zerr := s.kv.ZRem(ctx, s.keys.usageIndex(), got); zerr != nil { + return removed, fmt.Errorf("dispatch/redis: purge job usage index: %w", zerr) } - if _, perr := pipe.Exec(ctx); perr != nil { - return removed, fmt.Errorf("dispatch/redis: purge job usage: %w", perr) + if name != "" { + if _, zerr := s.kv.ZRem(ctx, s.keys.usageName(name), got); zerr != nil { + return removed, fmt.Errorf("dispatch/redis: purge job usage name index: %w", zerr) + } } removed++ diff --git a/store/redis/wake.go b/store/redis/wake.go index 70b8618..7dba4fe 100644 --- a/store/redis/wake.go +++ b/store/redis/wake.go @@ -14,48 +14,24 @@ var _ dispatchstore.WakeNotifier = (*Store)(nil) // failed publish only costs poll latency and is not worth failing the // enqueue over. func (s *Store) notifyWake(ctx context.Context) { - _ = s.rdb.Publish(ctx, s.keys.wakeChannel(), "").Err() //nolint:errcheck // best-effort: polling covers missed wakes + _ = s.kv.Publish(ctx, s.keys.wakeChannel(), nil) //nolint:errcheck // best-effort: polling covers missed wakes } // StartWakeListener subscribes to the dispatch wake channel and invokes -// wake for each message. go-redis re-subscribes automatically after -// connection loss, so no manual rebuild loop is needed; messages published -// while the connection was down are simply lost, which polling covers. The -// returned stop function terminates the subscriber and blocks until it has -// exited. +// wake for each message. +// +// The driver re-subscribes after connection loss, so there is no manual +// rebuild loop; messages published while the connection was down are +// simply lost, which polling covers. The returned stop function cancels +// the subscription. func (s *Store) StartWakeListener(ctx context.Context, wake func()) (func(), error) { ctx, cancel := context.WithCancel(ctx) - sub := s.rdb.Subscribe(ctx, s.keys.wakeChannel()) - // Confirm the subscription is established so callers know push is - // live before relying on it. - if _, err := sub.Receive(ctx); err != nil { + if err := s.kv.Subscribe(ctx, s.keys.wakeChannel(), func([]byte) { wake() }); err != nil { cancel() - _ = sub.Close() + return nil, fmt.Errorf("dispatch/redis: start wake listener: %w", err) } - done := make(chan struct{}) - go func() { - defer close(done) - ch := sub.Channel() - for { - select { - case <-ctx.Done(): - return - case _, ok := <-ch: - if !ok { - return - } - wake() - } - } - }() - - stop := func() { - cancel() - _ = sub.Close() - <-done - } - return stop, nil + return cancel, nil } From 96685182efe40e11946fe5340ca3dff3574f724e Mon Sep 17 00:00:00 2001 From: Rex Raphael Date: Mon, 17 Aug 2026 13:38:29 -0500 Subject: [PATCH 5/8] refactor(redis): run leases and event streams through grove kv The two paths that could not be converted before, now that grove kv has script and stream capabilities. Lease renewal and reclamation keep their Lua verbatim -- the compare-and-set is the correctness core of lease handoff and was never the problem; only the transport was. The scripts are plain strings passed to kv.Eval rather than go-redis Script values, and scriptInt reads the result, treating anything unexpected as a loss because the safe answer to whether this caller won a contested claim is no. Event streams move to kv.XAdd/XRange. The poll batch is now a named constant with the reason attached: a waiter wants the first match, so a larger batch costs latency on every miss and buys nothing on a hit. --- store/redis/event.go | 22 +++++++++------- store/redis/lease.go | 62 ++++++++++++++++++++++++++++---------------- 2 files changed, 52 insertions(+), 32 deletions(-) diff --git a/store/redis/event.go b/store/redis/event.go index 678b425..4189f95 100644 --- a/store/redis/event.go +++ b/store/redis/event.go @@ -5,8 +5,6 @@ import ( "fmt" "time" - goredis "github.com/redis/go-redis/v9" - "github.com/xraph/dispatch" "github.com/xraph/dispatch/event" "github.com/xraph/dispatch/id" @@ -53,6 +51,11 @@ func fromEventEntity(e *eventEntity) (*event.Event, error) { }, nil } +// eventScanBatch bounds one poll of an event stream. A waiter is looking +// for the first match, not draining the stream, so reading more per pass +// costs latency on every miss and buys nothing on a hit. +const eventScanBatch = 10 + // PublishEvent persists a new event and adds it to the name's stream. func (s *Store) PublishEvent(ctx context.Context, evt *event.Event) error { eID := evt.ID.String() @@ -64,12 +67,9 @@ func (s *Store) PublishEvent(ctx context.Context, evt *event.Event) error { } // Add to the named stream so subscribers get notified. - if err := s.rdb.XAdd(ctx, &goredis.XAddArgs{ - Stream: s.keys.eventStream(evt.Name), - Values: map[string]interface{}{ - "event_id": eID, - }, - }).Err(); err != nil { + if _, err := s.kv.XAdd(ctx, s.keys.eventStream(evt.Name), map[string][]byte{ + "event_id": []byte(eID), + }); err != nil { return fmt.Errorf("dispatch/redis: publish event stream: %w", err) } return nil @@ -93,17 +93,19 @@ func (s *Store) SubscribeEvent(ctx context.Context, name string, timeout time.Du } // Read oldest messages from the stream. - msgs, err := s.rdb.XRangeN(ctx, stream, "-", "+", 10).Result() + msgs, err := s.kv.XRange(ctx, stream, "", "", eventScanBatch) if err != nil { return nil, fmt.Errorf("dispatch/redis: subscribe xrange: %w", err) } for _, msg := range msgs { - eID, ok := msg.Values["event_id"].(string) + raw, ok := msg.Values["event_id"] if !ok { continue } + eID := string(raw) + key := s.keys.event(eID) var e eventEntity if getErr := s.getEntity(ctx, key, &e); getErr != nil { diff --git a/store/redis/lease.go b/store/redis/lease.go index 07f006d..01df9a7 100644 --- a/store/redis/lease.go +++ b/store/redis/lease.go @@ -3,12 +3,9 @@ package redis import ( "context" "encoding/json" - "errors" "fmt" "time" - goredis "github.com/redis/go-redis/v9" - kvdriver "github.com/xraph/grove/kv/driver" "github.com/xraph/dispatch" @@ -112,7 +109,7 @@ import ( // complete updated entity, pre-serialized by Go (see the file comment // above for why Lua never re-serializes it itself). // Returns 1 on renewal, 0 when the lease is no longer held. -var renewLeaseScript = goredis.NewScript(` +const renewLeaseScript = ` local raw = redis.call('GET', KEYS[1]) if not raw then return 0 @@ -129,7 +126,7 @@ if tostring(j.lease_epoch) ~= ARGV[2] then end redis.call('SET', KEYS[1], ARGV[3]) return 1 -`) +` // reclaimScript resets one job to pending only if it is still running at // the expected epoch. @@ -148,7 +145,7 @@ return 1 // above for why Lua never mutates or re-serializes it itself). // Returns 1 when this caller took the job, 0 when someone else did (or // the job moved out of running between Go's read and this script). -var reclaimScript = goredis.NewScript(` +const reclaimScript = ` local raw = redis.call('GET', KEYS[1]) if not raw then return 0 @@ -162,7 +159,7 @@ if tostring(j.lease_epoch) ~= ARGV[1] then end redis.call('SET', KEYS[1], ARGV[2]) return 1 -`) +` // The grant is not in this file: it travels on job.DequeueOpts and is // applied by claimCandidates, in the same read-modify-write that writes @@ -206,16 +203,17 @@ func (s *Store) RenewLease( return fmt.Errorf("dispatch/redis: renew lease marshal: %w", marshalErr) } - res, err := renewLeaseScript.Run(ctx, s.rdb, + raw, err := s.kv.Eval(ctx, renewLeaseScript, []string{key}, workerID.String(), epoch, blob, - ).Int64() - if err != nil && !errors.Is(err, goredis.Nil) { + ) + if err != nil { return fmt.Errorf("dispatch/redis: renew lease: %w", err) } - if res != 1 { + + if scriptInt(raw) != 1 { return job.ErrLeaseLost } @@ -362,16 +360,36 @@ func (s *Store) ReclaimExpiredLeases(ctx context.Context, limit int) ([]*job.Job // found. Go no longer needs to ask Redis what the row says; it already // knows, because it wrote it. func (s *Store) claimExpired(ctx context.Context, jID string, epoch int, blob []byte) (bool, error) { - res, err := reclaimScript.Run(ctx, s.rdb, + raw, err := s.kv.Eval(ctx, reclaimScript, []string{s.keys.job(jID)}, epoch, blob, - ).Int64() - if err != nil && !errors.Is(err, goredis.Nil) { + ) + if err != nil { return false, fmt.Errorf("dispatch/redis: reclaim claim: %w", err) } - return res == 1, nil + return scriptInt(raw) == 1, nil +} + +// scriptInt reads the integer a Lua script returned. +// +// Both scripts here answer with 1 or 0, and a script that returned +// nothing at all yields nil, which is not a failure: it means the guard +// did not match, so the caller did not win. Anything unexpected reads as +// zero for the same reason -- these are guards, and the safe answer to +// "did I win?" is no. +func scriptInt(v any) int64 { + switch typed := v.(type) { + case int64: + return typed + case int: + return int64(typed) + case float64: + return int64(typed) + default: + return 0 + } } // UpdateLeasedJob persists j only while the caller still holds the @@ -450,22 +468,22 @@ func (s *Store) UpdateLeasedJob(ctx context.Context, j *job.Job, workerID id.Wor runnable := job.State(next.State) == job.StatePending || job.State(next.State) == job.StateRetrying if runnable { - z := goredis.Z{Score: jobScore(next.Priority, next.RunAt), Member: jID} - if zErr := s.rdb.ZAdd(ctx, qk, z).Err(); zErr != nil { + z := kvdriver.ScoredMember{Score: jobScore(next.Priority, next.RunAt), Member: jID} + if _, zErr := s.kv.ZAdd(ctx, qk, z); zErr != nil { return fmt.Errorf("dispatch/redis: update leased job index add: %w", zErr) } } - res, err := renewLeaseScript.Run(ctx, s.rdb, + raw, err := s.kv.Eval(ctx, renewLeaseScript, []string{key}, workerID.String(), epoch, blob, - ).Int64() - if err != nil && !errors.Is(err, goredis.Nil) { + ) + if err != nil { return fmt.Errorf("dispatch/redis: update leased job: %w", err) } - if res != 1 { + if scriptInt(raw) != 1 { // The read above found the row, so a failed compare-and-set here // means the lease moved on between that read and the script // running — not that the row is gone. dispatch.ErrJobNotFound is @@ -476,7 +494,7 @@ func (s *Store) UpdateLeasedJob(ctx context.Context, j *job.Job, workerID id.Wor } if !runnable { - if zErr := s.rdb.ZRem(ctx, qk, jID).Err(); zErr != nil { + if _, zErr := s.kv.ZRem(ctx, qk, jID); zErr != nil { return fmt.Errorf("dispatch/redis: update leased job index remove: %w", zErr) } } From 75ba0ab21b15a96078fcfd2de41e7791819be179 Mon Sep 17 00:00:00 2001 From: Rex Raphael Date: Mon, 17 Aug 2026 13:47:38 -0500 Subject: [PATCH 6/8] refactor(redis)!: complete the grove kv conversion No production file in store/redis imports go-redis any more. The queue's sorted sets, the id sets, the artifact link hashes, the lease scripts, and the event streams all go through kv.Store, so the backend is whichever driver the caller opened rather than Redis by construction. Two changes are behavioural rather than mechanical. The dequeue claim issues one ZRem per candidate instead of one pipelined batch. Pipeline was never providing atomicity -- it was a Pipeline, not a TxPipeline -- so a single ZRem returning 1 still means this worker took the job, and the claim semantics are identical. What it costs is a round trip per candidate, bounded by the dequeue limit. The batch entity read keeps its single round trip: kv MGetRaw pipelines its GETs rather than issuing MGET, which is also why it stays correct against a cluster, where one queue's job keys span slots. Everywhere a TxPipeline collapsed several writes, the comment now says what a crash between them costs. In every case it is an index entry pointing at a record that is gone, which reads already skip, because the entity is authoritative and the indexes are derived. New: MissingCapabilities reports at construction which capabilities a driver lacks, so a mis-chosen backend says so at startup instead of failing the first dequeue with ErrNotSupported. --- store/redis/artifact.go | 100 +++++++++++++++++++++++----------------- store/redis/cluster.go | 72 +++++++++++++++++------------ store/redis/dequeue.go | 58 ++++++++++++----------- store/redis/dlq.go | 17 +++---- store/redis/job.go | 56 +++++++++++++--------- store/redis/store.go | 60 +++++++++++++++++++----- store/redis/workflow.go | 14 +++--- 7 files changed, 230 insertions(+), 147 deletions(-) diff --git a/store/redis/artifact.go b/store/redis/artifact.go index a5df9ee..872c2b8 100644 --- a/store/redis/artifact.go +++ b/store/redis/artifact.go @@ -9,7 +9,8 @@ import ( "strconv" "time" - goredis "github.com/redis/go-redis/v9" + "github.com/xraph/grove/kv" + kvdriver "github.com/xraph/grove/kv/driver" "github.com/xraph/dispatch/artifact" "github.com/xraph/dispatch/id" @@ -132,23 +133,25 @@ func linkField(name string, attempt int) string { func (s *Store) CreateArtifact(ctx context.Context, a *artifact.Artifact, link *artifact.Link) error { guard := s.keys.artifactGuard(a.Backend, a.Bucket, a.Key) - ok, err := s.rdb.SetNX(ctx, guard, a.ID.String(), 0).Result() + // The guard is claimed with a set-if-absent, which is what makes two + // concurrent creates at the same coordinates resolve to one winner. + err := s.kv.SetRaw(ctx, guard, []byte(a.ID.String()), kv.WithNX()) if err != nil { - return fmt.Errorf("dispatch/redis: create artifact guard: %w", err) - } + if errors.Is(err, kv.ErrConflict) { + return artifact.ErrExists + } - if !ok { - return artifact.ErrExists + return fmt.Errorf("dispatch/redis: create artifact guard: %w", err) } if err := s.setEntity(ctx, s.keys.artifact(a.ID.String()), toArtifactEntity(a)); err != nil { // Release the guard so the coordinates are not permanently burned. - s.rdb.Del(ctx, guard) + _ = s.kv.Delete(ctx, guard) //nolint:errcheck // releasing the guard is best effort return fmt.Errorf("dispatch/redis: create artifact: %w", err) } - if err := s.rdb.SAdd(ctx, s.keys.artifactIDs(), a.ID.String()).Err(); err != nil { + if _, err := s.kv.SAdd(ctx, s.keys.artifactIDs(), a.ID.String()); err != nil { return fmt.Errorf("dispatch/redis: index artifact: %w", err) } @@ -157,7 +160,8 @@ func (s *Store) CreateArtifact(ctx context.Context, a *artifact.Artifact, link * // no durable artifact is ever a member. if a.Lifecycle == artifact.Ephemeral { score := float64(a.CreatedAt.UnixNano()) - if err := s.rdb.ZAdd(ctx, s.keys.artifactEphemeral(), goredis.Z{Score: score, Member: a.ID.String()}).Err(); err != nil { + if _, err := s.kv.ZAdd(ctx, s.keys.artifactEphemeral(), + kvdriver.ScoredMember{Score: score, Member: a.ID.String()}); err != nil { return fmt.Errorf("dispatch/redis: index ephemeral artifact: %w", err) } } @@ -200,12 +204,12 @@ func (s *Store) loadArtifact(ctx context.Context, artifactID string) (*artifact. // FindArtifactByKey retrieves a live artifact by its storage coordinates. func (s *Store) FindArtifactByKey(ctx context.Context, backend, bucket, key string) (*artifact.Artifact, error) { - got, err := s.rdb.Get(ctx, s.keys.artifactGuard(backend, bucket, key)).Result() + got, err := s.kv.GetRaw(ctx, s.keys.artifactGuard(backend, bucket, key)) if err != nil { return nil, artifact.ErrNotFound } - return s.GetArtifact(ctx, id.MustParse(got)) + return s.GetArtifact(ctx, id.MustParse(string(got))) } // UpdateArtifact persists size, hash, content type, and expiry. Lifecycle, @@ -230,7 +234,7 @@ func (s *Store) UpdateArtifact(ctx context.Context, a *artifact.Artifact) error // ListArtifacts returns artifacts matching the given options, newest first. func (s *Store) ListArtifacts(ctx context.Context, opts artifact.ListOpts) ([]*artifact.Artifact, error) { - ids, err := s.rdb.SMembers(ctx, s.keys.artifactIDs()).Result() + ids, err := s.kv.SMembers(ctx, s.keys.artifactIDs()) if err != nil { return nil, fmt.Errorf("dispatch/redis: list artifact ids: %w", err) } @@ -300,12 +304,13 @@ func (s *Store) LinkArtifact(ctx context.Context, link *artifact.Link) error { owner := artifact.OwnerRef{Kind: link.OwnerKind, ID: link.OwnerID} field := linkField(link.Name, link.Attempt) - if err := s.rdb.HSet(ctx, s.keys.ownerLinks(string(owner.Kind), owner.ID), field, raw).Err(); err != nil { + if _, err := s.kv.HSet(ctx, s.keys.ownerLinks(string(owner.Kind), owner.ID), + map[string][]byte{field: raw}); err != nil { return fmt.Errorf("dispatch/redis: link artifact: %w", err) } member := string(link.OwnerKind) + "\x00" + link.OwnerID + "\x00" + field - if err := s.rdb.SAdd(ctx, s.keys.artifactLinks(link.ArtifactID.String()), member).Err(); err != nil { + if _, err := s.kv.SAdd(ctx, s.keys.artifactLinks(link.ArtifactID.String()), member); err != nil { return fmt.Errorf("dispatch/redis: index artifact link: %w", err) } @@ -314,7 +319,7 @@ func (s *Store) LinkArtifact(ctx context.Context, link *artifact.Link) error { // ListLinks returns every link belonging to the given owner. func (s *Store) ListLinks(ctx context.Context, owner artifact.OwnerRef) ([]*artifact.Link, error) { - vals, err := s.rdb.HGetAll(ctx, s.keys.ownerLinks(string(owner.Kind), owner.ID)).Result() + vals, err := s.kv.HGetAll(ctx, s.keys.ownerLinks(string(owner.Kind), owner.ID)) if err != nil { return nil, fmt.Errorf("dispatch/redis: list links: %w", err) } @@ -323,7 +328,7 @@ func (s *Store) ListLinks(ctx context.Context, owner artifact.OwnerRef) ([]*arti for _, raw := range vals { var e linkEntity - if uerr := json.Unmarshal([]byte(raw), &e); uerr != nil { + if uerr := json.Unmarshal(raw, &e); uerr != nil { return nil, fmt.Errorf("dispatch/redis: unmarshal link: %w", uerr) } @@ -449,7 +454,7 @@ func (s *Store) SweepEphemeral( limit = defaultSweepLimit } - ids, err := s.rdb.ZRange(ctx, s.keys.artifactEphemeral(), 0, -1).Result() + ids, err := s.kv.ZRange(ctx, s.keys.artifactEphemeral(), kvdriver.RangeSpec{}) if err != nil { return nil, fmt.Errorf("dispatch/redis: sweep ephemeral: %w", err) } @@ -515,7 +520,7 @@ func (s *Store) SweepEphemeral( // linksForArtifact resolves every link pointing at an artifact. func (s *Store) linksForArtifact(ctx context.Context, artifactID string) ([]*artifact.Link, error) { - members, err := s.rdb.SMembers(ctx, s.keys.artifactLinks(artifactID)).Result() + members, err := s.kv.SMembers(ctx, s.keys.artifactLinks(artifactID)) if err != nil { return nil, fmt.Errorf("dispatch/redis: list artifact links: %w", err) } @@ -530,13 +535,13 @@ func (s *Store) linksForArtifact(ctx context.Context, artifactID string) ([]*art owner := artifact.OwnerRef{Kind: artifact.OwnerKind(kind), ID: ownerID} - raw, herr := s.rdb.HGet(ctx, s.keys.ownerLinks(string(owner.Kind), owner.ID), field).Result() + raw, herr := s.kv.HGet(ctx, s.keys.ownerLinks(string(owner.Kind), owner.ID), field) if herr != nil { continue } var e linkEntity - if uerr := json.Unmarshal([]byte(raw), &e); uerr != nil { + if uerr := json.Unmarshal(raw, &e); uerr != nil { return nil, fmt.Errorf("dispatch/redis: unmarshal link: %w", uerr) } @@ -668,10 +673,10 @@ func (s *Store) SweepOrphans( // The ephemeral index is scored by creation time, so the cutoff is a // range query rather than a scan. - ids, err := s.rdb.ZRangeByScore(ctx, s.keys.artifactEphemeral(), &goredis.ZRangeBy{ - Min: "-inf", - Max: strconv.FormatInt(cutoff.UnixNano(), 10), - }).Result() + ids, err := s.kv.ZRange(ctx, s.keys.artifactEphemeral(), kvdriver.RangeSpec{ + Max: float64(cutoff.UnixNano()), + HasMax: true, + }) if err != nil { return nil, fmt.Errorf("dispatch/redis: sweep orphans: %w", err) } @@ -700,7 +705,7 @@ func (s *Store) SweepOrphans( continue } - n, cerr := s.rdb.SCard(ctx, s.keys.artifactLinks(a.ID.String())).Result() + n, cerr := s.kv.SCard(ctx, s.keys.artifactLinks(a.ID.String())) if cerr != nil { return nil, fmt.Errorf("dispatch/redis: count artifact links: %w", cerr) } @@ -743,14 +748,14 @@ func (s *Store) markDeleted( // Release the live-key guard so the coordinates become reusable, // and index the deletion time for the purge pass. - s.rdb.Del(ctx, s.keys.artifactGuard(clone.Backend, clone.Bucket, clone.Key)) + _ = s.kv.Delete(ctx, s.keys.artifactGuard(clone.Backend, clone.Bucket, clone.Key)) //nolint:errcheck // releasing the guard is best effort - if err := s.rdb.ZAdd(ctx, s.keys.artifactDeleted(), - goredis.Z{Score: float64(at.UnixNano()), Member: clone.ID.String()}).Err(); err != nil { + if _, err := s.kv.ZAdd(ctx, s.keys.artifactDeleted(), + kvdriver.ScoredMember{Score: float64(at.UnixNano()), Member: clone.ID.String()}); err != nil { return nil, fmt.Errorf("dispatch/redis: index deleted artifact: %w", err) } - if err := s.rdb.ZRem(ctx, s.keys.artifactEphemeral(), clone.ID.String()).Err(); err != nil { + if _, err := s.kv.ZRem(ctx, s.keys.artifactEphemeral(), clone.ID.String()); err != nil { return nil, fmt.Errorf("dispatch/redis: deindex ephemeral artifact: %w", err) } @@ -772,10 +777,10 @@ func (s *Store) ListPurgeable( cutoff := time.Now().UTC().Add(-grace) - ids, err := s.rdb.ZRangeByScore(ctx, s.keys.artifactDeleted(), &goredis.ZRangeBy{ - Min: "-inf", - Max: strconv.FormatInt(cutoff.UnixNano(), 10), - }).Result() + ids, err := s.kv.ZRange(ctx, s.keys.artifactDeleted(), kvdriver.RangeSpec{ + Max: float64(cutoff.UnixNano()), + HasMax: true, + }) if err != nil { return nil, fmt.Errorf("dispatch/redis: list purgeable: %w", err) } @@ -817,26 +822,35 @@ func (s *Store) PurgeArtifact(ctx context.Context, artifactID id.ArtifactID) err for _, l := range links { owner := artifact.OwnerRef{Kind: l.OwnerKind, ID: l.OwnerID} - if herr := s.rdb.HDel(ctx, s.keys.ownerLinks(string(owner.Kind), owner.ID), linkField(l.Name, l.Attempt)).Err(); herr != nil { + if _, herr := s.kv.HDel(ctx, s.keys.ownerLinks(string(owner.Kind), owner.ID), + linkField(l.Name, l.Attempt)); herr != nil { return fmt.Errorf("dispatch/redis: purge artifact link: %w", herr) } } a, err := s.loadArtifact(ctx, key) if err == nil { - s.rdb.Del(ctx, s.keys.artifactGuard(a.Backend, a.Bucket, a.Key)) + _ = s.kv.Delete(ctx, s.keys.artifactGuard(a.Backend, a.Bucket, a.Key)) //nolint:errcheck // releasing the guard is best effort } - pipe := s.rdb.TxPipeline() - pipe.Del(ctx, s.keys.artifact(key)) - pipe.Del(ctx, s.keys.artifactLinks(key)) - pipe.SRem(ctx, s.keys.artifactIDs(), key) - pipe.ZRem(ctx, s.keys.artifactEphemeral(), key) - pipe.ZRem(ctx, s.keys.artifactDeleted(), key) - - if _, err := pipe.Exec(ctx); err != nil { + // Purge is the end of a two-phase deletion: the artifact was already + // soft-deleted and its bytes are gone, so these writes only reclaim + // index space. A crash partway leaves entries pointing at a record + // that no longer exists, which every read already skips, and the next + // purge pass finishes the job. + if err := s.kv.Delete(ctx, s.keys.artifact(key), s.keys.artifactLinks(key)); err != nil { return fmt.Errorf("dispatch/redis: purge artifact: %w", err) } + if _, err := s.kv.SRem(ctx, s.keys.artifactIDs(), key); err != nil { + return fmt.Errorf("dispatch/redis: purge artifact id index: %w", err) + } + + for _, index := range []string{s.keys.artifactEphemeral(), s.keys.artifactDeleted()} { + if _, err := s.kv.ZRem(ctx, index, key); err != nil { + return fmt.Errorf("dispatch/redis: purge artifact index: %w", err) + } + } + return nil } diff --git a/store/redis/cluster.go b/store/redis/cluster.go index 45f0668..30cd85d 100644 --- a/store/redis/cluster.go +++ b/store/redis/cluster.go @@ -2,9 +2,12 @@ package redis import ( "context" + "errors" "fmt" "time" + "github.com/xraph/grove/kv" + "github.com/xraph/dispatch" "github.com/xraph/dispatch/cluster" "github.com/xraph/dispatch/id" @@ -70,7 +73,7 @@ func (s *Store) RegisterWorker(ctx context.Context, w *cluster.Worker) error { return fmt.Errorf("dispatch/redis: register worker set: %w", err) } - if err := s.rdb.SAdd(ctx, s.keys.workerIDs(), wID).Err(); err != nil { + if _, err := s.kv.SAdd(ctx, s.keys.workerIDs(), wID); err != nil { return fmt.Errorf("dispatch/redis: register worker index: %w", err) } return nil @@ -89,13 +92,14 @@ func (s *Store) DeregisterWorker(ctx context.Context, workerID id.WorkerID) erro return dispatch.ErrWorkerNotFound } - pipe := s.rdb.TxPipeline() - pipe.Del(ctx, key) - pipe.SRem(ctx, s.keys.workerIDs(), wID) - _, err = pipe.Exec(ctx) - if err != nil { + if err = s.kv.Delete(ctx, key); err != nil { return fmt.Errorf("dispatch/redis: deregister worker: %w", err) } + + if _, err = s.kv.SRem(ctx, s.keys.workerIDs(), wID); err != nil { + return fmt.Errorf("dispatch/redis: deregister worker index: %w", err) + } + return nil } @@ -117,7 +121,7 @@ func (s *Store) HeartbeatWorker(ctx context.Context, workerID id.WorkerID) error // ListWorkers returns all registered workers. func (s *Store) ListWorkers(ctx context.Context) ([]*cluster.Worker, error) { - ids, err := s.rdb.SMembers(ctx, s.keys.workerIDs()).Result() + ids, err := s.kv.SMembers(ctx, s.keys.workerIDs()) if err != nil { return nil, fmt.Errorf("dispatch/redis: list workers: %w", err) } @@ -142,7 +146,7 @@ func (s *Store) ListWorkers(ctx context.Context) ([]*cluster.Worker, error) { func (s *Store) DeleteStaleWorkers(ctx context.Context, threshold time.Duration) (int64, error) { cutoff := now().Add(-threshold) - ids, err := s.rdb.SMembers(ctx, s.keys.workerIDs()).Result() + ids, err := s.kv.SMembers(ctx, s.keys.workerIDs()) if err != nil { return 0, fmt.Errorf("dispatch/redis: delete stale smembers: %w", err) } @@ -154,17 +158,18 @@ func (s *Store) DeleteStaleWorkers(ctx context.Context, threshold time.Duration) // Orphaned set member without a backing entity — treat as // stale and remove from the index. if isNotFound(getErr) { - if remErr := s.rdb.SRem(ctx, s.keys.workerIDs(), wID).Err(); remErr == nil { + if _, remErr := s.kv.SRem(ctx, s.keys.workerIDs(), wID); remErr == nil { deleted++ } } continue } if e.LastSeen.Before(cutoff) { - pipe := s.rdb.TxPipeline() - pipe.Del(ctx, s.keys.worker(wID)) - pipe.SRem(ctx, s.keys.workerIDs(), wID) - if _, execErr := pipe.Exec(ctx); execErr == nil { + if execErr := s.kv.Delete(ctx, s.keys.worker(wID)); execErr != nil { + continue + } + + if _, execErr := s.kv.SRem(ctx, s.keys.workerIDs(), wID); execErr == nil { deleted++ } } @@ -176,7 +181,7 @@ func (s *Store) DeleteStaleWorkers(ctx context.Context, threshold time.Duration) func (s *Store) ReapDeadWorkers(ctx context.Context, threshold time.Duration) ([]*cluster.Worker, error) { cutoff := now().Add(-threshold) - ids, err := s.rdb.SMembers(ctx, s.keys.workerIDs()).Result() + ids, err := s.kv.SMembers(ctx, s.keys.workerIDs()) if err != nil { return nil, fmt.Errorf("dispatch/redis: reap smembers: %w", err) } @@ -212,10 +217,18 @@ func (s *Store) AcquireLeadership(ctx context.Context, workerID id.WorkerID, ttl return false, dispatch.ErrWorkerNotFound } - // Try SET NX with TTL (atomic acquire). - ok, err := s.rdb.SetNX(ctx, s.keys.leader(), wID, ttl).Result() - if err != nil { - return false, fmt.Errorf("dispatch/redis: acquire leadership setnx: %w", err) + // Set-if-absent with a TTL is the atomic acquire: a conflict means + // someone else already holds it, which is an answer rather than a + // failure. + setErr := s.kv.SetRaw(ctx, s.keys.leader(), []byte(wID), kv.WithNX(), kv.WithTTL(ttl)) + + ok := setErr == nil + if setErr != nil && errors.Is(setErr, kv.ErrConflict) { + setErr = nil + } + + if setErr != nil { + return false, fmt.Errorf("dispatch/redis: acquire leadership setnx: %w", setErr) } if ok { // We got the lock -- update worker fields. @@ -230,13 +243,14 @@ func (s *Store) AcquireLeadership(ctx context.Context, workerID id.WorkerID, ttl } // Check if we already hold it. - current, err := s.rdb.Get(ctx, s.keys.leader()).Result() - if err != nil && !isRedisNil(err) { + currentRaw, err := s.kv.GetRaw(ctx, s.keys.leader()) + if err != nil && !isNotFound(err) { return false, fmt.Errorf("dispatch/redis: acquire leadership get: %w", err) } - if current == wID { + + if string(currentRaw) == wID { // Re-acquire: extend TTL. - _ = s.rdb.Expire(ctx, s.keys.leader(), ttl).Err() //nolint:errcheck // best-effort + _ = s.kv.Expire(ctx, s.keys.leader(), ttl) //nolint:errcheck // best-effort until := now().Add(ttl) var e workerEntity if getErr := s.getEntity(ctx, wKey, &e); getErr == nil { @@ -254,18 +268,18 @@ func (s *Store) AcquireLeadership(ctx context.Context, workerID id.WorkerID, ttl func (s *Store) RenewLeadership(ctx context.Context, workerID id.WorkerID, ttl time.Duration) (bool, error) { wID := workerID.String() - current, err := s.rdb.Get(ctx, s.keys.leader()).Result() + currentRaw, err := s.kv.GetRaw(ctx, s.keys.leader()) if err != nil { - if isRedisNil(err) { + if isNotFound(err) { return false, nil // no leader } return false, fmt.Errorf("dispatch/redis: renew leadership get: %w", err) } - if current != wID { + if string(currentRaw) != wID { return false, nil // not the leader } - _ = s.rdb.Expire(ctx, s.keys.leader(), ttl).Err() //nolint:errcheck // best-effort + _ = s.kv.Expire(ctx, s.keys.leader(), ttl) //nolint:errcheck // best-effort until := now().Add(ttl) var e workerEntity if getErr := s.getEntity(ctx, s.keys.worker(wID), &e); getErr == nil { @@ -277,16 +291,16 @@ func (s *Store) RenewLeadership(ctx context.Context, workerID id.WorkerID, ttl t // GetLeader returns the current cluster leader, or nil if there is no leader. func (s *Store) GetLeader(ctx context.Context) (*cluster.Worker, error) { - wID, err := s.rdb.Get(ctx, s.keys.leader()).Result() + wIDRaw, err := s.kv.GetRaw(ctx, s.keys.leader()) if err != nil { - if isRedisNil(err) { + if isNotFound(err) { return nil, nil // no leader } return nil, fmt.Errorf("dispatch/redis: get leader: %w", err) } var e workerEntity - if getErr := s.getEntity(ctx, s.keys.worker(wID), &e); getErr != nil { + if getErr := s.getEntity(ctx, s.keys.worker(string(wIDRaw)), &e); getErr != nil { return nil, nil // leader key exists but worker gone } return fromWorkerEntity(&e) diff --git a/store/redis/dequeue.go b/store/redis/dequeue.go index 116fcdb..99348f9 100644 --- a/store/redis/dequeue.go +++ b/store/redis/dequeue.go @@ -3,12 +3,11 @@ package redis import ( "context" "encoding/json" - "errors" "fmt" "sort" "time" - goredis "github.com/redis/go-redis/v9" + kvdriver "github.com/xraph/grove/kv/driver" "github.com/xraph/dispatch/job" ) @@ -243,13 +242,16 @@ func (s *Store) scanQueue( ) for { - stop := int64(-1) // the whole index + // An unbounded score range paged by offset visits members in the + // same order as a rank range, so this is the index scan it + // replaces -- expressed in the one range form kv exposes. + spec := kvdriver.RangeSpec{Offset: scanned} if !full { - stop = scanned + window - 1 + spec.Count = window } - ids, err := s.rdb.ZRange(ctx, key, scanned, stop).Result() - if err != nil && !isRedisNil(err) { + ids, err := s.kv.ZRange(ctx, key, spec) + if err != nil && !isNotFound(err) { return nil, fmt.Errorf("dispatch/redis: dequeue scan %q: %w", q, err) } @@ -349,26 +351,25 @@ func (s *Store) eligibleIn( // against a clustered client, where the job keys of one queue are spread // over many slots. func (s *Store) readJobEntities(ctx context.Context, ids []string) ([]*jobEntity, error) { - pipe := s.rdb.Pipeline() - - cmds := make([]*goredis.StringCmd, len(ids)) + keys := make([]string, len(ids)) for i, jID := range ids { - cmds[i] = pipe.Get(ctx, s.keys.job(jID)) + keys[i] = s.keys.job(jID) } - // A missing key makes Exec report goredis.Nil for the batch as a - // whole; the per-command results below distinguish the misses, and a - // job that vanished between the index read and this one is simply not - // a candidate. - if _, err := pipe.Exec(ctx); err != nil && !errors.Is(err, goredis.Nil) { + // MGetRaw pipelines its GETs rather than issuing one MGET, so this + // stays a single round trip and stays correct against a clustered + // client, where one queue's job keys are spread over many slots. A + // job that vanished between the index read and this one comes back + // nil and is simply not a candidate. + raws, err := s.kv.MGetRaw(ctx, keys) + if err != nil { return nil, fmt.Errorf("dispatch/redis: dequeue read entities: %w", err) } out := make([]*jobEntity, len(ids)) - for i, cmd := range cmds { - raw, err := cmd.Bytes() - if err != nil { + for i, raw := range raws { + if raw == nil { continue } @@ -425,23 +426,28 @@ func (s *Store) claimCandidates( opts job.DequeueOpts, candidates []dequeueCandidate, ) ([]*job.Job, error) { - pipe := s.rdb.Pipeline() + // Each ZRem is its own round trip now rather than one pipelined + // batch. The pipeline was never providing atomicity -- it was a + // Pipeline, not a TxPipeline -- so the claim semantics are unchanged: + // a single ZRem is atomic on its own, and returning 1 still means + // this worker took the job. What it costs is one round trip per + // candidate instead of one per batch, bounded by the dequeue limit. + wins := make([]bool, len(candidates)) - rems := make([]*goredis.IntCmd, len(candidates)) for i, c := range candidates { - rems[i] = pipe.ZRem(ctx, s.keys.queue(c.queue), c.id) - } + won, err := s.kv.ZRem(ctx, s.keys.queue(c.queue), c.id) + if err != nil { + return nil, fmt.Errorf("dispatch/redis: dequeue claim: %w", err) + } - if _, err := pipe.Exec(ctx); err != nil { - return nil, fmt.Errorf("dispatch/redis: dequeue claim: %w", err) + wins[i] = won == 1 } t := now() claimed := make([]*job.Job, 0, len(candidates)) for i, c := range candidates { - won, err := rems[i].Result() - if err != nil || won != 1 { + if !wins[i] { continue // another worker removed it first } diff --git a/store/redis/dlq.go b/store/redis/dlq.go index d79863b..c50f486 100644 --- a/store/redis/dlq.go +++ b/store/redis/dlq.go @@ -115,7 +115,7 @@ func (s *Store) PushDLQ(ctx context.Context, entry *dlq.Entry) error { return fmt.Errorf("dispatch/redis: push dlq set: %w", err) } - if err := s.rdb.SAdd(ctx, s.keys.dlqIDs(), eID).Err(); err != nil { + if _, err := s.kv.SAdd(ctx, s.keys.dlqIDs(), eID); err != nil { return fmt.Errorf("dispatch/redis: push dlq index: %w", err) } return nil @@ -123,7 +123,7 @@ func (s *Store) PushDLQ(ctx context.Context, entry *dlq.Entry) error { // ListDLQ returns DLQ entries matching the given options. func (s *Store) ListDLQ(ctx context.Context, opts dlq.ListOpts) ([]*dlq.Entry, error) { - ids, err := s.rdb.SMembers(ctx, s.keys.dlqIDs()).Result() + ids, err := s.kv.SMembers(ctx, s.keys.dlqIDs()) if err != nil { return nil, fmt.Errorf("dispatch/redis: list dlq: %w", err) } @@ -177,7 +177,7 @@ func (s *Store) ReplayDLQ(ctx context.Context, entryID id.DLQID) error { // PurgeDLQ removes DLQ entries with FailedAt before the given time. func (s *Store) PurgeDLQ(ctx context.Context, before time.Time) (int64, error) { - ids, err := s.rdb.SMembers(ctx, s.keys.dlqIDs()).Result() + ids, err := s.kv.SMembers(ctx, s.keys.dlqIDs()) if err != nil { return 0, fmt.Errorf("dispatch/redis: purge dlq smembers: %w", err) } @@ -191,12 +191,13 @@ func (s *Store) PurgeDLQ(ctx context.Context, before time.Time) (int64, error) { } if e.FailedAt.Before(before) { - pipe := s.rdb.TxPipeline() - pipe.Del(ctx, key) - pipe.SRem(ctx, s.keys.dlqIDs(), eID) - if _, pErr := pipe.Exec(ctx); pErr != nil { + if pErr := s.kv.Delete(ctx, key); pErr != nil { return purged, fmt.Errorf("dispatch/redis: purge dlq del: %w", pErr) } + + if _, pErr := s.kv.SRem(ctx, s.keys.dlqIDs(), eID); pErr != nil { + return purged, fmt.Errorf("dispatch/redis: purge dlq index: %w", pErr) + } purged++ } } @@ -205,7 +206,7 @@ func (s *Store) PurgeDLQ(ctx context.Context, before time.Time) (int64, error) { // CountDLQ returns the total number of entries in the dead letter queue. func (s *Store) CountDLQ(ctx context.Context) (int64, error) { - count, err := s.rdb.SCard(ctx, s.keys.dlqIDs()).Result() + count, err := s.kv.SCard(ctx, s.keys.dlqIDs()) if err != nil { return 0, fmt.Errorf("dispatch/redis: count dlq: %w", err) } diff --git a/store/redis/job.go b/store/redis/job.go index ed325d0..4799ccc 100644 --- a/store/redis/job.go +++ b/store/redis/job.go @@ -7,7 +7,7 @@ import ( "strconv" "time" - goredis "github.com/redis/go-redis/v9" + kvdriver "github.com/xraph/grove/kv/driver" "github.com/xraph/dispatch" "github.com/xraph/dispatch/id" @@ -200,16 +200,21 @@ func (s *Store) EnqueueJob(ctx context.Context, j *job.Job) error { return fmt.Errorf("dispatch/redis: enqueue set entity: %w", setErr) } - pipe := s.rdb.TxPipeline() - pipe.SAdd(ctx, s.keys.jobIDs(), jID) + // The entity is written above and is authoritative; these two index + // writes follow it. A crash between them leaves the job in the id set + // but off its queue, where the stale-job reaper finds it -- the same + // state a crash between a pipelined SADD and ZADD produced, since + // this was never a transaction that either index write could roll + // back. + if _, err = s.kv.SAdd(ctx, s.keys.jobIDs(), jID); err != nil { + return fmt.Errorf("dispatch/redis: enqueue job indexes: %w", err) + } - // Add to queue sorted set: score = priority (negated for DESC) + time component. + // Queue score is priority (negated for DESC) plus a time component. score := jobScore(j.Priority, j.RunAt) - pipe.ZAdd(ctx, s.keys.queue(j.Queue), goredis.Z{Score: score, Member: jID}) - - _, err = pipe.Exec(ctx) - if err != nil { - return fmt.Errorf("dispatch/redis: enqueue job indexes: %w", err) + if _, err = s.kv.ZAdd(ctx, s.keys.queue(j.Queue), + kvdriver.ScoredMember{Score: score, Member: jID}); err != nil { + return fmt.Errorf("dispatch/redis: enqueue job queue index: %w", err) } s.notifyWake(ctx) return nil @@ -284,8 +289,8 @@ func (s *Store) UpdateJob(ctx context.Context, j *job.Job) error { runnable := j.State == job.StatePending || j.State == job.StateRetrying if runnable { - z := goredis.Z{Score: jobScore(j.Priority, j.RunAt), Member: jID} - if zErr := s.rdb.ZAdd(ctx, qk, z).Err(); zErr != nil { + z := kvdriver.ScoredMember{Score: jobScore(j.Priority, j.RunAt), Member: jID} + if _, zErr := s.kv.ZAdd(ctx, qk, z); zErr != nil { return fmt.Errorf("dispatch/redis: update job index add: %w", zErr) } } @@ -295,7 +300,7 @@ func (s *Store) UpdateJob(ctx context.Context, j *job.Job) error { } if !runnable { - if zErr := s.rdb.ZRem(ctx, qk, jID).Err(); zErr != nil { + if _, zErr := s.kv.ZRem(ctx, qk, jID); zErr != nil { return fmt.Errorf("dispatch/redis: update job index remove: %w", zErr) } } @@ -317,21 +322,28 @@ func (s *Store) DeleteJob(ctx context.Context, jobID id.JobID) error { return fmt.Errorf("dispatch/redis: delete job get: %w", err) } - // Delete entity via raw Redis DEL (KV store may not have Delete). - pipe := s.rdb.TxPipeline() - pipe.Del(ctx, key) - pipe.SRem(ctx, s.keys.jobIDs(), jID) - pipe.ZRem(ctx, s.keys.queue(e.Queue), jID) - _, err := pipe.Exec(ctx) - if err != nil { + // Entity first, then its indexes. A crash partway leaves an index + // entry pointing at a key that is gone, which every read already + // tolerates -- the entity is authoritative and a missing one means + // the job does not exist. + if err := s.kv.Delete(ctx, key); err != nil { return fmt.Errorf("dispatch/redis: delete job: %w", err) } + + if _, err := s.kv.SRem(ctx, s.keys.jobIDs(), jID); err != nil { + return fmt.Errorf("dispatch/redis: delete job id index: %w", err) + } + + if _, err := s.kv.ZRem(ctx, s.keys.queue(e.Queue), jID); err != nil { + return fmt.Errorf("dispatch/redis: delete job queue index: %w", err) + } + return nil } // ListJobsByState returns jobs matching the given state. func (s *Store) ListJobsByState(ctx context.Context, state job.State, opts job.ListOpts) ([]*job.Job, error) { - ids, err := s.rdb.SMembers(ctx, s.keys.jobIDs()).Result() + ids, err := s.kv.SMembers(ctx, s.keys.jobIDs()) if err != nil { return nil, fmt.Errorf("dispatch/redis: list jobs smembers: %w", err) } @@ -379,7 +391,7 @@ func (s *Store) HeartbeatJob(ctx context.Context, jobID id.JobID, _ id.WorkerID) func (s *Store) ReapStaleJobs(ctx context.Context, threshold time.Duration) ([]*job.Job, error) { cutoff := now().Add(-threshold) - ids, err := s.rdb.SMembers(ctx, s.keys.jobIDs()).Result() + ids, err := s.kv.SMembers(ctx, s.keys.jobIDs()) if err != nil { return nil, fmt.Errorf("dispatch/redis: reap smembers: %w", err) } @@ -411,7 +423,7 @@ func (s *Store) ReapStaleJobs(ctx context.Context, threshold time.Duration) ([]* // CountJobs returns the number of jobs matching the given options. func (s *Store) CountJobs(ctx context.Context, opts job.CountOpts) (int64, error) { - ids, err := s.rdb.SMembers(ctx, s.keys.jobIDs()).Result() + ids, err := s.kv.SMembers(ctx, s.keys.jobIDs()) if err != nil { return 0, fmt.Errorf("dispatch/redis: count smembers: %w", err) } diff --git a/store/redis/store.go b/store/redis/store.go index f6629d1..1669ca0 100644 --- a/store/redis/store.go +++ b/store/redis/store.go @@ -5,14 +5,12 @@ import ( "encoding/json" "errors" "fmt" + "strings" "time" - goredis "github.com/redis/go-redis/v9" - log "github.com/xraph/go-utils/log" "github.com/xraph/grove/kv" - "github.com/xraph/grove/kv/drivers/redisdriver" "github.com/xraph/dispatch/artifact" "github.com/xraph/dispatch/cluster" @@ -52,11 +50,15 @@ func WithKeyPrefix(prefix string) Option { return func(s *Store) { s.keys = newKeys(prefix) } } -// Store implements the composite store.Store interface backed by Redis -// via Grove KV. +// Store implements the composite store.Store interface over Grove KV. +// +// Every operation goes through kv.Store, including the sorted sets, +// hashes, scripts, and streams the job queue depends on. Nothing here +// imports a Redis client, so the backend this runs against is whichever +// kv driver the caller opened -- provided it supports those capabilities, +// which Store checks at construction. type Store struct { kv *kv.Store - rdb goredis.UniversalClient keys keys logger log.Logger } @@ -66,16 +68,55 @@ type Store struct { func New(store *kv.Store, opts ...Option) *Store { s := &Store{ kv: store, - rdb: redisdriver.UnwrapClient(store), keys: newKeys(""), logger: log.NewNoopLogger(), } for _, o := range opts { o(s) } + + if missing := MissingCapabilities(store); len(missing) > 0 { + // A driver without these cannot run jobs: the queue is a sorted + // set and the lease handoff is a script. Saying so at startup + // beats the first dequeue failing with ErrNotSupported, which + // reads like a bug rather than a mis-chosen backend. + s.logger.Warn("dispatch/redis: kv driver is missing required capabilities", + log.String("missing", strings.Join(missing, ", ")), + ) + } + return s } +// MissingCapabilities reports which of the kv capabilities this store +// needs the given driver does not provide. An empty result means the +// driver can back Dispatch. +// +// Sorted sets order the queue, sets enumerate ids, hashes hold artifact +// links, scripts make the lease compare-and-set atomic, and streams carry +// events. Pub/Sub is absent from this list deliberately: it only shortens +// wake latency, and polling covers its absence. +func MissingCapabilities(store *kv.Store) []string { + var missing []string + + for _, c := range []struct { + name string + has bool + }{ + {"sorted sets", store.SupportsSortedSets()}, + {"sets", store.SupportsSets()}, + {"hashes", store.SupportsHashes()}, + {"scripts", store.SupportsScripts()}, + {"streams", store.SupportsStreams()}, + } { + if !c.has { + missing = append(missing, c.name) + } + } + + return missing +} + // KV returns the underlying KV store. func (s *Store) KV() *kv.Store { return s.kv } @@ -106,11 +147,6 @@ func isNotFound(err error) bool { return errors.Is(err, kv.ErrNotFound) } -// isRedisNil checks if an error is a Redis nil (key not found). -func isRedisNil(err error) bool { - return errors.Is(err, goredis.Nil) -} - // getEntity retrieves and decodes a JSON entity from a KV key. func (s *Store) getEntity(ctx context.Context, key string, dest any) error { raw, err := s.kv.GetRaw(ctx, key) diff --git a/store/redis/workflow.go b/store/redis/workflow.go index e4a8396..c05d961 100644 --- a/store/redis/workflow.go +++ b/store/redis/workflow.go @@ -94,7 +94,7 @@ func (s *Store) CreateRun(ctx context.Context, run *workflow.Run) error { return fmt.Errorf("dispatch/redis: create run set: %w", err) } - if err := s.rdb.SAdd(ctx, s.keys.runIDs(), rID).Err(); err != nil { + if _, err := s.kv.SAdd(ctx, s.keys.runIDs(), rID); err != nil { return fmt.Errorf("dispatch/redis: create run index: %w", err) } return nil @@ -130,7 +130,7 @@ func (s *Store) UpdateRun(ctx context.Context, run *workflow.Run) error { // ListRuns returns workflow runs matching the given options. func (s *Store) ListRuns(ctx context.Context, opts workflow.ListOpts) ([]*workflow.Run, error) { - ids, err := s.rdb.SMembers(ctx, s.keys.runIDs()).Result() + ids, err := s.kv.SMembers(ctx, s.keys.runIDs()) if err != nil { return nil, fmt.Errorf("dispatch/redis: list runs smembers: %w", err) } @@ -171,7 +171,7 @@ func (s *Store) SaveCheckpoint(ctx context.Context, runID id.RunID, stepName str return fmt.Errorf("dispatch/redis: save checkpoint: %w", err) } - if err := s.rdb.SAdd(ctx, s.keys.checkpointIndex(rID), stepName).Err(); err != nil { + if _, err := s.kv.SAdd(ctx, s.keys.checkpointIndex(rID), stepName); err != nil { return fmt.Errorf("dispatch/redis: save checkpoint index: %w", err) } return nil @@ -193,7 +193,7 @@ func (s *Store) GetCheckpoint(ctx context.Context, runID id.RunID, stepName stri // ListCheckpoints returns all checkpoints for a workflow run. func (s *Store) ListCheckpoints(ctx context.Context, runID id.RunID) ([]*workflow.Checkpoint, error) { rID := runID.String() - steps, err := s.rdb.SMembers(ctx, s.keys.checkpointIndex(rID)).Result() + steps, err := s.kv.SMembers(ctx, s.keys.checkpointIndex(rID)) if err != nil { return nil, fmt.Errorf("dispatch/redis: list checkpoints: %w", err) } @@ -252,7 +252,7 @@ func (s *Store) DeleteCheckpointsAfter(ctx context.Context, runID id.RunID, afte } // List all step names for this run. - steps, err := s.rdb.SMembers(ctx, s.keys.checkpointIndex(rID)).Result() + steps, err := s.kv.SMembers(ctx, s.keys.checkpointIndex(rID)) if err != nil { return fmt.Errorf("dispatch/redis: list checkpoint steps: %w", err) } @@ -264,10 +264,10 @@ func (s *Store) DeleteCheckpointsAfter(ctx context.Context, runID id.RunID, afte continue } if e.CreatedAt.After(target.CreatedAt) { - if delErr := s.rdb.Del(ctx, key).Err(); delErr != nil { + if delErr := s.kv.Delete(ctx, key); delErr != nil { return fmt.Errorf("delete checkpoint %s: %w", key, delErr) } - if remErr := s.rdb.SRem(ctx, s.keys.checkpointIndex(rID), step).Err(); remErr != nil { + if _, remErr := s.kv.SRem(ctx, s.keys.checkpointIndex(rID), step); remErr != nil { return fmt.Errorf("remove checkpoint index %s: %w", step, remErr) } } From e580042d43cc9d726e338fc400b41edd0ec15c5f Mon Sep 17 00:00:00 2001 From: Rex Raphael Date: Mon, 17 Aug 2026 14:30:06 -0500 Subject: [PATCH 7/8] test(redis): drop the last go-redis imports The whole package, tests included, now goes through grove kv. The requeue index check just reads a sorted set, so it uses kv.ZRange instead of building a second Redis client alongside the store. The scan-cost guard needed grove to grow hooks on collection commands first, which it now has. It counts keys touched rather than commands sent: a batch read is one call but N keys of Redis CPU, and MGetRaw reports all of its keys, so a full scan cannot hide behind a single call. Counting at the kv layer also means the guard measures the same thing whatever driver is underneath. Mutation-verified, as the original was: restoring the unbounded scan still fails this test, at 29 keys passing versus a 2000-key backlog. --- store/redis/dequeue_test.go | 46 +++++++++++++++++++------------------ store/redis/requeue_test.go | 11 ++------- 2 files changed, 26 insertions(+), 31 deletions(-) diff --git a/store/redis/dequeue_test.go b/store/redis/dequeue_test.go index 6ce3015..b6ac5e1 100644 --- a/store/redis/dequeue_test.go +++ b/store/redis/dequeue_test.go @@ -10,7 +10,7 @@ import ( "testing" "time" - goredis "github.com/redis/go-redis/v9" + "github.com/xraph/grove/hook" "github.com/xraph/grove/kv/drivers/redisdriver" @@ -273,11 +273,17 @@ func TestDequeueOrdersNullPrimaryInputHashAsUnpreferred(t *testing.T) { // Scan cost // ────────────────────────────────────────────────── -// countingHook tallies every command this client sends, pipelined ones -// individually. A pipeline is one round trip but N commands of Redis -// CPU, and Redis is single-threaded, so it is the command count — not +// countingHook tallies the keys every kv command touches. +// +// Keys rather than calls, because a batch read is one call but N keys of +// Redis CPU, and Redis is single-threaded: it is the per-key work — not // the round-trip count — that decides whether a deep queue starves -// enqueues, heartbeats and lease renewals. +// enqueues, heartbeats and lease renewals. MGetRaw reporting all its keys +// is what keeps this honest, since a full scan hides behind a single call +// otherwise. +// +// It hooks the kv store rather than a Redis client, so this measures the +// same thing whatever driver is underneath. type countingHook struct { mu sync.Mutex n int @@ -304,22 +310,18 @@ func (h *countingHook) reset() { h.n = 0 } -func (h *countingHook) DialHook(next goredis.DialHook) goredis.DialHook { return next } +// BeforeQuery implements hook.PreQueryHook. +func (h *countingHook) BeforeQuery(_ context.Context, qc *hook.QueryContext) (*hook.HookResult, error) { + keys, _ := qc.Values["_kv_keys"].([]string) -func (h *countingHook) ProcessHook(next goredis.ProcessHook) goredis.ProcessHook { - return func(ctx context.Context, cmd goredis.Cmder) error { + // A command with no keys is still one unit of work. + if n := len(keys); n > 0 { + h.add(n) + } else { h.add(1) - - return next(ctx, cmd) } -} - -func (h *countingHook) ProcessPipelineHook(next goredis.ProcessPipelineHook) goredis.ProcessPipelineHook { - return func(ctx context.Context, cmds []goredis.Cmder) error { - h.add(len(cmds)) - return next(ctx, cmds) - } + return nil, nil } // TestUnboundedDequeueDoesNotScanTheWholeQueue pins the promise the rest @@ -366,10 +368,10 @@ func TestUnboundedDequeueDoesNotScanTheWholeQueue(t *testing.T) { } } - hook := &countingHook{} - redisdriver.UnwrapClient(s.KV()).AddHook(hook) + counter := &countingHook{} + s.KV().Hooks().AddHook(counter) - hook.reset() + counter.reset() got, err := s.DequeueJobs(ctx, job.DequeueOpts{ Queues: []string{queue}, @@ -392,14 +394,14 @@ func TestUnboundedDequeueDoesNotScanTheWholeQueue(t *testing.T) { } } - if n := hook.count(); n > maxCommands { + if n := counter.count(); n > maxCommands { t.Fatalf("an unbounded dequeue of %d jobs from a %d-deep queue cost %d Redis commands "+ "(want <= %d): the scan is proportional to the backlog, so the deeper the queue "+ "the more Redis CPU every poll of every worker burns", limit, backlog, n, maxCommands) } - t.Logf("unbounded dequeue: %d Redis commands at a backlog of %d", hook.count(), backlog) + t.Logf("unbounded dequeue: %d keys touched at a backlog of %d", counter.count(), backlog) } // TestBoundedDequeueStillSeesPastTheWindow is the other half of the diff --git a/store/redis/requeue_test.go b/store/redis/requeue_test.go index 20d715e..5474c52 100644 --- a/store/redis/requeue_test.go +++ b/store/redis/requeue_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - goredis "github.com/redis/go-redis/v9" + kvdriver "github.com/xraph/grove/kv/driver" "github.com/xraph/dispatch" "github.com/xraph/dispatch/id" @@ -188,17 +188,10 @@ func TestUpdateJob_MaintainsQueueIndex(t *testing.T) { s := openRedisStore(t, connStr) ctx := context.Background() - opt, err := goredis.ParseURL(connStr) - if err != nil { - t.Fatalf("parse redis url: %v", err) - } - rdb := goredis.NewClient(opt) - t.Cleanup(func() { _ = rdb.Close() }) - indexed := func() bool { t.Helper() - members, mErr := rdb.ZRange(ctx, queueKey, 0, -1).Result() + members, mErr := s.KV().ZRange(ctx, queueKey, kvdriver.RangeSpec{}) if mErr != nil { t.Fatalf("read queue index: %v", mErr) } From 269ee77aae9835ca9d010d2d75e1b1a0a0a4e99f Mon Sep 17 00:00:00 2001 From: Rex Raphael Date: Wed, 7 Oct 2026 09:16:17 -0500 Subject: [PATCH 8/8] build(deps): bump grove to v1.7.0 so namespaced kv stores work Every redis store operation now goes through kv.Store, so a namespace hook on the store you pass in has to reach sorted sets, sets, scripts and streams as well as plain gets and sets. Grove kv v1.6.3 applied the hook to some of those and not to others. Hand Dispatch a namespaced store on v1.6.3 and it writes the job under the namespace and the queue entry outside it, and then it can't dequeue the job it just enqueued. v1.7.0 resolves the rewritten key on every operation. The new integration test opens kv with middleware.NewNamespace, drives enqueue, dequeue, lease reclaim, events, workers, leadership, cron and usage, then lists Redis raw and fails on any key outside the namespace. It fails at the dequeue on v1.6.3 and passes on v1.7.0. go-redis drops to an indirect dependency, since nothing in store/redis imports it any more. --- go.mod | 23 ++----- go.sum | 46 ++++---------- store/redis/namespace_test.go | 116 ++++++++++++++++++++++++++++++++++ 3 files changed, 135 insertions(+), 50 deletions(-) create mode 100644 store/redis/namespace_test.go diff --git a/go.mod b/go.mod index 96bfacb..db34a4a 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,6 @@ go 1.26.0 require ( github.com/a-h/templ v0.3.1001 github.com/jackc/pgx/v5 v5.10.0 - github.com/redis/go-redis/v9 v9.21.0 github.com/robfig/cron/v3 v3.0.1 github.com/testcontainers/testcontainers-go v0.44.0 github.com/testcontainers/testcontainers-go/modules/mongodb v0.44.0 @@ -14,12 +13,12 @@ require ( github.com/vmihailenco/msgpack/v5 v5.4.1 github.com/xraph/forge v1.10.0 github.com/xraph/forgeui v1.4.1 - github.com/xraph/grove v1.6.3 - github.com/xraph/grove/drivers/mongodriver v1.6.3 - github.com/xraph/grove/drivers/pgdriver v1.6.3 - github.com/xraph/grove/drivers/sqlitedriver v1.6.3 - github.com/xraph/grove/kv v1.6.3 - github.com/xraph/grove/kv/drivers/redisdriver v1.6.3 + github.com/xraph/grove v1.7.0 + github.com/xraph/grove/drivers/mongodriver v1.7.0 + github.com/xraph/grove/drivers/pgdriver v1.7.0 + github.com/xraph/grove/drivers/sqlitedriver v1.7.0 + github.com/xraph/grove/kv v1.7.0 + github.com/xraph/grove/kv/drivers/redisdriver v1.7.0 github.com/xraph/relay v1.6.3 github.com/xraph/trove v1.6.5 github.com/xraph/vessel v1.0.4 @@ -152,7 +151,6 @@ require ( require ( github.com/Oudwins/tailwind-merge-go v0.2.1 // indirect - github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/dunglas/httpsfv v1.1.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/go-openapi/swag/cmdutils v0.28.0 // indirect @@ -167,7 +165,6 @@ require ( github.com/go-openapi/swag/typeutils v0.28.0 // indirect github.com/go-openapi/swag/yamlutils v0.28.0 // indirect github.com/gofrs/uuid/v5 v5.5.1 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect github.com/hashicorp/go-metrics v0.6.1 // indirect github.com/klauspost/cpuid/v2 v2.4.0 // indirect github.com/mdelapenya/tlscert v0.2.0 // indirect @@ -178,6 +175,7 @@ require ( github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.70.1 // indirect github.com/prometheus/procfs v0.21.1 // indirect + github.com/redis/go-redis/v9 v9.21.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect @@ -185,15 +183,8 @@ require ( github.com/xdg-go/scram v1.2.0 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect - go.opentelemetry.io/otel/exporters/jaeger v1.17.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 // indirect go.opentelemetry.io/otel/metric v1.44.0 // indirect - go.opentelemetry.io/proto/otlp v1.10.0 // indirect go.uber.org/atomic v1.11.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect - google.golang.org/grpc v1.83.2 // indirect modernc.org/libc v1.68.0 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect diff --git a/go.sum b/go.sum index fa23d28..bdea813 100644 --- a/go.sum +++ b/go.sum @@ -29,8 +29,6 @@ github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= -github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= -github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -148,8 +146,6 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/gnostic-models v0.7.1 h1:SisTfuFKJSKM5CPZkffwi6coztzzeYUhc3v4yxLWH8c= @@ -163,8 +159,6 @@ github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17k github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= github.com/hashicorp/consul/api v1.34.4 h1:0U4YZ1Yp7K9WK9ex0gTJraFim26l02wCvsmf2ukalVE= github.com/hashicorp/consul/api v1.34.4/go.mod h1:vz5gBNeycefpAAVNVbLBFObUu3isju6EK8UVZjXSTWc= github.com/hashicorp/consul/sdk v0.18.1 h1:RDTeBvAeOveI2xI86sV+8WkaN7OkP4zz+cG3fOobDCM= @@ -401,18 +395,18 @@ github.com/xraph/forgeui v1.4.1 h1:LHK1t/sZ+9zL+MNUZralO9/rc0f5UCa19dpbWTuRMNg= github.com/xraph/forgeui v1.4.1/go.mod h1:rH/+wb1tt2pXSHotWAvoP+Lt846xlIjuwPDSpS5K5mw= github.com/xraph/go-utils v1.2.2 h1:zwRV3jRWDSGTqoC4zoYqxyXhXOCEI8tp66GJIPhqf8Y= github.com/xraph/go-utils v1.2.2/go.mod h1:Mckdi+nR0bI4bUESKSYajJq4tNSPsvZiuLRYJ0+qDQw= -github.com/xraph/grove v1.6.3 h1:QxU2SaBTTOp9tbO/DTNIrnh4pJ+T+MvuE4o6WH+ChBc= -github.com/xraph/grove v1.6.3/go.mod h1:7RviFsJh4CgHtp7DKkgUtRgIPrzdGmWhzu2nLlC2eWU= -github.com/xraph/grove/drivers/mongodriver v1.6.3 h1:KthiN7sPpl9qDkV3n6M0VdOg++budzXUiBqfP7Cx8p8= -github.com/xraph/grove/drivers/mongodriver v1.6.3/go.mod h1:E7ikkdRhiJm1lLCJ5mOVYa4Z6CNqhG/w5UN8A8jnnac= -github.com/xraph/grove/drivers/pgdriver v1.6.3 h1:AxzXAUTsuJQaHc2R1FciSu2OJeTeqMylP4nDu7xzgPQ= -github.com/xraph/grove/drivers/pgdriver v1.6.3/go.mod h1:tN3pXpElsaWtSfZxnyHDr4jZAAl8pJPQfp+pCx/3R5c= -github.com/xraph/grove/drivers/sqlitedriver v1.6.3 h1:bk62lIB6Eyiv8mmPrC8YmR6S9KYWB3gmCsffo5FfdEc= -github.com/xraph/grove/drivers/sqlitedriver v1.6.3/go.mod h1:qvoSnNilfVwyB7S7RIYkgZ8WCYGL8CHuegilc4PrZMc= -github.com/xraph/grove/kv v1.6.3 h1:YHYIhleVbvmRSi+j7WdH+zUOn0nL1flI3OKtjbmDJBQ= -github.com/xraph/grove/kv v1.6.3/go.mod h1:DWru5OjKvxjeUKixtsgbDGm4BWrMltB6IA3nLQUrsFw= -github.com/xraph/grove/kv/drivers/redisdriver v1.6.3 h1:32DoOGGZyltTDRcmVyagHwKu2Jw9R5bV7Dw0crgyhgU= -github.com/xraph/grove/kv/drivers/redisdriver v1.6.3/go.mod h1:sSoFDCR105ys80xwoZOQphsqAGNd6JClry2QNm7cwp4= +github.com/xraph/grove v1.7.0 h1:ueox0LC0oj2m6smd02iRFSMugVm/bmcAeGMStWhAqPA= +github.com/xraph/grove v1.7.0/go.mod h1:7RviFsJh4CgHtp7DKkgUtRgIPrzdGmWhzu2nLlC2eWU= +github.com/xraph/grove/drivers/mongodriver v1.7.0 h1:C+Sm3l0kOhhsxxo84J1DezDg1dVjlu+8IgUXCcDsOQg= +github.com/xraph/grove/drivers/mongodriver v1.7.0/go.mod h1:E7ikkdRhiJm1lLCJ5mOVYa4Z6CNqhG/w5UN8A8jnnac= +github.com/xraph/grove/drivers/pgdriver v1.7.0 h1:WhyM0G9t1HoBK+53rVu51u3/wPx60V7VjLb8DxObwQQ= +github.com/xraph/grove/drivers/pgdriver v1.7.0/go.mod h1:tN3pXpElsaWtSfZxnyHDr4jZAAl8pJPQfp+pCx/3R5c= +github.com/xraph/grove/drivers/sqlitedriver v1.7.0 h1:yMgGRwQjSdR7UylTJVoLS7XnUJ3oSk85kQBp4Lf+4k4= +github.com/xraph/grove/drivers/sqlitedriver v1.7.0/go.mod h1:qvoSnNilfVwyB7S7RIYkgZ8WCYGL8CHuegilc4PrZMc= +github.com/xraph/grove/kv v1.7.0 h1:wvZ+f7Ck6fhQbQv4asNdhY8Xy33QlKw7vXhtFLW5lQk= +github.com/xraph/grove/kv v1.7.0/go.mod h1:DWru5OjKvxjeUKixtsgbDGm4BWrMltB6IA3nLQUrsFw= +github.com/xraph/grove/kv/drivers/redisdriver v1.7.0 h1:w6kXiN16qHCOcYqLP7xiVAamqdsVJlIOJ3Xgi0eJBbg= +github.com/xraph/grove/kv/drivers/redisdriver v1.7.0/go.mod h1:sSoFDCR105ys80xwoZOQphsqAGNd6JClry2QNm7cwp4= github.com/xraph/relay v1.6.3 h1:y/63Nk3SM02gEQe204v9bIoZbH+6lCM/JQurlXm1NvE= github.com/xraph/relay v1.6.3/go.mod h1:Tx3wRpjFJTlIeQ6FkVCuZ/SQjykSzflsVxFSprAVpq4= github.com/xraph/trove v1.6.5 h1:S7QZb9z2kcMijNkndAvqy5dFf9Y1Aqn/kEbJpAn/0O0= @@ -442,12 +436,6 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= -go.opentelemetry.io/otel/exporters/jaeger v1.17.0 h1:D7UpUy2Xc2wsi1Ras6V40q806WM07rqoCWzXu7Sqy+4= -go.opentelemetry.io/otel/exporters/jaeger v1.17.0/go.mod h1:nPCqOnEH9rNLKqH/+rrUjiMzHJdV1BlpKcTwRTyKkKI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 h1:88Y4s2C8oTui1LGM6bTWkw0ICGcOLCAI5l6zsD1j20k= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0/go.mod h1:Vl1/iaggsuRlrHf/hfPJPvVag77kKyvrLeD10kpMl+A= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 h1:3iZJKlCZufyRzPzlQhUIWVmfltrXuGyfjREgGP3UUjc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0/go.mod h1:/G+nUPfhq2e+qiXMGxMwumDrP5jtzU+mWN7/sjT2rak= go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= @@ -456,8 +444,6 @@ go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRk go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= -go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= -go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -538,14 +524,6 @@ golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= -gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= -google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= -google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= diff --git a/store/redis/namespace_test.go b/store/redis/namespace_test.go new file mode 100644 index 0000000..87f2d09 --- /dev/null +++ b/store/redis/namespace_test.go @@ -0,0 +1,116 @@ +//go:build integration + +package redis_test + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/xraph/grove/kv" + "github.com/xraph/grove/kv/drivers/redisdriver" + "github.com/xraph/grove/kv/middleware" + + "github.com/xraph/dispatch" + "github.com/xraph/dispatch/cluster" + "github.com/xraph/dispatch/cron" + "github.com/xraph/dispatch/event" + "github.com/xraph/dispatch/id" + "github.com/xraph/dispatch/job" + redisstore "github.com/xraph/dispatch/store/redis" +) + +// A host can hand Dispatch a kv store that already carries a namespace +// hook. Every operation now goes through kv.Store, so every key has to +// pass through that hook too: sorted sets, sets, hashes, scripts and +// streams as well as plain gets and sets. grove kv before v1.7.0 applied +// the hook to some of those and not others, which split one store across +// two keyspaces. This drives each kind of operation and then reads Redis +// raw to prove nothing landed outside the namespace. +func TestStore_NamespaceHook_keepsEveryKeyInside(t *testing.T) { + ctx := context.Background() + connStr := startRedis(t) + + drv := redisdriver.New() + if err := drv.Open(ctx, connStr); err != nil { + t.Fatalf("open redisdriver: %v", err) + } + kvStore, err := kv.Open(drv, kv.WithHook(middleware.NewNamespace("tenant_x"))) + if err != nil { + t.Fatalf("kv open: %v", err) + } + t.Cleanup(func() { _ = kvStore.Close() }) + + s := redisstore.New(kvStore) + now := time.Now().UTC() + + // Sets, sorted sets and plain entities. + pending := &job.Job{ + Entity: dispatch.NewEntity(), + ID: id.NewJobID(), + Name: "ns-pending", + Queue: "default", + Payload: []byte(`{}`), + State: job.StatePending, + MaxRetries: 3, + RunAt: now, + } + if err := s.EnqueueJob(ctx, pending); err != nil { + t.Fatalf("enqueue: %v", err) + } + got, err := s.DequeueJobs(ctx, job.DequeueOpts{Queues: []string{"default"}, Limit: 10}) + if err != nil { + t.Fatalf("dequeue: %v", err) + } + if len(got) != 1 { + t.Fatalf("dequeued %d jobs, want the 1 just enqueued: the queue and the entity disagree on where they live", len(got)) + } + + // The reclaim path runs a Lua script over the job key. + if err := s.EnqueueJob(ctx, runningJob("ns-stale", time.Hour)); err != nil { + t.Fatalf("enqueue running: %v", err) + } + if _, err := s.ReclaimExpiredLeases(ctx, 10); err != nil { + t.Fatalf("reclaim: %v", err) + } + + // Streams, hashes-backed cluster state, cron names, usage indexes. + if err := s.PublishEvent(ctx, &event.Event{ID: id.NewEventID(), Name: "ns-evt", CreatedAt: now}); err != nil { + t.Fatalf("publish event: %v", err) + } + w := &cluster.Worker{ID: id.NewWorkerID(), Hostname: "h", Queues: []string{"default"}, State: cluster.WorkerActive, LastSeen: now, CreatedAt: now} + if err := s.RegisterWorker(ctx, w); err != nil { + t.Fatalf("register worker: %v", err) + } + if _, err := s.AcquireLeadership(ctx, w.ID, time.Minute); err != nil { + t.Fatalf("acquire leadership: %v", err) + } + next := now.Add(time.Hour) + if err := s.RegisterCron(ctx, &cron.Entry{ + Entity: dispatch.NewEntity(), ID: id.NewCronID(), Name: "ns-cron", + Schedule: "0 0 * * *", JobName: "x", Payload: []byte(`{}`), Enabled: true, NextRunAt: &next, + }); err != nil { + t.Fatalf("register cron: %v", err) + } + if err := s.RecordJobUsage(ctx, &job.Usage{ + ID: id.NewUsageID(), JobID: pending.ID, Name: pending.Name, Queue: "default", + Status: job.StateCompleted, WallTime: time.Second, RecordedAt: now, + }); err != nil { + t.Fatalf("record usage: %v", err) + } + + raw := redisdriver.UnwrapClient(kvStore) + keys, err := raw.Keys(ctx, "*").Result() + if err != nil { + t.Fatalf("raw keys: %v", err) + } + if len(keys) == 0 { + t.Fatal("no keys written at all") + } + for _, k := range keys { + if !strings.HasPrefix(k, "tenant_x:") { + t.Errorf("key %q escaped the namespace hook", k) + } + } +}