feat(cache): implement stale-while-revalidate in OptimisticCache (#776)

src/cache.ts

@@ -61,6 +61,24 @@ export class SimpleCache<T> {
 
     this.hits++;
     return entry.value;
+
+  /**
+   * Peek at the raw cache entry for `key` without updating hit counters
+   * or evicting expired entries. Returns `undefined` when the key is absent.
+   */
+  peek(key: string): MethodCacheEntry | undefined {
+    if (!this.enabled) return undefined;
+    return this.store.get(key);
+  }
+
+  /** Return the TTL (in ms) that would be applied to `method`. */
+  resolveTtl(method: string): number {
+    return this.ttlConfig[method] ?? this.ttlConfig["default"] ?? 0;
+  }
+
+  get enabledState(): boolean {
+    return this.enabled;
+  }
   }
 
   set(key: string, value: T): void {

src/cache/OptimisticCache.ts

@@ -8,12 +8,17 @@
  * concurrent optimistic mutations to the same invoice queue up instead of
  * clobbering one another: rolling back mutation N leaves mutations N+1..M
  * (and the base cache) untouched.
+ *
+ * Supports stale-while-revalidate: when a base cache entry is within
+ * `staleWhileRevalidateMs` of its TTL, the stale value is returned
+ * immediately and a background refresh is triggered.
  */
 
 import { SimpleCache } from "../cache.js";
 
 export type CommitFn = () => void;
 export type RollbackFn = () => void;
+export type RevalidateFn<T> = (invoiceId: string) => Promise<T>;
 
 export interface OptimisticEntry<T> {
   key: string;
@@ -32,30 +37,71 @@ export interface RollbackEvent<T> {
   restoredValue: T;
 }
 
+export interface RevalidateErrorEvent {
+  invoiceId: string;
+  error: unknown;
+}
+
+export interface OptimisticCacheOptions<T> {
+  /** Underlying base cache. Created automatically when omitted. */
+  base?: SimpleCache<T>;
+  /**
+   * How long before TTL expiry a cached value is considered stale but
+   * still served while a background revalidation runs. `0` disables
+   * stale-while-revalidate (default).
+   */
+  staleWhileRevalidateMs?: number;
+  /**
+   * Called in the background when a stale entry is served.
+   * Must resolve with the fresh value to write back into the base cache.
+   */
+  revalidate?: RevalidateFn<T>;
+}
+
 const DEFAULT_BASE_TTL_MS = 60_000;
 
 export class OptimisticCache<T = unknown> {
   private readonly base: SimpleCache<T>;
+  private readonly staleWhileRevalidateMs: number;
+  private readonly revalidate?: RevalidateFn<T>;
   /** Per-invoice FIFO queue of pending (uncommitted, unrolled-back) predictions. */
   private readonly pending = new Map<string, OptimisticEntry<T>[]>();
   private readonly rollbackHandlers = new Set<(event: RollbackEvent<T>) => void>();
+  private readonly revalidateErrorHandlers = new Set<(event: RevalidateErrorEvent) => void>();
   private readonly versionCounters = new Map<string, number>();
+  /** Track in-flight background revalidations so only one runs per key. */
+  private readonly inFlightRevalidations = new Set<string>();
 
-  constructor(base?: SimpleCache<T>) {
-    this.base = base ?? new SimpleCache<T>({ enabled: true, ttlMs: DEFAULT_BASE_TTL_MS });
+  constructor(options: OptimisticCacheOptions<T> = {}) {
+    this.base = options.base ?? new SimpleCache<T>({ enabled: true, ttlMs: DEFAULT_BASE_TTL_MS });
+    this.staleWhileRevalidateMs = options.staleWhileRevalidateMs ?? 0;
+    this.revalidate = options.revalidate;
   }
 
   /**
    * Read the current UI-facing value for an invoice: the most recently
    * applied still-pending optimistic prediction if one exists, otherwise
    * the committed base value.
+   *
+   * When stale-while-revalidate is configured and the base value is within
+   * the stale window, the value is returned immediately and a background
+   * revalidation is started (if not already in-flight for this key).
    */
   get(invoiceId: string): T | undefined {
     const queue = this.pending.get(invoiceId);
     if (queue && queue.length > 0) {
       return queue[queue.length - 1]!.predictedValue;
     }
-    return this.base.get(invoiceId);
+
+    const value = this.base.get(invoiceId);
+
+    // Stale-while-revalidate: if the entry is within the stale window,
+    // trigger a background refresh without blocking the caller.
+    if (this.staleWhileRevalidateMs > 0 && this.revalidate && value !== undefined) {
+      this._maybeRevalidate(invoiceId);
+    }
+
+    return value;
   }
 
   /** Number of optimistic mutations across all invoices awaiting commit/rollback. */
@@ -71,6 +117,12 @@ export class OptimisticCache<T = unknown> {
     return () => this.rollbackHandlers.delete(handler);
   }
 
+  /** Register a listener invoked when a background revalidation fails. */
+  onRevalidateError(handler: (event: RevalidateErrorEvent) => void): () => void {
+    this.revalidateErrorHandlers.add(handler);
+    return () => this.revalidateErrorHandlers.delete(handler);
+  }
+
   /**
    * Apply a predicted value for `invoiceId` immediately. Returns a
    * `{ commit, rollback }` pair: `commit()` writes the prediction into the
@@ -126,6 +178,46 @@ export class OptimisticCache<T = unknown> {
     return { commit, rollback, key };
   }
 
+  /**
+   * Check whether the base cache entry for `invoiceId` is stale (within
+   * `staleWhileRevalidateMs` of expiry) and trigger a background
+   * revalidation if so. Does nothing when SWR is disabled or already
+   * in-flight for this key.
+   */
+  private _maybeRevalidate(invoiceId: string): void {
+    if (!this.revalidate || this.inFlightRevalidations.has(invoiceId)) return;
+
+    const raw = this.base.peek(invoiceId);
+    if (!raw) return;
+
+    const ttl = this.base.resolveTtl(invoiceId);
+    if (ttl <= 0) return;
+
+    const staleThreshold = raw.expiresAt - this.staleWhileRevalidateMs;
+    const now = Date.now();
+    if (now < staleThreshold) return; // not yet stale
+
+    this.inFlightRevalidations.add(invoiceId);
+
+    this.revalidate(invoiceId)
+      .then((fresh) => {
+        this.base.set(invoiceId, fresh);
+      })
+      .catch((err) => {
+        const event: RevalidateErrorEvent = { invoiceId, error: err };
+        for (const handler of this.revalidateErrorHandlers) {
+          try {
+            handler(event);
+          } catch {
+            // Isolate listener failures.
+          }
+        }
+      })
+      .finally(() => {
+        this.inFlightRevalidations.delete(invoiceId);
+      });
+  }
+
   private _removeEntry(entry: OptimisticEntry<T>): void {
     const queue = this.pending.get(entry.invoiceId);
     if (!queue) return;

test/optimisticCacheSwr.test.ts

@@ -0,0 +1,139 @@
+import { describe, it, expect, vi } from "vitest";
+import { OptimisticCache, type RevalidateErrorEvent } from "../src/cache/OptimisticCache.js";
+import { SimpleCache } from "../src/cache.js";
+
+describe("OptimisticCache stale-while-revalidate", () => {
+  it("returns value immediately and triggers background revalidation when stale", async () => {
+    const base = new SimpleCache<string>({ enabled: true, ttlMs: 100 });
+    base.set("inv-1", "old");
+
+    const revalidate = vi.fn().mockResolvedValue("fresh");
+    const cache = new OptimisticCache<string>({
+      base,
+      staleWhileRevalidateMs: 50,
+      revalidate,
+    });
+
+    // Advance time so the entry is within the stale window
+    vi.advanceTimersByTime?.(60);
+
+    const value = cache.get("inv-1");
+    expect(value).toBe("old");
+
+    // Wait for the background revalidation
+    await new Promise((r) => setTimeout(r, 10));
+    expect(revalidate).toHaveBeenCalledWith("inv-1");
+    expect(base.get("inv-1")).toBe("fresh");
+  });
+
+  it("does not trigger revalidation when entry is not yet stale", async () => {
+    const base = new SimpleCache<string>({ enabled: true, ttlMs: 1000 });
+    base.set("inv-1", "fresh");
+
+    const revalidate = vi.fn().mockResolvedValue(" newer");
+    const cache = new OptimisticCache<string>({
+      base,
+      staleWhileRevalidateMs: 100,
+      revalidate,
+    });
+
+    const value = cache.get("inv-1");
+    expect(value).toBe("fresh");
+    expect(revalidate).not.toHaveBeenCalled();
+  });
+
+  it("does not trigger revalidation when staleWhileRevalidateMs is 0", () => {
+    const base = new SimpleCache<string>({ enabled: true, ttlMs: 100 });
+    base.set("inv-1", "old");
+
+    const revalidate = vi.fn().mockResolvedValue("fresh");
+    const cache = new OptimisticCache<string>({ base, revalidate });
+
+    const value = cache.get("inv-1");
+    expect(value).toBe("old");
+    expect(revalidate).not.toHaveBeenCalled();
+  });
+
+  it("does not trigger concurrent revalidations for the same key", async () => {
+    const base = new SimpleCache<string>({ enabled: true, ttlMs: 100 });
+    base.set("inv-1", "old");
+
+    let callCount = 0;
+    const revalidate = vi.fn().mockImplementation(async () => {
+      callCount++;
+      await new Promise((r) => setTimeout(r, 50));
+      return `fresh-${callCount}`;
+    });
+
+    const cache = new OptimisticCache<string>({
+      base,
+      staleWhileRevalidateMs: 50,
+      revalidate,
+    });
+
+    // Advance into stale window
+    vi.advanceTimersByTime?.(60);
+
+    cache.get("inv-1");
+    cache.get("inv-1");
+    cache.get("inv-1");
+
+    await new Promise((r) => setTimeout(r, 100));
+    expect(revalidate).toHaveBeenCalledTimes(1);
+  });
+
+  it("emits revalidateError when background refresh fails", async () => {
+    const base = new SimpleCache<string>({ enabled: true, ttlMs: 100 });
+    base.set("inv-1", "old");
+
+    const revalidate = vi.fn().mockRejectedValue(new Error("network down"));
+    const cache = new OptimisticCache<string>({
+      base,
+      staleWhileRevalidateMs: 50,
+      revalidate,
+    });
+
+    const errors: RevalidateErrorEvent[] = [];
+    cache.onRevalidateError((e) => errors.push(e));
+
+    vi.advanceTimersByTime?.(60);
+    cache.get("inv-1");
+
+    await new Promise((r) => setTimeout(r, 10));
+    expect(errors).toHaveLength(1);
+    expect(errors[0]!.invoiceId).toBe("inv-1");
+    expect(errors[0]!.error).toBeInstanceOf(Error);
+  });
+
+  it("serves stale value even after background error", async () => {
+    const base = new SimpleCache<string>({ enabled: true, ttlMs: 100 });
+    base.set("inv-1", "old");
+
+    const revalidate = vi.fn().mockRejectedValue(new Error("boom"));
+    const cache = new OptimisticCache<string>({
+      base,
+      staleWhileRevalidateMs: 50,
+      revalidate,
+    });
+
+    vi.advanceTimersByTime?.(60);
+    const v1 = cache.get("inv-1");
+    await new Promise((r) => setTimeout(r, 10));
+    const v2 = cache.get("inv-1");
+
+    expect(v1).toBe("old");
+    expect(v2).toBe("old");
+  });
+
+  it("does not attempt revalidation when no revalidate callback is provided", () => {
+    const base = new SimpleCache<string>({ enabled: true, ttlMs: 100 });
+    base.set("inv-1", "old");
+    const cache = new OptimisticCache<string>({
+      base,
+      staleWhileRevalidateMs: 50,
+    });
+
+    vi.advanceTimersByTime?.(60);
+    expect(cache.get("inv-1")).toBe("old");
+  });
+});