From 1ac39c901cd821d9316ab2f0399028b65d43b53a Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sat, 19 Sep 2026 00:27:30 +0200 Subject: [PATCH] refactor(cli): make the Maple Local eventing stack Effect-first The control store and eventing runtime are now Context.Services with their own layers. The store opens and checkpoints its database through the layer scope, and every method returns an Effect with typed failures instead of throwing. Each bun:sqlite transaction stays one synchronous Effect.try step, so SQL, ordering, leases, pruning and durability are unchanged. serve.ts runs the eventing handlers and ingest on the server's one ManagedRuntime and maps failures with catchTags to the same HTTP statuses as before. Eventing telemetry records Effect metrics directly, OTLP field errors are a Schema.TaggedError in the error channel, and checkpoints use the Effect snapshot helpers. --- apps/cli/src/server/checkpoints.ts | 32 +- apps/cli/src/server/eventing/consumer-auth.ts | 17 +- apps/cli/src/server/eventing/control-store.ts | 2141 ++++++++++------- apps/cli/src/server/eventing/otlp.ts | 92 +- apps/cli/src/server/eventing/runtime.ts | 627 +++-- apps/cli/src/server/eventing/telemetry.ts | 54 +- apps/cli/src/server/local-token.ts | 9 +- apps/cli/src/server/otlp/encode.ts | 20 +- apps/cli/src/server/serve.ts | 1151 ++++----- apps/cli/test/checkpoints.test.ts | 9 +- apps/cli/test/eventing-test-support.ts | 82 + .../test/local-eventing-consumer-auth.test.ts | 7 +- .../test/local-eventing-control-store.test.ts | 509 ++-- apps/cli/test/local-eventing-ingest.test.ts | 608 ++--- apps/cli/test/local-eventing-overflow.test.ts | 170 +- apps/cli/test/local-eventing-runtime.test.ts | 283 +-- .../cli/test/local-eventing-telemetry.test.ts | 9 +- 17 files changed, 3244 insertions(+), 2576 deletions(-) create mode 100644 apps/cli/test/eventing-test-support.ts diff --git a/apps/cli/src/server/checkpoints.ts b/apps/cli/src/server/checkpoints.ts index 97fbf063d..78109c51a 100644 --- a/apps/cli/src/server/checkpoints.ts +++ b/apps/cli/src/server/checkpoints.ts @@ -22,7 +22,9 @@ import { } from "./durable-files" import { eventingControlSnapshotPath, - LocalEventingControlStore, + openControlStore, + restoreControlSnapshot, + validateControlSnapshot, type EventingControlSnapshotValidation, } from "./eventing/control-store" import { CURRENT_LOCAL_SCHEMA, SCHEMA_FINGERPRINT } from "./schema-identity" @@ -885,7 +887,7 @@ const resolveCheckpointById = async ( const controlSha256 = await sha256File(controlPath) if (controlSha256 !== manifest.controlSha256) throw new Error("checkpoint control-store digest mismatch") - const controlValidation = LocalEventingControlStore.validateSnapshot(controlPath) + const controlValidation = await Effect.runPromise(validateControlSnapshot(controlPath)) if (!controlValidationMatches(manifest.controlValidation, controlValidation)) throw new Error("checkpoint control-store validation does not match its manifest") } else if (existsSync(controlPath)) { @@ -933,13 +935,12 @@ const restoreResolvedInto = async ( "SETTINGS allow_different_database_def=1", ) if (resolvedCheckpoint.manifest.formatVersion === MANIFEST_FORMAT_VERSION) { - await LocalEventingControlStore.restoreSnapshot( - join(resolvedCheckpoint.snapshotDir, "control.sqlite"), - targetDataDir, + await Effect.runPromise( + restoreControlSnapshot(join(resolvedCheckpoint.snapshotDir, "control.sqlite"), targetDataDir), ) } else { - const controlStore = await LocalEventingControlStore.open(targetDataDir) - controlStore.close() + // A legacy checkpoint has no control snapshot: open and close to create an empty store. + await Effect.runPromise(Effect.scoped(Effect.asVoid(openControlStore(targetDataDir)))) } return { db, validation: validateRestoredDatabase(db) } } catch (error) { @@ -1687,15 +1688,22 @@ const createCheckpointTraced = Effect.fn("CheckpointService.create")(function* ( : createError(error), ), ) - return yield* Effect.tryPromise({ + const controlPath = eventingControlSnapshotPath(options.dataDir, checkpointId) + yield* Effect.tryPromise({ try: async () => { - const { oldState, snapshot, startedAt } = prepared - let { operation } = prepared await syncTree(snapshotBackupDir(options.dataDir, checkpointId)) - const controlPath = eventingControlSnapshotPath(options.dataDir, checkpointId) await assertNoSymlink(checkpointSnapshotsRoot(options.dataDir), controlPath) await assertRealFile(controlPath, "checkpoint eventing control snapshot") - const controlValidation = LocalEventingControlStore.validateSnapshot(controlPath) + }, + catch: createError, + }) + const controlValidation = yield* validateControlSnapshot(controlPath).pipe( + Effect.mapError(createError), + ) + return yield* Effect.tryPromise({ + try: async () => { + const { oldState, snapshot, startedAt } = prepared + let { operation } = prepared operation = { ...operation, phase: "backup-complete" } await writeOperation(options.dataDir, operation, options.faults) const provisionalManifest: CheckpointManifest = { diff --git a/apps/cli/src/server/eventing/consumer-auth.ts b/apps/cli/src/server/eventing/consumer-auth.ts index 3318a1b11..7b8889295 100644 --- a/apps/cli/src/server/eventing/consumer-auth.ts +++ b/apps/cli/src/server/eventing/consumer-auth.ts @@ -1,7 +1,20 @@ import { resolve } from "node:path" +import { Effect, Schema } from "effect" import { ensureLocalToken, localTokenMatches } from "../local-token" +export class EventConsumerTokenError extends Schema.TaggedError()( + "@maple/cli/eventing/EventConsumerTokenFailed", + { message: Schema.String, cause: Schema.Defect() }, +) {} + export const eventConsumerTokenPath = (dataDir: string): string => `${resolve(dataDir)}.event-consumer-token` -export const ensureEventConsumerToken = (dataDir: string): Promise => - ensureLocalToken(eventConsumerTokenPath(dataDir), "event consumer token") +export const ensureEventConsumerToken = (dataDir: string): Effect.Effect => + Effect.tryPromise({ + try: () => ensureLocalToken(eventConsumerTokenPath(dataDir), "event consumer token"), + catch: (cause) => + new EventConsumerTokenError({ + message: cause instanceof Error ? cause.message : String(cause), + cause, + }), + }) export const eventConsumerTokenMatches = localTokenMatches diff --git a/apps/cli/src/server/eventing/control-store.ts b/apps/cli/src/server/eventing/control-store.ts index 260b67459..2d1f7d9be 100644 --- a/apps/cli/src/server/eventing/control-store.ts +++ b/apps/cli/src/server/eventing/control-store.ts @@ -15,9 +15,9 @@ import { type ProjectionFailure, type SignalProjectionSpec, } from "@maple/eventing-core" -import { Result, Schema } from "effect" +import { Context, Effect, Layer, Result, Schema, type Scope } from "effect" import { durableWrite, ensurePrivateDirectory } from "../durable-files" -import { NOOP_EVENTING_TELEMETRY, type EventingTelemetry } from "./telemetry" +import { observeEventing } from "./telemetry" const CONTROL_DIRECTORY = "control" const CONTROL_DATABASE = "eventing.sqlite" @@ -241,9 +241,51 @@ interface DeliveryGapRow { readonly last_dropped_at: string } +/** Any other control-store failure: SQLite, an invariant, or a corrupt row. */ +export class EventingControlStoreError extends Schema.TaggedError()( + "@maple/cli/eventing/ControlStoreFailed", + { message: Schema.String, cause: Schema.optionalKey(Schema.Defect()) }, +) {} + +export type EventConsumerFailure = + | EventConsumerInputError + | EventConsumerNotFoundError + | EventConsumerConflictError + | EventConsumerLeaseError + | EventConsumerDeliveryGapError + | EventingControlStoreError + +const storeError = (message: string): EventingControlStoreError => new EventingControlStoreError({ message }) + +const isStoreError = Schema.is(EventingControlStoreError) +const isConsumerFailure = Schema.is( + Schema.Union([ + EventConsumerInputError, + EventConsumerNotFoundError, + EventConsumerConflictError, + EventConsumerLeaseError, + EventConsumerDeliveryGapError, + EventingControlStoreError, + ]), +) +const isOutboxAdministrationInvalid = Schema.is(OutboxAdministrationInvalid) + +/** Maps a failure thrown inside one synchronous SQLite step onto the typed channel. */ +const storeFailure = (error: unknown): EventingControlStoreError => + isStoreError(error) + ? error + : new EventingControlStoreError({ + message: error instanceof Error ? error.message : String(error), + cause: error, + }) +const consumerFailure = (error: unknown): EventConsumerFailure => + isConsumerFailure(error) ? error : storeFailure(error) +const administrationFailure = (error: unknown): OutboxAdministrationInvalid | EventingControlStoreError => + isOutboxAdministrationInvalid(error) ? error : storeFailure(error) + const asNumber = (value: number | bigint): number => { const number = Number(value) - if (!Number.isSafeInteger(number) || number < 0) throw new Error(`invalid SQLite integer: ${value}`) + if (!Number.isSafeInteger(number) || number < 0) throw storeError(`invalid SQLite integer: ${value}`) return number } @@ -251,21 +293,21 @@ const decodeProjection = (json: string): SignalProjectionSpec => decodeSignalProjectionSpec(Schema.decodeUnknownSync(Schema.fromJsonString(Schema.Unknown))(json)) const decodeEvent = (json: string): MapleCloudEvent => { - return Result.getOrThrow( - validateMapleCloudEvent(Schema.decodeUnknownSync(Schema.fromJsonString(Schema.Unknown))(json)), - ).event + const validated = validateMapleCloudEvent( + Schema.decodeUnknownSync(Schema.fromJsonString(Schema.Unknown))(json), + ) + if (Result.isFailure(validated)) throw validated.failure + return validated.success.event } const assertRealDatabaseFile = (path: string): void => { - let info - try { - info = lstatSync(path) - } catch (error) { - if (Schema.is(Schema.Struct({ code: Schema.Literal("ENOENT") }))(error)) return - throw error + const info = Result.try(() => lstatSync(path)) + if (Result.isFailure(info)) { + if (Schema.is(Schema.Struct({ code: Schema.Literal("ENOENT") }))(info.failure)) return + throw info.failure } - if (info.isSymbolicLink() || !info.isFile()) - throw new Error(`eventing control database is not a real file: ${path}`) + if (info.success.isSymbolicLink() || !info.success.isFile()) + throw storeError(`eventing control database is not a real file: ${path}`) } const configure = (db: Database): void => { @@ -276,25 +318,25 @@ const configure = (db: Database): void => { const checkpointWal = (db: Database): void => { const result = db.query("PRAGMA wal_checkpoint(TRUNCATE)").get() - if (!result) throw new Error("eventing control WAL checkpoint returned no result") + if (!result) throw storeError("eventing control WAL checkpoint returned no result") const busy = asNumber(result.busy) const log = asNumber(result.log) const checkpointed = asNumber(result.checkpointed) if (busy !== 0 || log !== 0) - throw new Error( + throw storeError( `eventing control WAL checkpoint incomplete (busy=${busy}, log=${log}, checkpointed=${checkpointed})`, ) } const validateLimits = (limits: LocalEventingControlLimits): ResolvedLocalEventingControlLimits => { if (!Number.isSafeInteger(limits.maxOutboxEvents) || limits.maxOutboxEvents < 1) - throw new Error("maxOutboxEvents must be a positive safe integer") + throw storeError("maxOutboxEvents must be a positive safe integer") if (!Number.isSafeInteger(limits.maxOutboxBytes) || limits.maxOutboxBytes < 1) - throw new Error("maxOutboxBytes must be a positive safe integer") + throw storeError("maxOutboxBytes must be a positive safe integer") const retainAcknowledgedReadyEvents = limits.retainAcknowledgedReadyEvents ?? DEFAULT_RETAIN_ACKNOWLEDGED_READY_EVENTS if (!Number.isSafeInteger(retainAcknowledgedReadyEvents) || retainAcknowledgedReadyEvents < 0) - throw new Error("retainAcknowledgedReadyEvents must be a non-negative safe integer") + throw storeError("retainAcknowledgedReadyEvents must be a non-negative safe integer") return { ...limits, retainAcknowledgedReadyEvents } } @@ -303,12 +345,12 @@ const validateOpenDatabase = ( acceptedSchemaVersions: readonly number[] = [CONTROL_SCHEMA_VERSION], ): EventingControlSnapshotValidation => { const quick = db.query("PRAGMA quick_check").get() - if (quick?.quick_check !== "ok") throw new Error(`eventing control database quick_check failed`) + if (quick?.quick_check !== "ok") throw storeError(`eventing control database quick_check failed`) const version = db.query("PRAGMA user_version").get() - if (!version) throw new Error("eventing control database has no schema version") + if (!version) throw storeError("eventing control database has no schema version") const schemaVersion = asNumber(version.user_version) if (!acceptedSchemaVersions.includes(schemaVersion)) - throw new Error( + throw storeError( `unsupported eventing control schema ${schemaVersion}; expected ${acceptedSchemaVersions.join(" or ")}`, ) // Full accounting verification belongs at open/restore, never on the ingest hot path. @@ -321,19 +363,19 @@ const validateOpenDatabase = ( try { const row = accounting.get() if (row === null || asNumber(row.count) !== 1) - throw new Error("eventing control outbox accounting is inconsistent") + throw storeError("eventing control outbox accounting is inconsistent") } finally { accounting.finalize() } const count = (where: string): number => { const row = db.query(`SELECT count(*) AS count FROM outbox_events ${where}`).get() - if (!row) throw new Error("eventing control count query returned no row") + if (!row) throw storeError("eventing control count query returned no row") return asNumber(row.count) } const revisions = db.query("SELECT count(*) AS count FROM projection_revisions").get() - if (!revisions) throw new Error("eventing projection count query returned no row") + if (!revisions) throw storeError("eventing projection count query returned no row") const failures = db.query("SELECT count(*) AS count FROM projection_failures").get() - if (!failures) throw new Error("eventing projection-failure count query returned no row") + if (!failures) throw storeError("eventing projection-failure count query returned no row") const invalidReadiness = db .query( `SELECT count(*) AS count @@ -346,9 +388,9 @@ const validateOpenDatabase = ( ))`, ) .get() - if (!invalidReadiness) throw new Error("eventing readiness validation query returned no row") + if (!invalidReadiness) throw storeError("eventing readiness validation query returned no row") if (asNumber(invalidReadiness.count) !== 0) - throw new Error("eventing control database has inconsistent outbox readiness state") + throw storeError("eventing control database has inconsistent outbox readiness state") { const consumers = db .query, []>( @@ -383,9 +425,9 @@ const validateOpenDatabase = ( statement.finalize() } if (invalidFingerprints === null) - throw new Error("eventing staged source-fingerprint validation returned no row") + throw storeError("eventing staged source-fingerprint validation returned no row") if (asNumber(invalidFingerprints.count) > 0) - throw new Error("eventing control database has an invalid staged source fingerprint") + throw storeError("eventing control database has an invalid staged source fingerprint") } return { schemaVersion, @@ -435,145 +477,424 @@ const decodeConsumer = (row: ConsumerRow): EventConsumer => ({ disabledAt: row.disabled_at, }) -export class LocalEventingControlStore { - readonly #db: Database - #stagedSourceKinds = new Set() - readonly #limits: ResolvedLocalEventingControlLimits - readonly #telemetry: EventingTelemetry - readonly path: string +const DEFAULT_LIMITS: LocalEventingControlLimits = { + maxOutboxEvents: DEFAULT_MAX_OUTBOX_EVENTS, + maxOutboxBytes: DEFAULT_MAX_OUTBOX_BYTES, + retainAcknowledgedReadyEvents: DEFAULT_RETAIN_ACKNOWLEDGED_READY_EVENTS, +} - private constructor( +/** Where the control store lives; supplied by the application root. */ +export class LocalEventingControlConfig extends Context.Service< + LocalEventingControlConfig, + { readonly dataDir: string; readonly limits?: LocalEventingControlLimits } +>()("@maple/cli/eventing/LocalEventingControlConfig") {} + +export type OutboxCapacity = LocalEventingControlLimits & { + readonly currentEvents: number + readonly currentBytes: number +} + +export interface LocalEventingControlStoreApi { + readonly path: string + readonly saveProjection: ( + spec: SignalProjectionSpec, + createdAt?: string, + ) => Effect.Effect + readonly loadEnabledProjections: ( + tenantId: string, + ) => Effect.Effect + readonly stageEvents: ( + events: readonly MapleCloudEvent[], + sourceFingerprints?: ReadonlyMap, + stagedAt?: string, + ) => Effect.Effect + readonly deliveryGap: (tenantId: string) => Effect.Effect + readonly acceptDeliveryGap: ( + tenantId: string, + consumerId: string, + generation: number, + ) => Effect.Effect + /** Operator-authorized loss; the HTTP caller drains admission before invoking this transaction. */ + readonly abandonEvents: ( + tenantId: string, + eventIds: readonly string[], + ) => Effect.Effect< + { readonly abandoned: number; readonly gap: DeliveryGap }, + OutboxAdministrationInvalid | EventingControlStoreError + > + readonly hasStagedSourceKind: (tenantId: string, sourceKind: string) => Effect.Effect + readonly hasStagedSourceOccurrence: ( + tenantId: string, + sourceKind: string, + source: string, + sourceOccurrenceId: string, + ) => Effect.Effect + readonly stagedEventIdsForOccurrence: ( + tenantId: string, + sourceKind: string, + source: string, + sourceOccurrenceId: string, + sourceFingerprint: string, + ) => Effect.Effect + readonly markReady: ( + eventIds: readonly string[], + readyAt?: string, + ) => Effect.Effect + readonly listReady: ( + limit?: number, + after?: number, + ) => Effect.Effect + readonly listStaged: ( + limit?: number, + after?: number, + ) => Effect.Effect + readonly listConsumers: ( + tenantId: string, + ) => Effect.Effect + readonly registerConsumer: ( + tenantId: string, + consumerId: string, + startAt: EventConsumerStart, + registeredAt?: string, + ) => Effect.Effect + readonly disableConsumer: ( + tenantId: string, + consumerId: string, + disabledAt?: string, + ) => Effect.Effect + readonly claimReady: ( + tenantId: string, + consumerId: string, + limit: number, + leaseSeconds: number, + now?: string, + ) => Effect.Effect + readonly acknowledgeClaim: ( + tenantId: string, + consumerId: string, + leaseToken: string, + throughSequence: number, + now?: string, + ) => Effect.Effect + readonly outboxCapacity: Effect.Effect + readonly recordProjectionFailures: ( + tenantId: string, + failures: readonly ProjectionFailure[], + createdAt?: string, + ) => Effect.Effect + readonly validate: Effect.Effect + /** Synchronous so a caller can pair it with another synchronous capture. */ + readonly captureSnapshot: Effect.Effect + readonly backupTo: ( path: string, - db: Database, - limits: ResolvedLocalEventingControlLimits, - telemetry: EventingTelemetry, - ) { - this.path = path - this.#db = db - this.#limits = limits - this.#telemetry = telemetry - this.#refreshStagedSourceKinds() - } + ) => Effect.Effect +} + +const now = (): string => new Date().toISOString() - static async open( - dataDir: string, - limits: LocalEventingControlLimits = { - maxOutboxEvents: DEFAULT_MAX_OUTBOX_EVENTS, - maxOutboxBytes: DEFAULT_MAX_OUTBOX_BYTES, - retainAcknowledgedReadyEvents: DEFAULT_RETAIN_ACKNOWLEDGED_READY_EVENTS, +/** Opens, migrates, and validates the database; a failure here closes the handle it opened. */ +const openDatabase = (path: string): Effect.Effect => + Effect.try({ + try: () => { + assertRealDatabaseFile(path) + return new Database(path, { create: true, readwrite: true, strict: true, safeIntegers: true }) }, - telemetry: EventingTelemetry = NOOP_EVENTING_TELEMETRY, - ): Promise { - const validatedLimits = validateLimits(limits) - const directory = eventingControlDirectory(dataDir) - await ensurePrivateDirectory(directory) - const path = eventingControlPath(dataDir) - assertRealDatabaseFile(path) - const db = new Database(path, { create: true, readwrite: true, strict: true, safeIntegers: true }) - try { - configure(db) - db.exec("PRAGMA journal_mode = WAL") - db.exec("PRAGMA synchronous = FULL") - const version = db.query("PRAGMA user_version").get() - if (!version) throw new Error("eventing control database has no schema version") - let schemaVersion = asNumber(version.user_version) - if (schemaVersion === 0) { - db.transaction(() => db.exec(CREATE_SCHEMA)).exclusive() - schemaVersion = CONTROL_SCHEMA_VERSION + catch: storeFailure, + }).pipe( + Effect.flatMap((db) => + Effect.try({ + try: () => { + configure(db) + db.exec("PRAGMA journal_mode = WAL") + db.exec("PRAGMA synchronous = FULL") + const version = db.query("PRAGMA user_version").get() + if (!version) throw storeError("eventing control database has no schema version") + let schemaVersion = asNumber(version.user_version) + if (schemaVersion === 0) { + db.transaction(() => db.exec(CREATE_SCHEMA)).exclusive() + schemaVersion = CONTROL_SCHEMA_VERSION + } + if (schemaVersion !== CONTROL_SCHEMA_VERSION) + throw storeError( + `unsupported eventing control schema ${schemaVersion}; expected ${CONTROL_SCHEMA_VERSION}`, + ) + chmodSync(path, 0o600) + validateOpenDatabase(db) + return db + }, + catch: storeFailure, + }).pipe(Effect.onError(() => Effect.sync(() => db.close()))), + ), + ) + +/** A clean close truncates the WAL so the next open and any file-level copy see one file. */ +const closeDatabase = (db: Database): Effect.Effect => + Effect.try({ + try: () => { + checkpointWal(db) + db.close(true) + }, + catch: (cause) => + new EventingControlStoreError({ message: "failed to close eventing control store", cause }), + }).pipe(Effect.catchTag("@maple/cli/eventing/ControlStoreFailed", (error) => Effect.logError(error))) + +export class LocalEventingControlStore extends Context.Service< + LocalEventingControlStore, + LocalEventingControlStoreApi +>()("@maple/cli/eventing/LocalEventingControlStore") { + static readonly make: Effect.Effect< + LocalEventingControlStoreApi, + EventingControlStoreError, + LocalEventingControlConfig | Scope.Scope + > = Effect.gen(function* () { + const config = yield* LocalEventingControlConfig + const limits = yield* Effect.try({ + try: () => validateLimits(config.limits ?? DEFAULT_LIMITS), + catch: storeFailure, + }) + yield* Effect.tryPromise({ + try: () => ensurePrivateDirectory(eventingControlDirectory(config.dataDir)), + catch: storeFailure, + }) + const path = eventingControlPath(config.dataDir) + const db = yield* Effect.acquireRelease(openDatabase(path), closeDatabase) + + const readStagedSourceKinds = (): Set => { + const statement = db.prepare<{ tenant_id: string; source_kind: string }, []>( + "SELECT DISTINCT tenant_id, source_kind FROM outbox_events WHERE state = 'staged' AND source_kind IS NOT NULL", + ) + try { + return new Set(statement.all().map((row) => JSON.stringify([row.tenant_id, row.source_kind]))) + } finally { + statement.finalize() } - if (schemaVersion !== CONTROL_SCHEMA_VERSION) - throw new Error( - `unsupported eventing control schema ${schemaVersion}; expected ${CONTROL_SCHEMA_VERSION}`, - ) - chmodSync(path, 0o600) - validateOpenDatabase(db) - return new LocalEventingControlStore(path, db, validatedLimits, telemetry) - } catch (error) { - db.close() - throw error } - } + // Read on every ingest request, so it is cached and refreshed after each committed write. + let stagedSourceKinds = yield* Effect.try({ try: readStagedSourceKinds, catch: storeFailure }) + const refreshStagedSourceKinds = Effect.try({ + try: () => { + stagedSourceKinds = readStagedSourceKinds() + }, + catch: storeFailure, + }) - close(): void { - checkpointWal(this.#db) - this.#db.close(true) - } + const readDeliveryGap = (tenantId: string): DeliveryGap => { + const statement = db.prepare( + "SELECT generation, dropped_events, last_dropped_at FROM delivery_gaps WHERE tenant_id = ?", + ) + try { + const row = statement.get(tenantId) + return row === null + ? { generation: 0, droppedEvents: 0, lastDroppedAt: null } + : { + generation: asNumber(row.generation), + droppedEvents: asNumber(row.dropped_events), + lastDroppedAt: row.last_dropped_at, + } + } finally { + statement.finalize() + } + } + const recordDeliveryGap = (tenantId: string, count: number, at: string): void => { + if (count === 0) return + db.run( + `INSERT INTO delivery_gaps (tenant_id, generation, dropped_events, last_dropped_at) VALUES (?, 1, ?, ?) + ON CONFLICT (tenant_id) DO UPDATE SET generation = generation + 1, dropped_events = dropped_events + excluded.dropped_events, last_dropped_at = excluded.last_dropped_at`, + [tenantId, count, at], + ) + } + const outboxUsage = (): OutboxUsageRow => { + const statement = db.prepare( + "SELECT count, bytes FROM outbox_usage WHERE singleton = 1", + ) + try { + const usage = statement.get() + if (usage === null) throw storeError("event outbox usage query returned no row") + return usage + } finally { + statement.finalize() + } + } + const consumerRow = (tenantId: string, consumerId: string): ConsumerRow | null => + db + .query( + `SELECT consumer_id, tenant_id, active, last_acked_sequence, accepted_gap_generation, lease_token_hash, + lease_expires_at, claimed_through_sequence, registered_at, disabled_at + FROM event_consumers + WHERE tenant_id = ? AND consumer_id = ?`, + ) + .get(tenantId, consumerId) + const consumerLag = (tenantId: string, lastAcknowledgedSequence: number): number => { + const latest = db + .query( + `SELECT max(readiness.sequence) AS sequence + FROM outbox_ready_events AS readiness + INNER JOIN outbox_events AS event ON event.event_id = readiness.event_id + WHERE event.tenant_id = ? AND event.state = 'ready'`, + ) + .get(tenantId) + return Math.max( + 0, + (latest?.sequence == null ? 0 : asNumber(latest.sequence)) - lastAcknowledgedSequence, + ) + } + const pruneAcknowledgedReady = (tenantId: string): number => { + const boundary = db + .query( + "SELECT min(last_acked_sequence) AS sequence FROM event_consumers WHERE tenant_id = ? AND active = 1", + ) + .get(tenantId) + if (boundary?.sequence == null) return 0 + const rows = db + .query( + `SELECT readiness.event_id + FROM outbox_ready_events AS readiness + INNER JOIN outbox_events AS event ON event.event_id = readiness.event_id + WHERE event.tenant_id = ? AND readiness.sequence <= ? + ORDER BY readiness.sequence`, + ) + .all(tenantId, asNumber(boundary.sequence)) + const pruneCount = Math.max(0, rows.length - limits.retainAcknowledgedReadyEvents) + for (const { event_id } of rows.slice(0, pruneCount)) { + db.run("DELETE FROM outbox_ready_events WHERE event_id = ?", [event_id]) + db.run("DELETE FROM outbox_events WHERE event_id = ? AND state = 'ready'", [event_id]) + } + return pruneCount + } + const listOutbox = ( + state: "ready" | "staged", + limit = 100, + after = 0, + ): Effect.Effect => + Effect.try({ + try: () => { + if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1_000) + throw storeError("outbox-event limit must be between 1 and 1000") + if (!Number.isSafeInteger(after) || after < 0) + throw storeError("outbox cursor must be a non-negative safe integer") + const rows = + state === "ready" + ? db + .query( + `SELECT readiness.sequence, event.event_json, event.staged_at, readiness.ready_at + FROM outbox_ready_events AS readiness + INNER JOIN outbox_events AS event ON event.event_id = readiness.event_id + WHERE event.state = 'ready' AND readiness.sequence > ? + ORDER BY readiness.sequence + LIMIT ?`, + ) + .all(after, limit + 1) + : db + .query( + `SELECT sequence, event_json, staged_at, ready_at + FROM outbox_events + WHERE state = 'staged' AND sequence > ? + ORDER BY sequence + LIMIT ?`, + ) + .all(after, limit + 1) + const hasMore = rows.length > limit + const pageRows = hasMore ? rows.slice(0, limit) : rows + const page = pageRows.map(({ sequence, event_json, staged_at, ready_at }) => ({ + sequence: asNumber(sequence), + event: decodeEvent(event_json), + stagedAt: staged_at, + readyAt: ready_at, + })) + return { + events: page, + nextCursor: hasMore ? (page.at(-1)?.sequence ?? null) : null, + } + }, + catch: storeFailure, + }) + const captureSnapshot = Effect.try({ + try: () => { + checkpointWal(db) + return db.serialize() + }, + catch: storeFailure, + }) - saveProjection(spec: SignalProjectionSpec, createdAt = new Date().toISOString()): void { - const decoded = decodeSignalProjectionSpec(spec) - if (!isJsonValue(decoded)) throw new Error("projection spec must be finite JSON") - const specJson = canonicalJson(decoded) - this.#db - .transaction(() => { - const latest = this.#db - .query( - "SELECT max(revision) AS revision FROM projection_revisions WHERE tenant_id = ? AND projection_id = ?", - ) - .get(decoded.tenantId, decoded.id) - const latestRevision = latest?.revision == null ? null : asNumber(latest.revision) - const existing = this.#db - .query( - "SELECT spec_json FROM projection_revisions WHERE tenant_id = ? AND projection_id = ? AND revision = ?", - ) - .get(decoded.tenantId, decoded.id, decoded.revision) - if (existing) { - if (existing.spec_json !== specJson) - throw new Error( - `projection revision is immutable: ${decoded.tenantId}:${decoded.id}@${decoded.revision}`, - ) - if (latestRevision !== decoded.revision) - throw new Error( - `stale projection revision: ${decoded.tenantId}:${decoded.id}@${decoded.revision}; latest is ${latestRevision}`, - ) - const active = this.#db - .query( - "SELECT revision FROM active_projections WHERE tenant_id = ? AND projection_id = ?", - ) - .get(decoded.tenantId, decoded.id) - const activeRevision = active === null ? null : asNumber(active.revision) - const expectedActiveRevision = decoded.enabled ? decoded.revision : null - if (activeRevision !== expectedActiveRevision) - throw new Error( - `projection active state conflicts with exact revision replay: ${decoded.tenantId}:${decoded.id}@${decoded.revision}`, - ) - return - } else { - const expected = latestRevision === null ? 1 : latestRevision + 1 - if (decoded.revision !== expected) - throw new Error( - `projection revision must be ${expected}: ${decoded.tenantId}:${decoded.id}@${decoded.revision}`, - ) - this.#db.run( - "INSERT INTO projection_revisions (tenant_id, projection_id, revision, enabled, spec_json, created_at) VALUES (?, ?, ?, ?, ?, ?)", - [ - decoded.tenantId, - decoded.id, - decoded.revision, - decoded.enabled ? 1 : 0, - specJson, - createdAt, - ], - ) - } + const saveProjection: LocalEventingControlStoreApi["saveProjection"] = (spec, createdAt = now()) => + Effect.try({ + try: () => { + const decoded = decodeSignalProjectionSpec(spec) + if (!isJsonValue(decoded)) throw storeError("projection spec must be finite JSON") + const specJson = canonicalJson(decoded) + db.transaction(() => { + const latest = db + .query( + "SELECT max(revision) AS revision FROM projection_revisions WHERE tenant_id = ? AND projection_id = ?", + ) + .get(decoded.tenantId, decoded.id) + const latestRevision = latest?.revision == null ? null : asNumber(latest.revision) + const existing = db + .query( + "SELECT spec_json FROM projection_revisions WHERE tenant_id = ? AND projection_id = ? AND revision = ?", + ) + .get(decoded.tenantId, decoded.id, decoded.revision) + if (existing) { + if (existing.spec_json !== specJson) + throw storeError( + `projection revision is immutable: ${decoded.tenantId}:${decoded.id}@${decoded.revision}`, + ) + if (latestRevision !== decoded.revision) + throw storeError( + `stale projection revision: ${decoded.tenantId}:${decoded.id}@${decoded.revision}; latest is ${latestRevision}`, + ) + const active = db + .query( + "SELECT revision FROM active_projections WHERE tenant_id = ? AND projection_id = ?", + ) + .get(decoded.tenantId, decoded.id) + const activeRevision = active === null ? null : asNumber(active.revision) + const expectedActiveRevision = decoded.enabled ? decoded.revision : null + if (activeRevision !== expectedActiveRevision) + throw storeError( + `projection active state conflicts with exact revision replay: ${decoded.tenantId}:${decoded.id}@${decoded.revision}`, + ) + return + } else { + const expected = latestRevision === null ? 1 : latestRevision + 1 + if (decoded.revision !== expected) + throw storeError( + `projection revision must be ${expected}: ${decoded.tenantId}:${decoded.id}@${decoded.revision}`, + ) + db.run( + "INSERT INTO projection_revisions (tenant_id, projection_id, revision, enabled, spec_json, created_at) VALUES (?, ?, ?, ?, ?, ?)", + [ + decoded.tenantId, + decoded.id, + decoded.revision, + decoded.enabled ? 1 : 0, + specJson, + createdAt, + ], + ) + } - if (decoded.enabled) - this.#db.run( - "INSERT INTO active_projections (tenant_id, projection_id, revision) VALUES (?, ?, ?) ON CONFLICT (tenant_id, projection_id) DO UPDATE SET revision = excluded.revision", - [decoded.tenantId, decoded.id, decoded.revision], - ) - else - this.#db.run("DELETE FROM active_projections WHERE tenant_id = ? AND projection_id = ?", [ - decoded.tenantId, - decoded.id, - ]) + if (decoded.enabled) + db.run( + "INSERT INTO active_projections (tenant_id, projection_id, revision) VALUES (?, ?, ?) ON CONFLICT (tenant_id, projection_id) DO UPDATE SET revision = excluded.revision", + [decoded.tenantId, decoded.id, decoded.revision], + ) + else + db.run( + "DELETE FROM active_projections WHERE tenant_id = ? AND projection_id = ?", + [decoded.tenantId, decoded.id], + ) + }).immediate() + }, + catch: storeFailure, }) - .immediate() - } - loadEnabledProjections(tenantId: string): readonly SignalProjectionSpec[] { - return this.#db - .query( - `SELECT r.spec_json + const loadEnabledProjections: LocalEventingControlStoreApi["loadEnabledProjections"] = (tenantId) => + Effect.try({ + try: () => + db + .query( + `SELECT r.spec_json FROM active_projections a JOIN projection_revisions r ON r.tenant_id = a.tenant_id @@ -581,795 +902,825 @@ export class LocalEventingControlStore { AND r.revision = a.revision WHERE a.tenant_id = ? ORDER BY a.projection_id`, - ) - .all(tenantId) - .map(({ spec_json }) => decodeProjection(spec_json)) - } + ) + .all(tenantId) + .map(({ spec_json }) => decodeProjection(spec_json)), + catch: storeFailure, + }) - stageEvents( - events: readonly MapleCloudEvent[], - sourceFingerprints: ReadonlyMap = new Map(), - stagedAt = new Date().toISOString(), - ): StageEventsResult { - let inserted = 0 - let deduplicated = 0 - const droppedByTenant = new Map() - let dropped = 0 - const eventIds: string[] = [] - try { - this.#db - .transaction(() => { - const usage = this.#outboxUsage() - if (!usage) throw new Error("event outbox usage query returned no row") - let outboxEvents = asNumber(usage.count) - let outboxBytes = asNumber(usage.bytes) - for (const candidate of events) { - const validated = Result.getOrThrow(validateMapleCloudEvent(candidate)) - const { event, canonicalJson: eventJson, byteLength: eventBytes } = validated - const sourceFingerprint = sourceFingerprints.get(event.id) ?? null - if (sourceFingerprint !== null && !/^sha256:[0-9a-f]{64}$/.test(sourceFingerprint)) - throw new Error(`event has invalid source fingerprint: ${event.id}`) - if (event.sourceoccurrenceid !== undefined && sourceFingerprint === null) - throw new Error( - `event with source occurrence ID requires a source fingerprint: ${event.id}`, - ) - let sourceKind: string | null = null - if (event.sourceoccurrenceid !== undefined) { - const projection = this.#db - .query( - "SELECT spec_json FROM projection_revisions WHERE tenant_id = ? AND projection_id = ? AND revision = ?", - ) - .get(event.tenantid, event.projectionid, event.projectionrevision) - if (projection === null) - throw new Error( - `event references unknown projection revision: ${event.tenantid}:${event.projectionid}@${event.projectionrevision}`, + const stageEvents: LocalEventingControlStoreApi["stageEvents"] = ( + events, + sourceFingerprints = new Map(), + stagedAt = now(), + ) => + Effect.try({ + try: () => + db + .transaction(() => { + let inserted = 0 + let deduplicated = 0 + const droppedByTenant = new Map() + let dropped = 0 + const eventIds: string[] = [] + const usage = outboxUsage() + let outboxEvents = asNumber(usage.count) + let outboxBytes = asNumber(usage.bytes) + for (const candidate of events) { + const validation = validateMapleCloudEvent(candidate) + if (Result.isFailure(validation)) throw validation.failure + const { + event, + canonicalJson: eventJson, + byteLength: eventBytes, + } = validation.success + const sourceFingerprint = sourceFingerprints.get(event.id) ?? null + if ( + sourceFingerprint !== null && + !/^sha256:[0-9a-f]{64}$/.test(sourceFingerprint) ) - sourceKind = decodeProjection(projection.spec_json).sourceKind - } - const existing = this.#db - .query( - "SELECT event_id, event_json, state, source_fingerprint FROM outbox_events WHERE event_id = ?", - ) - .get(event.id) - if (existing) { - if (existing.event_json !== eventJson) - throw new Error(`event ID collision with different payload: ${event.id}`) - if ( - sourceFingerprint !== null && - existing.source_fingerprint !== null && - existing.source_fingerprint !== sourceFingerprint - ) - throw new Error( - `event ID collision with different source occurrence: ${event.id}`, + throw storeError(`event has invalid source fingerprint: ${event.id}`) + if (event.sourceoccurrenceid !== undefined && sourceFingerprint === null) + throw storeError( + `event with source occurrence ID requires a source fingerprint: ${event.id}`, + ) + let sourceKind: string | null = null + if (event.sourceoccurrenceid !== undefined) { + const projection = db + .query( + "SELECT spec_json FROM projection_revisions WHERE tenant_id = ? AND projection_id = ? AND revision = ?", + ) + .get(event.tenantid, event.projectionid, event.projectionrevision) + if (projection === null) + throw storeError( + `event references unknown projection revision: ${event.tenantid}:${event.projectionid}@${event.projectionrevision}`, + ) + sourceKind = decodeProjection(projection.spec_json).sourceKind + } + const existing = db + .query( + "SELECT event_id, event_json, state, source_fingerprint FROM outbox_events WHERE event_id = ?", + ) + .get(event.id) + if (existing) { + if (existing.event_json !== eventJson) + throw storeError( + `event ID collision with different payload: ${event.id}`, + ) + if ( + sourceFingerprint !== null && + existing.source_fingerprint !== null && + existing.source_fingerprint !== sourceFingerprint + ) + throw storeError( + `event ID collision with different source occurrence: ${event.id}`, + ) + if ( + existing.state === "staged" && + sourceFingerprint !== null && + existing.source_fingerprint === null + ) + throw storeError( + `staged event has no recovery fingerprint: ${event.id}`, + ) + deduplicated += 1 + } else { + if ( + outboxEvents + 1 > limits.maxOutboxEvents || + outboxBytes + eventBytes > limits.maxOutboxBytes + ) { + dropped += 1 + droppedByTenant.set( + event.tenantid, + (droppedByTenant.get(event.tenantid) ?? 0) + 1, + ) + continue + } + db.run( + "INSERT INTO outbox_events (event_id, tenant_id, projection_id, projection_revision, source_kind, source, source_occurrence_id, source_fingerprint, state, event_json, staged_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'staged', ?, ?)", + [ + event.id, + event.tenantid, + event.projectionid, + event.projectionrevision, + sourceKind, + event.sourceoccurrenceid === undefined ? null : event.source, + event.sourceoccurrenceid ?? null, + sourceFingerprint, + eventJson, + stagedAt, + ], + ) + inserted += 1 + outboxEvents += 1 + outboxBytes += eventBytes + } + eventIds.push(event.id) + } + for (const [tenantId, count] of droppedByTenant) + recordDeliveryGap(tenantId, count, stagedAt) + return { inserted, deduplicated, dropped, eventIds } + }) + .immediate(), + catch: storeFailure, + }).pipe( + Effect.tapError(() => observeEventing({ operation: "outbox_stage", outcome: "failure" })), + Effect.tap(() => refreshStagedSourceKinds), + Effect.tap((result) => + Effect.all( + [ + observeEventing({ + operation: "outbox_stage", + outcome: "success", + count: result.inserted, + }), + observeEventing({ + operation: "outbox_dedup", + outcome: "success", + count: result.deduplicated, + }), + observeEventing({ + operation: "outbox_stage", + outcome: "dropped", + count: result.dropped, + }), + ], + { discard: true }, + ), + ), + ) + + const deliveryGap: LocalEventingControlStoreApi["deliveryGap"] = (tenantId) => + Effect.try({ try: () => readDeliveryGap(tenantId), catch: storeFailure }) + + const acceptDeliveryGap: LocalEventingControlStoreApi["acceptDeliveryGap"] = ( + tenantId, + consumerId, + generation, + ) => + Effect.try({ + try: () => { + validateConsumerId(consumerId) + return db + .transaction(() => { + const consumer = consumerRow(tenantId, consumerId) + if (consumer === null) + throw EventConsumerNotFoundError.create( + `event consumer not found: ${consumerId}`, + consumerId, ) + const gap = readDeliveryGap(tenantId) if ( - existing.state === "staged" && - sourceFingerprint !== null && - existing.source_fingerprint === null + !Number.isSafeInteger(generation) || + generation < 1 || + generation !== gap.generation ) - throw new Error(`staged event has no recovery fingerprint: ${event.id}`) - deduplicated += 1 - } else { - if ( - outboxEvents + 1 > this.#limits.maxOutboxEvents || - outboxBytes + eventBytes > this.#limits.maxOutboxBytes - ) { - dropped += 1 - droppedByTenant.set( - event.tenantid, - (droppedByTenant.get(event.tenantid) ?? 0) + 1, + throw EventConsumerConflictError.create( + "delivery gap generation changed; inspect current health before accepting", + consumerId, ) - continue - } - this.#db.run( - "INSERT INTO outbox_events (event_id, tenant_id, projection_id, projection_revision, source_kind, source, source_occurrence_id, source_fingerprint, state, event_json, staged_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'staged', ?, ?)", - [ - event.id, - event.tenantid, - event.projectionid, - event.projectionrevision, - sourceKind, - event.sourceoccurrenceid === undefined ? null : event.source, - event.sourceoccurrenceid ?? null, - sourceFingerprint, - eventJson, - stagedAt, - ], + db.run( + "UPDATE event_consumers SET accepted_gap_generation = ? WHERE tenant_id = ? AND consumer_id = ?", + [generation, tenantId, consumerId], ) - inserted += 1 - outboxEvents += 1 - outboxBytes += eventBytes - } - eventIds.push(event.id) - } - for (const [tenantId, count] of droppedByTenant) - this.#recordDeliveryGap(tenantId, count, stagedAt) - }) - .immediate() - } catch (error) { - this.#telemetry.record({ operation: "outbox_stage", outcome: "failure" }) - throw error - } - this.#refreshStagedSourceKinds() - this.#telemetry.record({ operation: "outbox_stage", outcome: "success", count: inserted }) - this.#telemetry.record({ operation: "outbox_dedup", outcome: "success", count: deduplicated }) - this.#telemetry.record({ operation: "outbox_stage", outcome: "dropped", count: dropped }) - return { inserted, deduplicated, dropped, eventIds } - } - - deliveryGap(tenantId: string): DeliveryGap { - const statement = this.#db.prepare( - "SELECT generation, dropped_events, last_dropped_at FROM delivery_gaps WHERE tenant_id = ?", - ) - try { - const row = statement.get(tenantId) - return row === null - ? { generation: 0, droppedEvents: 0, lastDroppedAt: null } - : { - generation: asNumber(row.generation), - droppedEvents: asNumber(row.dropped_events), - lastDroppedAt: row.last_dropped_at, - } - } finally { - statement.finalize() - } - } - #recordDeliveryGap(tenantId: string, count: number, at: string): void { - if (count === 0) return - this.#db.run( - `INSERT INTO delivery_gaps (tenant_id, generation, dropped_events, last_dropped_at) VALUES (?, 1, ?, ?) - ON CONFLICT (tenant_id) DO UPDATE SET generation = generation + 1, dropped_events = dropped_events + excluded.dropped_events, last_dropped_at = excluded.last_dropped_at`, - [tenantId, count, at], - ) - } - acceptDeliveryGap(tenantId: string, consumerId: string, generation: number): DeliveryGap { - validateConsumerId(consumerId) - return this.#db - .transaction(() => { - const consumer = this.#consumer(tenantId, consumerId) - if (consumer === null) - throw EventConsumerNotFoundError.create( - `event consumer not found: ${consumerId}`, - consumerId, - ) - const gap = this.deliveryGap(tenantId) - if (!Number.isSafeInteger(generation) || generation < 1 || generation !== gap.generation) - throw EventConsumerConflictError.create( - "delivery gap generation changed; inspect current health before accepting", - consumerId, - ) - this.#db.run( - "UPDATE event_consumers SET accepted_gap_generation = ? WHERE tenant_id = ? AND consumer_id = ?", - [generation, tenantId, consumerId], - ) - return gap - }) - .immediate() - } - /** Operator-authorized loss; the HTTP caller drains admission before invoking this transaction. */ - abandonEvents( - tenantId: string, - eventIds: readonly string[], - ): { readonly abandoned: number; readonly gap: DeliveryGap } { - if (eventIds.length < 1 || eventIds.length > 1000 || new Set(eventIds).size !== eventIds.length) - throw new OutboxAdministrationInvalid({ message: "abandon requires 1–1000 distinct event IDs" }) - const result = this.#db - .transaction(() => { - const lookup = this.#db.prepare( - "SELECT event_id FROM outbox_events WHERE tenant_id = ? AND event_id = ?", - ) - try { - for (const eventId of eventIds) - if (lookup.get(tenantId, eventId) === null) - throw new OutboxAdministrationInvalid({ - message: `unknown event ID for abandonment: ${eventId}`, - }) - } finally { - lookup.finalize() - } - for (const eventId of eventIds) { - this.#db.run("DELETE FROM outbox_ready_events WHERE event_id = ?", [eventId]) - this.#db.run("DELETE FROM outbox_events WHERE tenant_id = ? AND event_id = ?", [ - tenantId, - eventId, - ]) - } - this.#recordDeliveryGap(tenantId, eventIds.length, new Date().toISOString()) - this.#db.run( - "UPDATE event_consumers SET lease_token_hash = NULL, lease_expires_at = NULL, claimed_through_sequence = NULL WHERE tenant_id = ?", - [tenantId], - ) - return { abandoned: eventIds.length, gap: this.deliveryGap(tenantId) } + return gap + }) + .immediate() + }, + catch: consumerFailure, }) - .immediate() - this.#refreshStagedSourceKinds() - this.#telemetry.record({ operation: "outbox_abandon", outcome: "success", count: result.abandoned }) - return result - } - - #refreshStagedSourceKinds(): void { - const statement = this.#db.prepare<{ tenant_id: string; source_kind: string }, []>( - "SELECT DISTINCT tenant_id, source_kind FROM outbox_events WHERE state = 'staged' AND source_kind IS NOT NULL", - ) - try { - this.#stagedSourceKinds = new Set( - statement.all().map((row) => JSON.stringify([row.tenant_id, row.source_kind])), - ) - } finally { - statement.finalize() - } - } - #outboxUsage(): OutboxUsageRow { - const statement = this.#db.prepare( - "SELECT count, bytes FROM outbox_usage WHERE singleton = 1", - ) - try { - const usage = statement.get() - if (usage === null) throw new Error("event outbox usage query returned no row") - return usage - } finally { - statement.finalize() - } - } - hasStagedSourceKind(tenantId: string, sourceKind: string): boolean { - return this.#stagedSourceKinds.has(JSON.stringify([tenantId, sourceKind])) - } - hasStagedSourceOccurrence( - tenantId: string, - sourceKind: string, - source: string, - sourceOccurrenceId: string, - ): boolean { - const row = this.#db - .query( - "SELECT count(*) AS count FROM outbox_events WHERE tenant_id = ? AND source_kind = ? AND source = ? AND source_occurrence_id = ? AND state = 'staged'", - ) - .get(tenantId, sourceKind, source, sourceOccurrenceId) - if (row === null) throw new Error("staged source-occurrence query returned no row") - return asNumber(row.count) > 0 - } - - stagedEventIdsForOccurrence( - tenantId: string, - sourceKind: string, - source: string, - sourceOccurrenceId: string, - sourceFingerprint: string, - ): readonly string[] { - const rows = this.#db - .query( - "SELECT event_id, source_fingerprint FROM outbox_events WHERE tenant_id = ? AND source_kind = ? AND source = ? AND source_occurrence_id = ? AND state = 'staged' ORDER BY sequence", + const abandonEvents: LocalEventingControlStoreApi["abandonEvents"] = (tenantId, eventIds) => + Effect.try({ + try: () => { + if ( + eventIds.length < 1 || + eventIds.length > 1000 || + new Set(eventIds).size !== eventIds.length + ) + throw new OutboxAdministrationInvalid({ + message: "abandon requires 1–1000 distinct event IDs", + }) + return db + .transaction(() => { + const lookup = db.prepare( + "SELECT event_id FROM outbox_events WHERE tenant_id = ? AND event_id = ?", + ) + try { + for (const eventId of eventIds) + if (lookup.get(tenantId, eventId) === null) + throw new OutboxAdministrationInvalid({ + message: `unknown event ID for abandonment: ${eventId}`, + }) + } finally { + lookup.finalize() + } + for (const eventId of eventIds) { + db.run("DELETE FROM outbox_ready_events WHERE event_id = ?", [eventId]) + db.run("DELETE FROM outbox_events WHERE tenant_id = ? AND event_id = ?", [ + tenantId, + eventId, + ]) + } + recordDeliveryGap(tenantId, eventIds.length, now()) + db.run( + "UPDATE event_consumers SET lease_token_hash = NULL, lease_expires_at = NULL, claimed_through_sequence = NULL WHERE tenant_id = ?", + [tenantId], + ) + return { abandoned: eventIds.length, gap: readDeliveryGap(tenantId) } + }) + .immediate() + }, + catch: administrationFailure, + }).pipe( + Effect.tap(() => refreshStagedSourceKinds), + Effect.tap((result) => + observeEventing({ + operation: "outbox_abandon", + outcome: "success", + count: result.abandoned, + }), + ), ) - .all(tenantId, sourceKind, source, sourceOccurrenceId) - for (const row of rows) { - if (row.source_fingerprint === null) - throw new Error(`staged source occurrence has no recovery fingerprint: ${row.event_id}`) - if (row.source_fingerprint !== sourceFingerprint) - throw new Error(`staged source occurrence collision: ${sourceOccurrenceId}`) - } - return rows.map(({ event_id }) => event_id) - } - markReady(eventIds: readonly string[], readyAt = new Date().toISOString()): void { - let markedReady = 0 - try { - this.#db - .transaction(() => { - for (const eventId of eventIds) { - const row = this.#db - .query, [string]>( - "SELECT state FROM outbox_events WHERE event_id = ?", - ) - .get(eventId) - if (!row) throw new Error(`cannot mark unknown event ready: ${eventId}`) - if (row.state === "ready") continue - this.#db.run("INSERT INTO outbox_ready_events (event_id, ready_at) VALUES (?, ?)", [ - eventId, - readyAt, - ]) - this.#db.run( - "UPDATE outbox_events SET state = 'ready', ready_at = ? WHERE event_id = ? AND state = 'staged'", - [readyAt, eventId], + const hasStagedSourceKind: LocalEventingControlStoreApi["hasStagedSourceKind"] = ( + tenantId, + sourceKind, + ) => Effect.sync(() => stagedSourceKinds.has(JSON.stringify([tenantId, sourceKind]))) + + const hasStagedSourceOccurrence: LocalEventingControlStoreApi["hasStagedSourceOccurrence"] = ( + tenantId, + sourceKind, + source, + sourceOccurrenceId, + ) => + Effect.try({ + try: () => { + const row = db + .query( + "SELECT count(*) AS count FROM outbox_events WHERE tenant_id = ? AND source_kind = ? AND source = ? AND source_occurrence_id = ? AND state = 'staged'", ) - markedReady += 1 - } - }) - .immediate() - } catch (error) { - this.#telemetry.record({ operation: "outbox_ready", outcome: "failure" }) - throw error - } - this.#refreshStagedSourceKinds() - this.#telemetry.record({ operation: "outbox_ready", outcome: "success", count: markedReady }) - } + .get(tenantId, sourceKind, source, sourceOccurrenceId) + if (row === null) throw storeError("staged source-occurrence query returned no row") + return asNumber(row.count) > 0 + }, + catch: storeFailure, + }) - #listOutbox(state: "ready" | "staged", limit = 100, after = 0): EventingOutboxPage { - if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1_000) - throw new Error("outbox-event limit must be between 1 and 1000") - if (!Number.isSafeInteger(after) || after < 0) - throw new Error("outbox cursor must be a non-negative safe integer") - const rows = - state === "ready" - ? this.#db - .query( - `SELECT readiness.sequence, event.event_json, event.staged_at, readiness.ready_at - FROM outbox_ready_events AS readiness - INNER JOIN outbox_events AS event ON event.event_id = readiness.event_id - WHERE event.state = 'ready' AND readiness.sequence > ? - ORDER BY readiness.sequence - LIMIT ?`, - ) - .all(after, limit + 1) - : this.#db - .query( - `SELECT sequence, event_json, staged_at, ready_at - FROM outbox_events - WHERE state = 'staged' AND sequence > ? - ORDER BY sequence - LIMIT ?`, + const stagedEventIdsForOccurrence: LocalEventingControlStoreApi["stagedEventIdsForOccurrence"] = ( + tenantId, + sourceKind, + source, + sourceOccurrenceId, + sourceFingerprint, + ) => + Effect.try({ + try: () => { + const rows = db + .query( + "SELECT event_id, source_fingerprint FROM outbox_events WHERE tenant_id = ? AND source_kind = ? AND source = ? AND source_occurrence_id = ? AND state = 'staged' ORDER BY sequence", ) - .all(after, limit + 1) - const hasMore = rows.length > limit - const pageRows = hasMore ? rows.slice(0, limit) : rows - const page = pageRows.map(({ sequence, event_json, staged_at, ready_at }) => ({ - sequence: asNumber(sequence), - event: decodeEvent(event_json), - stagedAt: staged_at, - readyAt: ready_at, - })) - return { - events: page, - nextCursor: hasMore ? (page.at(-1)?.sequence ?? null) : null, - } - } - - listReady(limit = 100, after = 0): EventingOutboxPage { - return this.#listOutbox("ready", limit, after) - } + .all(tenantId, sourceKind, source, sourceOccurrenceId) + for (const row of rows) { + if (row.source_fingerprint === null) + throw storeError( + `staged source occurrence has no recovery fingerprint: ${row.event_id}`, + ) + if (row.source_fingerprint !== sourceFingerprint) + throw storeError(`staged source occurrence collision: ${sourceOccurrenceId}`) + } + return rows.map(({ event_id }) => event_id) + }, + catch: storeFailure, + }) - listStaged(limit = 100, after = 0): EventingOutboxPage { - return this.#listOutbox("staged", limit, after) - } + const markReady: LocalEventingControlStoreApi["markReady"] = (eventIds, readyAt = now()) => + Effect.try({ + try: () => + db + .transaction(() => { + let markedReady = 0 + for (const eventId of eventIds) { + const row = db + .query, [string]>( + "SELECT state FROM outbox_events WHERE event_id = ?", + ) + .get(eventId) + if (!row) throw storeError(`cannot mark unknown event ready: ${eventId}`) + if (row.state === "ready") continue + db.run("INSERT INTO outbox_ready_events (event_id, ready_at) VALUES (?, ?)", [ + eventId, + readyAt, + ]) + db.run( + "UPDATE outbox_events SET state = 'ready', ready_at = ? WHERE event_id = ? AND state = 'staged'", + [readyAt, eventId], + ) + markedReady += 1 + } + return markedReady + }) + .immediate(), + catch: storeFailure, + }).pipe( + Effect.tapError(() => observeEventing({ operation: "outbox_ready", outcome: "failure" })), + Effect.tap(() => refreshStagedSourceKinds), + Effect.flatMap((markedReady) => + observeEventing({ operation: "outbox_ready", outcome: "success", count: markedReady }), + ), + ) - listConsumers(tenantId: string): readonly EventConsumer[] { - return this.#db - .query( - `SELECT consumer_id, tenant_id, active, last_acked_sequence, accepted_gap_generation, lease_token_hash, + const listConsumers: LocalEventingControlStoreApi["listConsumers"] = (tenantId) => + Effect.try({ + try: () => + db + .query( + `SELECT consumer_id, tenant_id, active, last_acked_sequence, accepted_gap_generation, lease_token_hash, lease_expires_at, claimed_through_sequence, registered_at, disabled_at FROM event_consumers WHERE tenant_id = ? ORDER BY consumer_id`, - ) - .all(tenantId) - .map(decodeConsumer) - } + ) + .all(tenantId) + .map(decodeConsumer), + catch: storeFailure, + }) - registerConsumer( - tenantId: string, - consumerId: string, - startAt: EventConsumerStart, - registeredAt = new Date().toISOString(), - ): EventConsumer { - validateConsumerId(consumerId) - if (startAt !== "beginning" && startAt !== "latest") - throw EventConsumerInputError.create("startAt must be beginning or latest") - canonicalInstant(registeredAt, "event consumer registeredAt") - return this.#db - .transaction(() => { - const existing = this.#consumer(tenantId, consumerId) - if (existing) - throw EventConsumerConflictError.create( - `event consumer already exists: ${consumerId}`, - consumerId, - ) - const boundary = this.#db - .query( - startAt === "latest" - ? `SELECT max(readiness.sequence) AS sequence + const registerConsumer: LocalEventingControlStoreApi["registerConsumer"] = ( + tenantId, + consumerId, + startAt, + registeredAt = now(), + ) => + Effect.try({ + try: () => { + validateConsumerId(consumerId) + if (startAt !== "beginning" && startAt !== "latest") + throw EventConsumerInputError.create("startAt must be beginning or latest") + canonicalInstant(registeredAt, "event consumer registeredAt") + return db + .transaction(() => { + const existing = consumerRow(tenantId, consumerId) + if (existing) + throw EventConsumerConflictError.create( + `event consumer already exists: ${consumerId}`, + consumerId, + ) + const boundary = db + .query( + startAt === "latest" + ? `SELECT max(readiness.sequence) AS sequence FROM outbox_ready_events AS readiness INNER JOIN outbox_events AS event ON event.event_id = readiness.event_id WHERE event.tenant_id = ?` - : `SELECT min(readiness.sequence) AS sequence + : `SELECT min(readiness.sequence) AS sequence FROM outbox_ready_events AS readiness INNER JOIN outbox_events AS event ON event.event_id = readiness.event_id WHERE event.tenant_id = ?`, - ) - .get(tenantId) - const sequence = boundary?.sequence == null ? 0 : asNumber(boundary.sequence) - const lastAcknowledged = startAt === "beginning" ? Math.max(0, sequence - 1) : sequence - this.#db.run( - "INSERT INTO event_consumers (consumer_id, tenant_id, active, last_acked_sequence, registered_at) VALUES (?, ?, 1, ?, ?)", - [consumerId, tenantId, lastAcknowledged, registeredAt], - ) - if (startAt === "latest") - this.#db.run( - "UPDATE event_consumers SET accepted_gap_generation = ? WHERE tenant_id = ? AND consumer_id = ?", - [this.deliveryGap(tenantId).generation, tenantId, consumerId], - ) - const updated = this.#consumer(tenantId, consumerId) - if (updated === null) - throw EventConsumerNotFoundError.create( - `event consumer not found: ${consumerId}`, - consumerId, - ) - return decodeConsumer(updated) + ) + .get(tenantId) + const sequence = boundary?.sequence == null ? 0 : asNumber(boundary.sequence) + const lastAcknowledged = + startAt === "beginning" ? Math.max(0, sequence - 1) : sequence + db.run( + "INSERT INTO event_consumers (consumer_id, tenant_id, active, last_acked_sequence, registered_at) VALUES (?, ?, 1, ?, ?)", + [consumerId, tenantId, lastAcknowledged, registeredAt], + ) + if (startAt === "latest") + db.run( + "UPDATE event_consumers SET accepted_gap_generation = ? WHERE tenant_id = ? AND consumer_id = ?", + [readDeliveryGap(tenantId).generation, tenantId, consumerId], + ) + const updated = consumerRow(tenantId, consumerId) + if (updated === null) + throw EventConsumerNotFoundError.create( + `event consumer not found: ${consumerId}`, + consumerId, + ) + return decodeConsumer(updated) + }) + .immediate() + }, + catch: consumerFailure, }) - .immediate() - } - disableConsumer( - tenantId: string, - consumerId: string, - disabledAt = new Date().toISOString(), - ): EventConsumer { - validateConsumerId(consumerId) - canonicalInstant(disabledAt, "event consumer disabledAt") - return this.#db - .transaction(() => { - const existing = this.#consumer(tenantId, consumerId) - if (!existing) - throw EventConsumerNotFoundError.create( - `unknown event consumer: ${consumerId}`, - consumerId, - ) - if (asNumber(existing.active) === 0) return decodeConsumer(existing) - this.#db.run( - `UPDATE event_consumers + const disableConsumer: LocalEventingControlStoreApi["disableConsumer"] = ( + tenantId, + consumerId, + disabledAt = now(), + ) => + Effect.try({ + try: () => { + validateConsumerId(consumerId) + canonicalInstant(disabledAt, "event consumer disabledAt") + return db + .transaction(() => { + const existing = consumerRow(tenantId, consumerId) + if (!existing) + throw EventConsumerNotFoundError.create( + `unknown event consumer: ${consumerId}`, + consumerId, + ) + if (asNumber(existing.active) === 0) return decodeConsumer(existing) + db.run( + `UPDATE event_consumers SET active = 0, lease_token_hash = NULL, lease_expires_at = NULL, claimed_through_sequence = NULL, disabled_at = ? WHERE tenant_id = ? AND consumer_id = ?`, - [disabledAt, tenantId, consumerId], - ) - this.#pruneAcknowledgedReady(tenantId) - const updated = this.#consumer(tenantId, consumerId) - if (updated === null) - throw EventConsumerNotFoundError.create( - `event consumer not found: ${consumerId}`, - consumerId, - ) - return decodeConsumer(updated) + [disabledAt, tenantId, consumerId], + ) + pruneAcknowledgedReady(tenantId) + const updated = consumerRow(tenantId, consumerId) + if (updated === null) + throw EventConsumerNotFoundError.create( + `event consumer not found: ${consumerId}`, + consumerId, + ) + return decodeConsumer(updated) + }) + .immediate() + }, + catch: consumerFailure, }) - .immediate() - } - claimReady( - tenantId: string, - consumerId: string, - limit: number, - leaseSeconds: number, - now = new Date().toISOString(), - ): EventConsumerClaim { - let reclaimedExpiredLease = false - let lag = 0 - try { - validateConsumerId(consumerId) - if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1_000) - throw EventConsumerInputError.create("claim limit must be between 1 and 1000") - if (!Number.isSafeInteger(leaseSeconds) || leaseSeconds < 5 || leaseSeconds > 300) - throw EventConsumerInputError.create("leaseSeconds must be between 5 and 300") - const nowMilliseconds = canonicalInstant(now, "claim time") - const claim = this.#db - .transaction(() => { - const consumer = this.#consumer(tenantId, consumerId) - if (!consumer) - throw EventConsumerNotFoundError.create( - `unknown event consumer: ${consumerId}`, - consumerId, - ) - const gap = this.deliveryGap(tenantId) - if (gap.generation > asNumber(consumer.accepted_gap_generation)) - throw new EventConsumerDeliveryGapError({ - message: - "Event delivery has a gap; an operator must acknowledge the reported generation before claiming more events", - consumerId, - generation: gap.generation, - droppedEvents: gap.droppedEvents, - }) - if (asNumber(consumer.active) === 0) - throw EventConsumerConflictError.create( - `event consumer is disabled: ${consumerId}`, - consumerId, - ) - if ( - consumer.lease_expires_at !== null && - canonicalInstant(consumer.lease_expires_at, "event consumer leaseExpiresAt") > - nowMilliseconds - ) - throw EventConsumerLeaseError.create( - `event consumer already has an active lease: ${consumerId}`, - consumerId, - consumer.lease_expires_at, - ) - if (consumer.lease_expires_at !== null) reclaimedExpiredLease = true - lag = this.#consumerLag(tenantId, asNumber(consumer.last_acked_sequence)) + const observeConsumerFailure = ( + operation: "consumer_claim" | "consumer_ack", + error: EventConsumerFailure, + ) => + Effect.all( + [ + observeEventing({ operation, outcome: "failure" }), + error._tag === "@maple/cli/eventing/EventConsumerLeaseConflict" + ? observeEventing({ operation: "consumer_lease", outcome: "failure" }) + : Effect.void, + ], + { discard: true }, + ) + + const claimReady: LocalEventingControlStoreApi["claimReady"] = ( + tenantId, + consumerId, + limit, + leaseSeconds, + claimedAt = now(), + ) => + Effect.try({ + try: () => { + validateConsumerId(consumerId) + if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1_000) + throw EventConsumerInputError.create("claim limit must be between 1 and 1000") + if (!Number.isSafeInteger(leaseSeconds) || leaseSeconds < 5 || leaseSeconds > 300) + throw EventConsumerInputError.create("leaseSeconds must be between 5 and 300") + const nowMilliseconds = canonicalInstant(claimedAt, "claim time") + return db + .transaction(() => { + const consumer = consumerRow(tenantId, consumerId) + if (!consumer) + throw EventConsumerNotFoundError.create( + `unknown event consumer: ${consumerId}`, + consumerId, + ) + const gap = readDeliveryGap(tenantId) + if (gap.generation > asNumber(consumer.accepted_gap_generation)) + throw new EventConsumerDeliveryGapError({ + message: + "Event delivery has a gap; an operator must acknowledge the reported generation before claiming more events", + consumerId, + generation: gap.generation, + droppedEvents: gap.droppedEvents, + }) + if (asNumber(consumer.active) === 0) + throw EventConsumerConflictError.create( + `event consumer is disabled: ${consumerId}`, + consumerId, + ) + if ( + consumer.lease_expires_at !== null && + canonicalInstant(consumer.lease_expires_at, "event consumer leaseExpiresAt") > + nowMilliseconds + ) + throw EventConsumerLeaseError.create( + `event consumer already has an active lease: ${consumerId}`, + consumerId, + consumer.lease_expires_at, + ) + const reclaimedExpiredLease = consumer.lease_expires_at !== null + const lag = consumerLag(tenantId, asNumber(consumer.last_acked_sequence)) - const rows = this.#db - .query( - `SELECT readiness.sequence, event.event_json, event.staged_at, readiness.ready_at + const rows = db + .query( + `SELECT readiness.sequence, event.event_json, event.staged_at, readiness.ready_at FROM outbox_ready_events AS readiness INNER JOIN outbox_events AS event ON event.event_id = readiness.event_id WHERE event.tenant_id = ? AND event.state = 'ready' AND readiness.sequence > ? ORDER BY readiness.sequence LIMIT ?`, - ) - .all(tenantId, asNumber(consumer.last_acked_sequence), limit) - if (rows.length === 0) { - this.#db.run( - "UPDATE event_consumers SET lease_token_hash = NULL, lease_expires_at = NULL, claimed_through_sequence = NULL WHERE tenant_id = ? AND consumer_id = ?", - [tenantId, consumerId], - ) - return { - consumerId, - leaseToken: null, - leaseExpiresAt: null, - throughSequence: null, - events: [], - } - } + ) + .all(tenantId, asNumber(consumer.last_acked_sequence), limit) + if (rows.length === 0) { + db.run( + "UPDATE event_consumers SET lease_token_hash = NULL, lease_expires_at = NULL, claimed_through_sequence = NULL WHERE tenant_id = ? AND consumer_id = ?", + [tenantId, consumerId], + ) + const claim: EventConsumerClaim = { + consumerId, + leaseToken: null, + leaseExpiresAt: null, + throughSequence: null, + events: [], + } + return { claim, lag, reclaimedExpiredLease } + } - const leaseToken = randomBytes(32).toString("hex") - const leaseExpiresAt = new Date(nowMilliseconds + leaseSeconds * 1_000).toISOString() - const last = rows.at(-1) - if (last === undefined) throw EventConsumerConflictError.create("empty claim", consumerId) - const throughSequence = asNumber(last.sequence) - this.#db.run( - `UPDATE event_consumers + const leaseToken = randomBytes(32).toString("hex") + const leaseExpiresAt = new Date( + nowMilliseconds + leaseSeconds * 1_000, + ).toISOString() + const last = rows.at(-1) + if (last === undefined) + throw EventConsumerConflictError.create("empty claim", consumerId) + const throughSequence = asNumber(last.sequence) + db.run( + `UPDATE event_consumers SET lease_token_hash = ?, lease_expires_at = ?, claimed_through_sequence = ? WHERE tenant_id = ? AND consumer_id = ?`, - [tokenHash(leaseToken), leaseExpiresAt, throughSequence, tenantId, consumerId], - ) - return { - consumerId, - leaseToken, - leaseExpiresAt, - throughSequence, - events: rows.map(({ sequence, event_json, staged_at, ready_at }) => ({ - sequence: asNumber(sequence), - event: decodeEvent(event_json), - stagedAt: staged_at, - readyAt: ready_at, - })), - } - }) - .immediate() - this.#telemetry.record({ - operation: "consumer_claim", - outcome: claim.events.length === 0 ? "empty" : "success", - count: Math.max(1, claim.events.length), - }) - this.#telemetry.record({ operation: "consumer_lag", outcome: "observed", lag }) - if (reclaimedExpiredLease) - this.#telemetry.record({ operation: "consumer_lease", outcome: "reclaimed" }) - return claim - } catch (error) { - this.#telemetry.record({ operation: "consumer_claim", outcome: "failure" }) - if (Schema.is(EventConsumerLeaseError)(error)) - this.#telemetry.record({ operation: "consumer_lease", outcome: "failure" }) - throw error - } - } + [ + tokenHash(leaseToken), + leaseExpiresAt, + throughSequence, + tenantId, + consumerId, + ], + ) + const claim: EventConsumerClaim = { + consumerId, + leaseToken, + leaseExpiresAt, + throughSequence, + events: rows.map(({ sequence, event_json, staged_at, ready_at }) => ({ + sequence: asNumber(sequence), + event: decodeEvent(event_json), + stagedAt: staged_at, + readyAt: ready_at, + })), + } + return { claim, lag, reclaimedExpiredLease } + }) + .immediate() + }, + catch: consumerFailure, + }).pipe( + Effect.tapError((error) => observeConsumerFailure("consumer_claim", error)), + Effect.tap(({ claim, lag, reclaimedExpiredLease }) => + Effect.all( + [ + observeEventing({ + operation: "consumer_claim", + outcome: claim.events.length === 0 ? "empty" : "success", + count: Math.max(1, claim.events.length), + }), + observeEventing({ operation: "consumer_lag", outcome: "observed", lag }), + reclaimedExpiredLease + ? observeEventing({ operation: "consumer_lease", outcome: "reclaimed" }) + : Effect.void, + ], + { discard: true }, + ), + ), + Effect.map(({ claim }) => claim), + ) - acknowledgeClaim( - tenantId: string, - consumerId: string, - leaseToken: string, - throughSequence: number, - now = new Date().toISOString(), - ): EventConsumerAcknowledgement { - try { - validateConsumerId(consumerId) - if (!Number.isSafeInteger(throughSequence) || throughSequence < 1) - throw EventConsumerInputError.create("throughSequence must be a positive safe integer") - const nowMilliseconds = canonicalInstant(now, "acknowledgement time") - const acknowledgement = this.#db - .transaction(() => { - const consumer = this.#consumer(tenantId, consumerId) - if (!consumer) - throw EventConsumerNotFoundError.create( - `unknown event consumer: ${consumerId}`, - consumerId, - ) - if (asNumber(consumer.active) === 0) - throw EventConsumerConflictError.create( - `event consumer is disabled: ${consumerId}`, - consumerId, - ) - if ( - consumer.lease_token_hash === null || - consumer.lease_expires_at === null || - consumer.claimed_through_sequence === null - ) - throw EventConsumerLeaseError.create( - `event consumer has no active lease: ${consumerId}`, - consumerId, - consumer.lease_expires_at, - ) - if ( - canonicalInstant(consumer.lease_expires_at, "event consumer leaseExpiresAt") <= - nowMilliseconds - ) - throw EventConsumerLeaseError.create( - `event consumer lease has expired: ${consumerId}`, - consumerId, - consumer.lease_expires_at, - ) - if (!tokenHashMatches(consumer.lease_token_hash, leaseToken)) - throw EventConsumerLeaseError.create( - "event consumer lease token does not match", - consumerId, - consumer.lease_expires_at, - ) - const claimedThrough = asNumber(consumer.claimed_through_sequence) - if (throughSequence !== claimedThrough) - throw EventConsumerLeaseError.create( - `acknowledgement must cover the complete claimed batch through sequence ${claimedThrough}`, - consumerId, - consumer.lease_expires_at, + const acknowledgeClaim: LocalEventingControlStoreApi["acknowledgeClaim"] = ( + tenantId, + consumerId, + leaseToken, + throughSequence, + acknowledgedAt = now(), + ) => + Effect.try({ + try: () => { + validateConsumerId(consumerId) + if (!Number.isSafeInteger(throughSequence) || throughSequence < 1) + throw EventConsumerInputError.create( + "throughSequence must be a positive safe integer", ) - this.#db.run( - `UPDATE event_consumers + const nowMilliseconds = canonicalInstant(acknowledgedAt, "acknowledgement time") + return db + .transaction((): EventConsumerAcknowledgement => { + const consumer = consumerRow(tenantId, consumerId) + if (!consumer) + throw EventConsumerNotFoundError.create( + `unknown event consumer: ${consumerId}`, + consumerId, + ) + if (asNumber(consumer.active) === 0) + throw EventConsumerConflictError.create( + `event consumer is disabled: ${consumerId}`, + consumerId, + ) + if ( + consumer.lease_token_hash === null || + consumer.lease_expires_at === null || + consumer.claimed_through_sequence === null + ) + throw EventConsumerLeaseError.create( + `event consumer has no active lease: ${consumerId}`, + consumerId, + consumer.lease_expires_at, + ) + if ( + canonicalInstant( + consumer.lease_expires_at, + "event consumer leaseExpiresAt", + ) <= nowMilliseconds + ) + throw EventConsumerLeaseError.create( + `event consumer lease has expired: ${consumerId}`, + consumerId, + consumer.lease_expires_at, + ) + if (!tokenHashMatches(consumer.lease_token_hash, leaseToken)) + throw EventConsumerLeaseError.create( + "event consumer lease token does not match", + consumerId, + consumer.lease_expires_at, + ) + const claimedThrough = asNumber(consumer.claimed_through_sequence) + if (throughSequence !== claimedThrough) + throw EventConsumerLeaseError.create( + `acknowledgement must cover the complete claimed batch through sequence ${claimedThrough}`, + consumerId, + consumer.lease_expires_at, + ) + db.run( + `UPDATE event_consumers SET last_acked_sequence = ?, lease_token_hash = NULL, lease_expires_at = NULL, claimed_through_sequence = NULL WHERE tenant_id = ? AND consumer_id = ?`, - [throughSequence, tenantId, consumerId], - ) - return { - consumerId, - acknowledgedThrough: throughSequence, - prunedEvents: this.#pruneAcknowledgedReady(tenantId), - } - }) - .immediate() - this.#telemetry.record({ operation: "consumer_ack", outcome: "success" }) - this.#telemetry.record({ - operation: "consumer_lag", - outcome: "observed", - lag: this.#consumerLag(tenantId, acknowledgement.acknowledgedThrough), - }) - return acknowledgement - } catch (error) { - this.#telemetry.record({ operation: "consumer_ack", outcome: "failure" }) - if (Schema.is(EventConsumerLeaseError)(error)) - this.#telemetry.record({ operation: "consumer_lease", outcome: "failure" }) - throw error - } - } - - #consumerLag(tenantId: string, lastAcknowledgedSequence: number): number { - const latest = this.#db - .query( - `SELECT max(readiness.sequence) AS sequence - FROM outbox_ready_events AS readiness - INNER JOIN outbox_events AS event ON event.event_id = readiness.event_id - WHERE event.tenant_id = ? AND event.state = 'ready'`, - ) - .get(tenantId) - return Math.max( - 0, - (latest?.sequence == null ? 0 : asNumber(latest.sequence)) - lastAcknowledgedSequence, - ) - } - - #consumer(tenantId: string, consumerId: string): ConsumerRow | null { - return this.#db - .query( - `SELECT consumer_id, tenant_id, active, last_acked_sequence, accepted_gap_generation, lease_token_hash, - lease_expires_at, claimed_through_sequence, registered_at, disabled_at - FROM event_consumers - WHERE tenant_id = ? AND consumer_id = ?`, - ) - .get(tenantId, consumerId) - } - - #pruneAcknowledgedReady(tenantId: string): number { - const boundary = this.#db - .query( - "SELECT min(last_acked_sequence) AS sequence FROM event_consumers WHERE tenant_id = ? AND active = 1", - ) - .get(tenantId) - if (boundary?.sequence == null) return 0 - const rows = this.#db - .query( - `SELECT readiness.event_id - FROM outbox_ready_events AS readiness - INNER JOIN outbox_events AS event ON event.event_id = readiness.event_id - WHERE event.tenant_id = ? AND readiness.sequence <= ? - ORDER BY readiness.sequence`, + [throughSequence, tenantId, consumerId], + ) + return { + consumerId, + acknowledgedThrough: throughSequence, + prunedEvents: pruneAcknowledgedReady(tenantId), + } + }) + .immediate() + }, + catch: consumerFailure, + }).pipe( + Effect.flatMap((acknowledgement) => + Effect.try({ + try: () => consumerLag(tenantId, acknowledgement.acknowledgedThrough), + catch: storeFailure, + }).pipe( + Effect.tap((lag) => + Effect.all( + [ + observeEventing({ operation: "consumer_ack", outcome: "success" }), + observeEventing({ operation: "consumer_lag", outcome: "observed", lag }), + ], + { discard: true }, + ), + ), + Effect.as(acknowledgement), + ), + ), + Effect.tapError((error) => observeConsumerFailure("consumer_ack", error)), ) - .all(tenantId, asNumber(boundary.sequence)) - const pruneCount = Math.max(0, rows.length - this.#limits.retainAcknowledgedReadyEvents) - for (const { event_id } of rows.slice(0, pruneCount)) { - this.#db.run("DELETE FROM outbox_ready_events WHERE event_id = ?", [event_id]) - this.#db.run("DELETE FROM outbox_events WHERE event_id = ? AND state = 'ready'", [event_id]) - } - return pruneCount - } - outboxCapacity(): LocalEventingControlLimits & { - readonly currentEvents: number - readonly currentBytes: number - } { - const usage = this.#outboxUsage() - if (!usage) throw new Error("event outbox usage query returned no row") - return { - ...this.#limits, - currentEvents: asNumber(usage.count), - currentBytes: asNumber(usage.bytes), - } - } + const outboxCapacity: LocalEventingControlStoreApi["outboxCapacity"] = Effect.try({ + try: () => { + const usage = outboxUsage() + return { + ...limits, + currentEvents: asNumber(usage.count), + currentBytes: asNumber(usage.bytes), + } + }, + catch: storeFailure, + }) - recordProjectionFailures( - tenantId: string, - failures: readonly ProjectionFailure[], - createdAt = new Date().toISOString(), - ): void { - this.#db - .transaction(() => { - for (const failure of failures) - this.#db.run( - "INSERT OR IGNORE INTO projection_failures (tenant_id, projection_id, projection_revision, occurrence_id, message, created_at) VALUES (?, ?, ?, ?, ?, ?)", - [ - tenantId, - failure.projectionId, - failure.projectionRevision, - failure.occurrenceId, - failure.message.slice(0, 4_096), - createdAt, - ], - ) - this.#db.run( - "DELETE FROM projection_failures WHERE tenant_id = ? AND sequence NOT IN (SELECT sequence FROM projection_failures WHERE tenant_id = ? ORDER BY sequence DESC LIMIT ?)", - [tenantId, tenantId, MAX_FAILURES_PER_TENANT], - ) + const recordProjectionFailures: LocalEventingControlStoreApi["recordProjectionFailures"] = ( + tenantId, + failures, + createdAt = now(), + ) => + Effect.try({ + try: () => + db + .transaction(() => { + for (const failure of failures) + db.run( + "INSERT OR IGNORE INTO projection_failures (tenant_id, projection_id, projection_revision, occurrence_id, message, created_at) VALUES (?, ?, ?, ?, ?, ?)", + [ + tenantId, + failure.projectionId, + failure.projectionRevision, + failure.occurrenceId, + failure.message.slice(0, 4_096), + createdAt, + ], + ) + db.run( + "DELETE FROM projection_failures WHERE tenant_id = ? AND sequence NOT IN (SELECT sequence FROM projection_failures WHERE tenant_id = ? ORDER BY sequence DESC LIMIT ?)", + [tenantId, tenantId, MAX_FAILURES_PER_TENANT], + ) + }) + .immediate(), + catch: storeFailure, }) - .immediate() - } - - validate(): EventingControlSnapshotValidation { - return validateOpenDatabase(this.#db) - } - captureSnapshot(): Uint8Array { - checkpointWal(this.#db) - return this.#db.serialize() - } - - static async writeSnapshot(path: string, bytes: Uint8Array): Promise { - await durableWrite(path, bytes) - return LocalEventingControlStore.validateSnapshot(path) - } - - async backupTo(path: string): Promise { - return LocalEventingControlStore.writeSnapshot(path, this.captureSnapshot()) - } - - static validateSnapshot(path: string): EventingControlSnapshotValidation { - assertRealDatabaseFile(path) - if (!existsSync(path)) throw new Error(`eventing control snapshot is missing: ${path}`) - const uri = `${pathToFileURL(path).href}?immutable=1` - const db = new Database(uri, sqliteConstants.SQLITE_OPEN_READONLY | sqliteConstants.SQLITE_OPEN_URI) - try { - configure(db) - return validateOpenDatabase(db) - } finally { - db.close(true) - } - } + return { + path, + saveProjection, + loadEnabledProjections, + stageEvents, + deliveryGap, + acceptDeliveryGap, + abandonEvents, + hasStagedSourceKind, + hasStagedSourceOccurrence, + stagedEventIdsForOccurrence, + markReady, + listReady: (limit, after) => listOutbox("ready", limit, after), + listStaged: (limit, after) => listOutbox("staged", limit, after), + listConsumers, + registerConsumer, + disableConsumer, + claimReady, + acknowledgeClaim, + outboxCapacity, + recordProjectionFailures, + validate: Effect.try({ try: () => validateOpenDatabase(db), catch: storeFailure }), + captureSnapshot, + backupTo: (target) => + Effect.flatMap(captureSnapshot, (bytes) => writeControlSnapshot(target, bytes)), + } satisfies LocalEventingControlStoreApi + }) + + static readonly layer = Layer.effect(this, this.make) +} - static async restoreSnapshot(snapshotPath: string, dataDir: string): Promise { - LocalEventingControlStore.validateSnapshot(snapshotPath) - const stagingDataDir = mkdtempSync( - join(dirname(resolve(dataDir)), ".maple-eventing-control-restore-"), +/** Opens a store outside a layer graph (checkpoint restore, tests); it closes with the scope. */ +export const openControlStore = ( + dataDir: string, + limits?: LocalEventingControlLimits, +): Effect.Effect => + LocalEventingControlStore.make.pipe( + Effect.provideService( + LocalEventingControlConfig, + limits === undefined ? { dataDir } : { dataDir, limits }, + ), + ) + +export const validateControlSnapshot = ( + path: string, +): Effect.Effect => + Effect.try({ + try: () => { + assertRealDatabaseFile(path) + if (!existsSync(path)) throw storeError(`eventing control snapshot is missing: ${path}`) + const uri = `${pathToFileURL(path).href}?immutable=1` + const db = new Database( + uri, + sqliteConstants.SQLITE_OPEN_READONLY | sqliteConstants.SQLITE_OPEN_URI, + ) + try { + configure(db) + return validateOpenDatabase(db) + } finally { + db.close(true) + } + }, + catch: storeFailure, + }) + +export const writeControlSnapshot = ( + path: string, + bytes: Uint8Array, +): Effect.Effect => + Effect.tryPromise({ try: () => durableWrite(path, bytes), catch: storeFailure }).pipe( + Effect.andThen(validateControlSnapshot(path)), + ) + +/** Round-trips the snapshot through a staging store so the target is written by a clean close. */ +export const restoreControlSnapshot = ( + snapshotPath: string, + dataDir: string, +): Effect.Effect => + Effect.gen(function* () { + yield* validateControlSnapshot(snapshotPath) + const stagingDataDir = yield* Effect.acquireRelease( + Effect.try({ + try: () => mkdtempSync(join(dirname(resolve(dataDir)), ".maple-eventing-control-restore-")), + catch: storeFailure, + }), + (directory) => Effect.sync(() => rmSync(directory, { recursive: true, force: true })), ) - let restored: LocalEventingControlStore | undefined - try { - await durableWrite(eventingControlPath(stagingDataDir), readFileSync(snapshotPath)) - restored = await LocalEventingControlStore.open(stagingDataDir) - await restored.backupTo(eventingControlPath(dataDir)) - } finally { - restored?.close() - rmSync(stagingDataDir, { recursive: true, force: true }) - } - } -} + yield* Effect.tryPromise({ + try: () => durableWrite(eventingControlPath(stagingDataDir), readFileSync(snapshotPath)), + catch: storeFailure, + }) + const restored = yield* openControlStore(stagingDataDir) + yield* restored.backupTo(eventingControlPath(dataDir)) + }).pipe(Effect.scoped) diff --git a/apps/cli/src/server/eventing/otlp.ts b/apps/cli/src/server/eventing/otlp.ts index 667cdab59..a8acdba6d 100644 --- a/apps/cli/src/server/eventing/otlp.ts +++ b/apps/cli/src/server/eventing/otlp.ts @@ -6,10 +6,9 @@ import { type NormalizedSignal, type SignalFieldCatalogEntry, type SignalScalar, - type SignalSourceAdapter, type SignalSourceDefinition, } from "@maple/eventing-core" -import { Result, Schema } from "effect" +import { Effect, Result, Schema } from "effect" import { OtlpFieldError, spanIdHex, traceIdHex, type AnyValue, type KeyValue } from "../otlp/encode" const NumberOrString = Schema.Union([Schema.String, Schema.Number]) @@ -70,11 +69,12 @@ const LogsRequestSchema = Schema.Struct({ ), ), }) -const decodeLogsRequest = (request: unknown) => { - const decoded = Schema.decodeUnknownResult(LogsRequestSchema)(request ?? {}) - if (Result.isFailure(decoded)) throw new OtlpFieldError(`invalid OTLP logs: ${decoded.failure.message}`) - return decoded.success -} +const isOtlpFieldError = Schema.is(OtlpFieldError) + +const decodeLogsRequest = (request: unknown): Effect.Effect => + Schema.decodeUnknownEffect(LogsRequestSchema)(request ?? {}).pipe( + Effect.mapError((error) => new OtlpFieldError({ message: `invalid OTLP logs: ${error.message}` })), + ) const MAX_ATTRIBUTES = 256 const MAX_STRING_BYTES = 16 * 1024 @@ -147,20 +147,21 @@ interface ValueBudget { const assertStringBound = (value: string, label: string): string => { if (Buffer.byteLength(value, "utf8") > MAX_STRING_BYTES) - throw new OtlpFieldError(`${label} exceeds ${MAX_STRING_BYTES} UTF-8 bytes`) + throw new OtlpFieldError({ message: `${label} exceeds ${MAX_STRING_BYTES} UTF-8 bytes` }) return value } const int64 = (value: string | number, label: string): string => { if (typeof value === "number" && !Number.isSafeInteger(value)) - throw new OtlpFieldError( - `${label} must encode int64 as a decimal string when outside safe integer range`, - ) + throw new OtlpFieldError({ + message: `${label} must encode int64 as a decimal string when outside safe integer range`, + }) const decimal = String(value) - if (!/^-?(?:0|[1-9][0-9]*)$/.test(decimal)) throw new OtlpFieldError(`${label} is not an int64`) + if (!/^-?(?:0|[1-9][0-9]*)$/.test(decimal)) + throw new OtlpFieldError({ message: `${label} is not an int64` }) const parsed = BigInt(decimal) if (parsed < -(1n << 63n) || parsed > (1n << 63n) - 1n) - throw new OtlpFieldError(`${label} is outside the int64 range`) + throw new OtlpFieldError({ message: `${label} is outside the int64 range` }) return decimal } @@ -171,7 +172,8 @@ const anyValueScalar = (value: AnyValue | undefined, label: string): SignalScala if (value.boolValue !== undefined) return { type: "boolean", value: value.boolValue } if (value.intValue !== undefined) return { type: "int64", value: int64(value.intValue, label) } if (value.doubleValue !== undefined) { - if (!Number.isFinite(value.doubleValue)) throw new OtlpFieldError(`${label} must be finite`) + if (!Number.isFinite(value.doubleValue)) + throw new OtlpFieldError({ message: `${label} must be finite` }) return { type: "float64", value: value.doubleValue } } return null @@ -184,8 +186,9 @@ const anyValueJson = ( budget: ValueBudget = { nodes: 0 }, ): JsonValue | null => { budget.nodes += 1 - if (budget.nodes > MAX_VALUE_NODES) throw new OtlpFieldError(`${label} exceeds value node limit`) - if (depth > MAX_VALUE_DEPTH) throw new OtlpFieldError(`${label} exceeds value depth limit`) + if (budget.nodes > MAX_VALUE_NODES) + throw new OtlpFieldError({ message: `${label} exceeds value node limit` }) + if (depth > MAX_VALUE_DEPTH) throw new OtlpFieldError({ message: `${label} exceeds value depth limit` }) const scalar = anyValueScalar(value, label) if (scalar) return scalar.value if (!value) return null @@ -213,7 +216,7 @@ interface NormalizedAttributes { const attributes = (values: readonly KeyValue[] | undefined, label: string): NormalizedAttributes => { if ((values?.length ?? 0) > MAX_ATTRIBUTES) - throw new OtlpFieldError(`${label} exceeds ${MAX_ATTRIBUTES} attributes`) + throw new OtlpFieldError({ message: `${label} exceeds ${MAX_ATTRIBUTES} attributes` }) const scalars = new Map() const data: Record = Object.create(null) for (const [index, entry] of (values ?? []).entries()) { @@ -228,12 +231,8 @@ const attributes = (values: readonly KeyValue[] | undefined, label: string): Nor const epochNanos = (value: string | number | undefined): bigint | null => { if (value === undefined || value === "" || value === 0 || value === "0") return null - try { - const parsed = BigInt(value) - return parsed >= 0 ? parsed : null - } catch { - return null - } + const parsed = Result.try(() => BigInt(value)) + return Result.isSuccess(parsed) && parsed.success >= 0 ? parsed.success : null } const nanosToTimestamp = (nanos: bigint): string => { @@ -242,7 +241,7 @@ const nanosToTimestamp = (nanos: bigint): string => { const milliseconds = Number(seconds) * 1_000 const date = new Date(milliseconds) if (!Number.isFinite(milliseconds) || Number.isNaN(date.getTime())) - throw new OtlpFieldError("OTLP timestamp is outside the supported date range") + throw new OtlpFieldError({ message: "OTLP timestamp is outside the supported date range" }) return `${date.toISOString().slice(0, 19)}.${fraction.toString().padStart(9, "0")}Z` } @@ -327,12 +326,11 @@ const recoveryIdentity = ( const occurredNanos = epochNanos(log.timeUnixNano) ?? epochNanos(log.observedTimeUnixNano) let occurredAt: string | null = null - if (occurredNanos !== null) - try { - occurredAt = nanosToTimestamp(occurredNanos) - } catch (error) { - if (!(error instanceof OtlpFieldError)) throw error - } + if (occurredNanos !== null) { + const timestamp = Result.try(() => nanosToTimestamp(occurredNanos)) + if (Result.isSuccess(timestamp)) occurredAt = timestamp.success + else if (!isOtlpFieldError(timestamp.failure)) throw timestamp.failure + } return { sourceKind: "otel.log", source, tenantId, occurrenceId, occurredAt } } @@ -375,7 +373,7 @@ const normalizeLogRecord = ( }, } if (Buffer.byteLength(canonicalJson(data), "utf8") > MAX_DATA_BYTES) - throw new OtlpFieldError(`normalized log event exceeds ${MAX_DATA_BYTES} UTF-8 bytes`) + throw new OtlpFieldError({ message: `normalized log event exceeds ${MAX_DATA_BYTES} UTF-8 bytes` }) const source = sourceUri(resource, record) const occurrenceId = sourceOccurrenceId(record) const subject = stringAttribute(record, "event.subject") ?? stringAttribute(record, "cloudevents.subject") @@ -490,12 +488,10 @@ export interface OtlpLogNormalizationResult { } /** Projection limits isolate individual records; resource and scope attributes are normalized once per group. */ -export const normalizeOtlpLogsWithDiagnostics = ( - request: unknown, - _acceptedAt = new Date().toISOString(), - tenantId = "local", +const normalizeDecodedLogs = ( + input: typeof LogsRequestSchema.Type, + tenantId: string, ): OtlpLogNormalizationResult => { - const input = decodeLogsRequest(request) const signals: NormalizedSignal[] = [] const unprojectedIdentities: OtlpRecoveryIdentity[] = [] let ineligible = 0 @@ -515,7 +511,7 @@ export const normalizeOtlpLogsWithDiagnostics = ( ) }) if (Result.isFailure(normalized)) { - if (!(normalized.failure instanceof OtlpFieldError)) throw normalized.failure + if (!isOtlpFieldError(normalized.failure)) throw normalized.failure failures += 1 } else if (normalized.success === null) ineligible += 1 else { @@ -530,16 +526,20 @@ export const normalizeOtlpLogsWithDiagnostics = ( return { signals, unprojectedIdentities, ineligible, failures } } +/** An undecodable request fails as a whole; a malformed record is counted and skipped. */ +export const normalizeOtlpLogsWithDiagnostics = ( + request: unknown, + _acceptedAt = new Date().toISOString(), + tenantId = "local", +): Effect.Effect => + decodeLogsRequest(request).pipe( + // Per-record field errors are already counted, so anything thrown here is a defect. + Effect.flatMap((input) => Effect.sync(() => normalizeDecodedLogs(input, tenantId))), + ) + export const normalizeOtlpLogs = ( request: unknown, acceptedAt = new Date().toISOString(), tenantId = "local", -): readonly NormalizedSignal[] => normalizeOtlpLogsWithDiagnostics(request, acceptedAt, tenantId).signals - -export const OTLP_LOG_ADAPTER: SignalSourceAdapter< - unknown, - { readonly acceptedAt: string; readonly tenantId: string } -> = { - definition: OTLP_LOG_SOURCE, - normalize: (raw, context) => normalizeOtlpLogs(raw, context.acceptedAt, context.tenantId), -} +): Effect.Effect => + normalizeOtlpLogsWithDiagnostics(request, acceptedAt, tenantId).pipe(Effect.map(({ signals }) => signals)) diff --git a/apps/cli/src/server/eventing/runtime.ts b/apps/cli/src/server/eventing/runtime.ts index 956c0e9b3..c2de82651 100644 --- a/apps/cli/src/server/eventing/runtime.ts +++ b/apps/cli/src/server/eventing/runtime.ts @@ -1,4 +1,5 @@ -import { Result } from "effect" +// BOUNDARY: activation candidates and decoded OTLP bodies arrive unparsed; this runtime decodes them. +import { Context, Duration, Effect, Exit, Layer, Result, Schema, SynchronizedRef } from "effect" import { createHash } from "node:crypto" import { canonicalJson, @@ -12,13 +13,28 @@ import { type JsonValue, type NormalizedSignal, type ProjectionFailure, + type ProjectionInvalid, type SignalProjectionSpec, + type SignalSourceInvalid, } from "@maple/eventing-core" -import { Schema } from "effect" -import { LocalEventingControlStore } from "./control-store" -import type { EventConsumerStart } from "./control-store" -import { normalizeOtlpLogsWithDiagnostics, OTLP_LOG_ADAPTER, type OtlpRecoveryIdentity } from "./otlp" -import { NOOP_EVENTING_TELEMETRY, type EventingTelemetry } from "./telemetry" +import type { OtlpFieldError } from "../otlp/encode" +import { + type DeliveryGap, + type EventConsumer, + type EventConsumerAcknowledgement, + type EventConsumerClaim, + type EventConsumerFailure, + type EventConsumerStart, + type EventingControlSnapshotValidation, + type EventingControlStoreError, + type EventingOutboxPage, + LocalEventingControlStore, + type OutboxAdministrationInvalid, + type OutboxCapacity, + type StageEventsResult, +} from "./control-store" +import { normalizeOtlpLogsWithDiagnostics, OTLP_LOG_SOURCE, type OtlpRecoveryIdentity } from "./otlp" +import { observeEventing } from "./telemetry" const TENANT_ID = "local" @@ -34,9 +50,15 @@ export class StagedOccurrenceUnrecoverable extends Schema.TaggedError()( + "@maple/cli/eventing/SourceOccurrenceInvalid", + { message: Schema.String }, +) {} + export class ProjectionActivationInvalid extends Schema.TaggedError()( "@maple/cli/eventing/ProjectionActivationInvalid", - { message: Schema.String }, + { message: Schema.String, cause: Schema.optionalKey(Schema.Defect()) }, ) {} /** Another activation committed between prepare and commit; the request can be retried. */ @@ -45,6 +67,15 @@ export class ProjectionActivationConflict extends Schema.TaggedError @@ -60,6 +91,84 @@ export interface LocalProjectionActivation { readonly generation: number } +export interface LocalEventingHealth extends EventingControlSnapshotValidation { + readonly activeProjections: number + readonly deliveryGap: DeliveryGap + readonly outboxCapacity: OutboxCapacity +} + +export interface LocalEventingRuntimeApi { + readonly hasActiveSource: (sourceKind: string) => Effect.Effect + /** Validation and full registry compilation, run while ingest admission stays open. */ + readonly prepareActivation: ( + candidate: unknown, + ) => Effect.Effect + readonly commitActivation: ( + activation: LocalProjectionActivation, + ) => Effect.Effect + readonly activate: ( + candidate: unknown, + ) => Effect.Effect< + void, + ProjectionActivationInvalid | ProjectionActivationConflict | EventingControlStoreError + > + readonly listActive: Effect.Effect + readonly evaluateOtlp: ( + signal: "traces" | "logs" | "metrics", + decoded: unknown, + isRetiredUtcDay?: (rangeDate: string) => boolean, + ) => Effect.Effect + readonly persistFailures: ( + failures: readonly ProjectionFailure[], + ) => Effect.Effect + readonly stage: ( + events: readonly MapleCloudEvent[], + sourceFingerprints?: ReadonlyMap, + ) => Effect.Effect + readonly markReady: (eventIds: readonly string[]) => Effect.Effect + readonly listReady: ( + limit?: number, + after?: number, + ) => Effect.Effect + readonly listStaged: ( + limit?: number, + after?: number, + ) => Effect.Effect + readonly listConsumers: Effect.Effect + readonly registerConsumer: ( + consumerId: string, + startAt: EventConsumerStart, + ) => Effect.Effect + readonly disableConsumer: (consumerId: string) => Effect.Effect + readonly claimReady: ( + consumerId: string, + limit: number, + leaseSeconds: number, + ) => Effect.Effect + readonly acknowledgeClaim: ( + consumerId: string, + leaseToken: string, + throughSequence: number, + ) => Effect.Effect + readonly acceptDeliveryGap: ( + consumerId: string, + generation: number, + ) => Effect.Effect + readonly abandonEvents: ( + eventIds: readonly string[], + ) => Effect.Effect< + { readonly abandoned: number; readonly gap: DeliveryGap }, + OutboxAdministrationInvalid | EventingControlStoreError + > + readonly health: Effect.Effect +} + +/** The projector implementations the registry compiles against; tests register examples here. */ +export const LocalEventingProjectors = Context.Reference( + "@maple/cli/eventing/LocalEventingProjectors", + { defaultValue: () => new ProjectorRegistry() }, +) + const emptyEvaluation = (): LocalProjectionEvaluation => ({ events: [], eventSourceFingerprints: new Map(), @@ -68,8 +177,13 @@ const emptyEvaluation = (): LocalProjectionEvaluation => ({ typeMismatchFields: [], }) -export const sourceOccurrenceFingerprint = (signal: NormalizedSignal): string => { - if (!isJsonValue(signal.data)) throw new Error("normalized source occurrence must contain finite JSON") +export const sourceOccurrenceFingerprint = ( + signal: NormalizedSignal, +): Result.Result => { + if (!isJsonValue(signal.data)) + return Result.fail( + new SourceOccurrenceInvalid({ message: "normalized source occurrence must contain finite JSON" }), + ) const content: JsonValue = { sourceKind: signal.sourceKind, source: signal.source, @@ -84,7 +198,7 @@ export const sourceOccurrenceFingerprint = (signal: NormalizedSignal): string => .map(([key, value]) => ({ key, value })), data: signal.data, } - return `sha256:${createHash("sha256").update(canonicalJson(content)).digest("hex")}` + return Result.succeed(`sha256:${createHash("sha256").update(canonicalJson(content)).digest("hex")}`) } const sourceOccurrenceKey = ( @@ -102,262 +216,285 @@ const sourceOccurrenceKey = ( const recoveryIdentityKey = (identity: OtlpRecoveryIdentity): string => canonicalJson([identity.tenantId, identity.sourceKind, identity.source, identity.occurrenceId]) -export class LocalEventingRuntime { - readonly #store: LocalEventingControlStore - readonly #sources: SignalSourceRegistry - readonly #projectors: ProjectorRegistry - readonly #telemetry: EventingTelemetry - #compiled: CompiledProjectionRegistry - #activeSourceKinds = new Set() - #generation = 0 +interface RegistryState { + readonly compiled: CompiledProjectionRegistry + readonly activeSourceKinds: ReadonlySet + readonly generation: number +} + +const activationInvalid = (cause: unknown): ProjectionActivationInvalid => + new ProjectionActivationInvalid({ + message: cause instanceof Error ? cause.message : String(cause), + cause, + }) - constructor( - store: LocalEventingControlStore, - telemetry: EventingTelemetry = NOOP_EVENTING_TELEMETRY, - projectors: ProjectorRegistry = new ProjectorRegistry(), - ) { - this.#store = store - this.#telemetry = telemetry - this.#sources = Result.getOrThrow(new SignalSourceRegistry().register(OTLP_LOG_ADAPTER.definition)) - this.#projectors = projectors - const specs = store.loadEnabledProjections(TENANT_ID) - this.#compiled = Result.getOrThrow( - CompiledProjectionRegistry.compile(specs, this.#sources, this.#projectors), +export class LocalEventingRuntime extends Context.Service()( + "@maple/cli/eventing/LocalEventingRuntime", +) { + static readonly make: Effect.Effect< + LocalEventingRuntimeApi, + SignalSourceInvalid | ProjectionInvalid | EventingControlStoreError, + LocalEventingControlStore + > = Effect.gen(function* () { + const store = yield* LocalEventingControlStore + const projectors = yield* LocalEventingProjectors + const sources = yield* Effect.fromResult(new SignalSourceRegistry().register(OTLP_LOG_SOURCE)) + const specs = yield* store.loadEnabledProjections(TENANT_ID) + const compiled = yield* Effect.fromResult( + CompiledProjectionRegistry.compile(specs, sources, projectors), ) - this.#activeSourceKinds = new Set(specs.map(({ sourceKind }) => sourceKind)) - } + // Commits are serialized by the ref, so a stale activation always fails as a conflict. + const registry = yield* SynchronizedRef.make({ + compiled, + activeSourceKinds: new Set(specs.map(({ sourceKind }) => sourceKind)), + generation: 0, + }) - hasActiveSource(sourceKind: string): boolean { - return this.#activeSourceKinds.has(sourceKind) - } + const hasActiveSource = (sourceKind: string): Effect.Effect => + Effect.map(SynchronizedRef.get(registry), ({ activeSourceKinds }) => + activeSourceKinds.has(sourceKind), + ) - prepareActivation(candidate: unknown): LocalProjectionActivation { - assertSignalProjectionInputBudget(candidate) - const spec = Schema.decodeUnknownSync(SignalProjectionSpecSchema)(candidate) - if (spec.tenantId !== TENANT_ID) - throw new ProjectionActivationInvalid({ - message: `Maple Local only accepts projections for tenant ${TENANT_ID}`, - }) - const active = this.#store - .loadEnabledProjections(TENANT_ID) - .filter((candidate) => candidate.id !== spec.id) - const next = spec.enabled ? [...active, spec] : active - const compiled = Result.getOrThrow( - CompiledProjectionRegistry.compile(next, this.#sources, this.#projectors), - ) - return { spec, next, compiled, generation: this.#generation } - } + const listActive = store.loadEnabledProjections(TENANT_ID) - commitActivation(activation: LocalProjectionActivation): void { - if (activation.generation !== this.#generation) - throw new ProjectionActivationConflict({ - message: "projection registry changed during activation; retry the request", + const prepareActivation = Effect.fn("LocalEventing.prepareActivation")(function* ( + candidate: unknown, + ) { + const spec = yield* Effect.try({ + try: () => { + assertSignalProjectionInputBudget(candidate) + return Schema.decodeUnknownSync(SignalProjectionSpecSchema)(candidate) + }, + catch: activationInvalid, }) - this.#store.saveProjection(activation.spec) - this.#compiled = activation.compiled - this.#activeSourceKinds = new Set(activation.next.map(({ sourceKind }) => sourceKind)) - this.#generation += 1 - } - - activate(candidate: unknown): void { - this.commitActivation(this.prepareActivation(candidate)) - } + if (spec.tenantId !== TENANT_ID) + return yield* new ProjectionActivationInvalid({ + message: `Maple Local only accepts projections for tenant ${TENANT_ID}`, + }) + const active = (yield* listActive).filter((candidate) => candidate.id !== spec.id) + const next = spec.enabled ? [...active, spec] : active + const nextCompiled = yield* Effect.fromResult( + CompiledProjectionRegistry.compile(next, sources, projectors), + ).pipe(Effect.mapError(activationInvalid)) + const { generation } = yield* SynchronizedRef.get(registry) + return { spec, next, compiled: nextCompiled, generation } satisfies LocalProjectionActivation + }) - listActive(): readonly SignalProjectionSpec[] { - return this.#store.loadEnabledProjections(TENANT_ID) - } + const commitActivation = Effect.fn("LocalEventing.commitActivation")(function* ( + activation: LocalProjectionActivation, + ) { + yield* SynchronizedRef.modifyEffect( + registry, + ( + state, + ): Effect.Effect< + readonly [void, RegistryState], + ProjectionActivationConflict | EventingControlStoreError + > => + activation.generation !== state.generation + ? Effect.fail( + new ProjectionActivationConflict({ + message: + "projection registry changed during activation; retry the request", + }), + ) + : Effect.as(store.saveProjection(activation.spec), [ + undefined, + { + compiled: activation.compiled, + activeSourceKinds: new Set( + activation.next.map(({ sourceKind }) => sourceKind), + ), + generation: state.generation + 1, + } satisfies RegistryState, + ]), + ) + }) - evaluateOtlp( - signal: "traces" | "logs" | "metrics", - decoded: unknown, - isRetiredUtcDay: (rangeDate: string) => boolean = () => false, - ): LocalProjectionEvaluation { - const sourceKind = signal === "logs" ? "otel.log" : signal === "traces" ? "otel.span" : "otel.metric" - if (!this.hasActiveSource(sourceKind) && !this.#store.hasStagedSourceKind(TENANT_ID, sourceKind)) - return emptyEvaluation() - const startedAt = performance.now() - const acceptedAt = new Date().toISOString() - let normalized - let unprojectedIdentities: readonly OtlpRecoveryIdentity[] - try { - const result = - signal === "logs" - ? normalizeOtlpLogsWithDiagnostics(decoded, acceptedAt, TENANT_ID) - : { signals: [], unprojectedIdentities: [], ineligible: 0, failures: 0 } - normalized = result.signals - unprojectedIdentities = result.unprojectedIdentities - this.#telemetry.record({ + const evaluateOtlp = Effect.fn("LocalEventing.evaluateOtlp")(function* ( + signal: "traces" | "logs" | "metrics", + decoded: unknown, + isRetiredUtcDay: (rangeDate: string) => boolean = () => false, + ) { + const sourceKind = + signal === "logs" ? "otel.log" : signal === "traces" ? "otel.span" : "otel.metric" + if ( + !(yield* hasActiveSource(sourceKind)) && + !(yield* store.hasStagedSourceKind(TENANT_ID, sourceKind)) + ) + return emptyEvaluation() + const acceptedAt = new Date().toISOString() + const [elapsed, normalization] = yield* Effect.timed( + Effect.exit( + signal === "logs" + ? normalizeOtlpLogsWithDiagnostics(decoded, acceptedAt, TENANT_ID) + : Effect.succeed({ + signals: [], + unprojectedIdentities: [], + ineligible: 0, + failures: 0, + }), + ), + ) + if (Exit.isFailure(normalization)) { + yield* observeEventing({ + operation: "normalization", + outcome: "failure", + durationMs: Duration.toMillis(elapsed), + sourceKind, + }) + return yield* Effect.failCause(normalization.cause) + } + const { + signals: normalized, + unprojectedIdentities, + failures: normalizationFailures, + } = normalization.value + yield* observeEventing({ operation: "normalization", outcome: "success", count: normalized.length, - durationMs: performance.now() - startedAt, + durationMs: Duration.toMillis(elapsed), sourceKind, }) - if (result.failures > 0) - this.#telemetry.record({ + if (normalizationFailures > 0) + yield* observeEventing({ operation: "normalization", outcome: "failure", - count: result.failures, + count: normalizationFailures, sourceKind, }) - } catch (error) { - this.#telemetry.record({ - operation: "normalization", - outcome: "failure", - durationMs: performance.now() - startedAt, - sourceKind, - }) - throw error - } - const sourceFingerprints = new Map() - for (const occurrence of normalized) { - const key = sourceOccurrenceKey(occurrence) - if (key === null) continue - const fingerprint = sourceOccurrenceFingerprint(occurrence) - const prior = sourceFingerprints.get(key) - if (prior !== undefined && prior !== fingerprint) - throw new SourceOccurrenceCollision({ - message: `source occurrence collision within one ingest batch: ${occurrence.occurrenceId}`, - occurrenceId: occurrence.occurrenceId, - }) - sourceFingerprints.set(key, fingerprint) - } - for (const identity of unprojectedIdentities) { - if (sourceFingerprints.has(recoveryIdentityKey(identity))) - throw new SourceOccurrenceCollision({ - message: `source occurrence collision with an unprojectable record within one ingest batch: ${identity.occurrenceId}`, - occurrenceId: identity.occurrenceId, - }) - if ( - this.#store.hasStagedSourceOccurrence( - identity.tenantId, - identity.sourceKind, - identity.source, - identity.occurrenceId, + const sourceFingerprints = new Map() + for (const occurrence of normalized) { + const key = sourceOccurrenceKey(occurrence) + if (key === null) continue + const fingerprint = yield* Effect.fromResult(sourceOccurrenceFingerprint(occurrence)) + const prior = sourceFingerprints.get(key) + if (prior !== undefined && prior !== fingerprint) + return yield* new SourceOccurrenceCollision({ + message: `source occurrence collision within one ingest batch: ${occurrence.occurrenceId}`, + occurrenceId: occurrence.occurrenceId, + }) + sourceFingerprints.set(key, fingerprint) + } + for (const identity of unprojectedIdentities) { + if (sourceFingerprints.has(recoveryIdentityKey(identity))) + return yield* new SourceOccurrenceCollision({ + message: `source occurrence collision with an unprojectable record within one ingest batch: ${identity.occurrenceId}`, + occurrenceId: identity.occurrenceId, + }) + if ( + yield* store.hasStagedSourceOccurrence( + identity.tenantId, + identity.sourceKind, + identity.source, + identity.occurrenceId, + ) ) - ) - throw new StagedOccurrenceUnrecoverable({ - message: `cannot safely recover staged source occurrence after projection normalization failed: ${identity.occurrenceId}`, - occurrenceId: identity.occurrenceId, + return yield* new StagedOccurrenceUnrecoverable({ + message: `cannot safely recover staged source occurrence after projection normalization failed: ${identity.occurrenceId}`, + occurrenceId: identity.occurrenceId, + }) + } + const { compiled: snapshot } = yield* SynchronizedRef.get(registry) + const events: MapleCloudEvent[] = [] + const eventSourceFingerprints = new Map() + const recoveredEventIds: string[] = [] + const failures: ProjectionFailure[] = [] + const typeMismatchFields = new Set() + for (const occurrence of normalized) { + const sourceFingerprint = yield* Effect.fromResult(sourceOccurrenceFingerprint(occurrence)) + if (occurrence.occurrenceId !== null) { + const staged = yield* store.stagedEventIdsForOccurrence( + occurrence.tenantId, + occurrence.sourceKind, + occurrence.source, + occurrence.occurrenceId, + sourceFingerprint, + ) + if (staged.length > 0) { + recoveredEventIds.push(...staged) + continue + } + } + if (isRetiredUtcDay(occurrence.occurredAt.slice(0, 10))) continue + const result = yield* Effect.fromResult(snapshot.evaluate(occurrence, acceptedAt)) + yield* observeEventing({ + operation: "projection", + outcome: "success", + count: result.events.length, + sourceKind, }) - } - const snapshot = this.#compiled - const events: MapleCloudEvent[] = [] - const eventSourceFingerprints = new Map() - const recoveredEventIds: string[] = [] - const failures: ProjectionFailure[] = [] - const typeMismatchFields = new Set() - for (const occurrence of normalized) { - const sourceFingerprint = sourceOccurrenceFingerprint(occurrence) - if (occurrence.occurrenceId !== null) { - const staged = this.#store.stagedEventIdsForOccurrence( - occurrence.tenantId, - occurrence.sourceKind, - occurrence.source, - occurrence.occurrenceId, - sourceFingerprint, - ) - if (staged.length > 0) { - recoveredEventIds.push(...staged) - continue + yield* observeEventing({ + operation: "projection", + outcome: "failure", + count: result.failures.length, + sourceKind, + }) + events.push(...result.events) + for (const event of result.events) { + const priorFingerprint = eventSourceFingerprints.get(event.id) + if (priorFingerprint !== undefined && priorFingerprint !== sourceFingerprint) + return yield* new SourceOccurrenceCollision({ + message: `source occurrence collision within one ingest batch: ${event.id}`, + occurrenceId: occurrence.occurrenceId, + }) + eventSourceFingerprints.set(event.id, sourceFingerprint) } + failures.push(...result.failures) + for (const mismatch of result.typeMismatchFields) typeMismatchFields.add(mismatch) } - if (isRetiredUtcDay(occurrence.occurredAt.slice(0, 10))) continue - const result = Result.getOrThrow(snapshot.evaluate(occurrence, acceptedAt)) - this.#telemetry.record({ - operation: "projection", - outcome: "success", - count: result.events.length, - sourceKind, - }) - this.#telemetry.record({ - operation: "projection", - outcome: "failure", - count: result.failures.length, - sourceKind, - }) - events.push(...result.events) - for (const event of result.events) { - const priorFingerprint = eventSourceFingerprints.get(event.id) - if (priorFingerprint !== undefined && priorFingerprint !== sourceFingerprint) - throw new SourceOccurrenceCollision({ - message: `source occurrence collision within one ingest batch: ${event.id}`, - occurrenceId: occurrence.occurrenceId, - }) - eventSourceFingerprints.set(event.id, sourceFingerprint) - } - failures.push(...result.failures) - for (const mismatch of result.typeMismatchFields) typeMismatchFields.add(mismatch) - } - if (typeMismatchFields.size > 0) - this.#telemetry.record({ - operation: "selector_type_mismatch", - outcome: "observed", - count: typeMismatchFields.size, - sourceKind, - }) - return { - events, - eventSourceFingerprints, - recoveredEventIds, - failures, - typeMismatchFields: [...typeMismatchFields], - } - } - - persistFailures(failures: readonly ProjectionFailure[]): void { - if (failures.length > 0) this.#store.recordProjectionFailures(TENANT_ID, failures) - } - - stage(events: readonly MapleCloudEvent[], sourceFingerprints: ReadonlyMap = new Map()) { - return this.#store.stageEvents(events, sourceFingerprints) - } - - markReady(eventIds: readonly string[]): void { - this.#store.markReady(eventIds) - } - - listReady(limit?: number, after?: number) { - return this.#store.listReady(limit, after) - } - - listStaged(limit?: number, after?: number) { - return this.#store.listStaged(limit, after) - } - - listConsumers() { - return this.#store.listConsumers(TENANT_ID) - } - - registerConsumer(consumerId: string, startAt: EventConsumerStart) { - return this.#store.registerConsumer(TENANT_ID, consumerId, startAt) - } - - disableConsumer(consumerId: string) { - return this.#store.disableConsumer(TENANT_ID, consumerId) - } - - claimReady(consumerId: string, limit: number, leaseSeconds: number) { - return this.#store.claimReady(TENANT_ID, consumerId, limit, leaseSeconds) - } - - acknowledgeClaim(consumerId: string, leaseToken: string, throughSequence: number) { - return this.#store.acknowledgeClaim(TENANT_ID, consumerId, leaseToken, throughSequence) - } - - acceptDeliveryGap(consumerId: string, generation: number) { - return this.#store.acceptDeliveryGap(TENANT_ID, consumerId, generation) - } - abandonEvents(eventIds: readonly string[]) { - return this.#store.abandonEvents(TENANT_ID, eventIds) - } + if (typeMismatchFields.size > 0) + yield* observeEventing({ + operation: "selector_type_mismatch", + outcome: "observed", + count: typeMismatchFields.size, + sourceKind, + }) + return { + events, + eventSourceFingerprints, + recoveredEventIds, + failures, + typeMismatchFields: [...typeMismatchFields], + } satisfies LocalProjectionEvaluation + }) - health() { return { - activeProjections: this.listActive().length, - deliveryGap: this.#store.deliveryGap(TENANT_ID), - outboxCapacity: this.#store.outboxCapacity(), - ...this.#store.validate(), - } - } + hasActiveSource, + prepareActivation, + commitActivation, + activate: (candidate) => Effect.flatMap(prepareActivation(candidate), commitActivation), + listActive, + evaluateOtlp, + persistFailures: (failures) => + failures.length > 0 ? store.recordProjectionFailures(TENANT_ID, failures) : Effect.void, + stage: (events, sourceFingerprints = new Map()) => + store.stageEvents(events, sourceFingerprints).pipe(Effect.withSpan("LocalEventing.stage")), + markReady: (eventIds) => + store.markReady(eventIds).pipe(Effect.withSpan("LocalEventing.markReady")), + listReady: store.listReady, + listStaged: store.listStaged, + listConsumers: store.listConsumers(TENANT_ID), + registerConsumer: (consumerId, startAt) => store.registerConsumer(TENANT_ID, consumerId, startAt), + disableConsumer: (consumerId) => store.disableConsumer(TENANT_ID, consumerId), + claimReady: (consumerId, limit, leaseSeconds) => + store.claimReady(TENANT_ID, consumerId, limit, leaseSeconds), + acknowledgeClaim: (consumerId, leaseToken, throughSequence) => + store.acknowledgeClaim(TENANT_ID, consumerId, leaseToken, throughSequence), + acceptDeliveryGap: (consumerId, generation) => + store.acceptDeliveryGap(TENANT_ID, consumerId, generation), + abandonEvents: (eventIds) => store.abandonEvents(TENANT_ID, eventIds), + health: Effect.gen(function* () { + return { + activeProjections: (yield* listActive).length, + deliveryGap: yield* store.deliveryGap(TENANT_ID), + outboxCapacity: yield* store.outboxCapacity, + ...(yield* store.validate), + } + }), + } satisfies LocalEventingRuntimeApi + }) + + static readonly layer = Layer.effect(this, this.make).pipe(Layer.provide(LocalEventingControlStore.layer)) } diff --git a/apps/cli/src/server/eventing/telemetry.ts b/apps/cli/src/server/eventing/telemetry.ts index 89475a4ae..da8523b49 100644 --- a/apps/cli/src/server/eventing/telemetry.ts +++ b/apps/cli/src/server/eventing/telemetry.ts @@ -35,12 +35,6 @@ export interface EventingTelemetryObservation { readonly sourceKind?: EventingTelemetrySourceKind } -export interface EventingTelemetry { - record(observation: EventingTelemetryObservation): void -} - -export const NOOP_EVENTING_TELEMETRY: EventingTelemetry = { record: () => {} } - const operations = Metric.counter("maple.eventing.operations_total", { description: "Eventing operations by bounded operation and outcome", incremental: true, @@ -54,30 +48,24 @@ const consumerLag = Metric.histogram("maple.eventing.consumer_lag_events", { boundaries: [0, 1, 5, 10, 50, 100, 500, 1_000, 10_000], }) -export const makeEffectEventingTelemetry = ( - run: (effect: Effect.Effect) => void, -): EventingTelemetry => ({ - record(observation) { - const attributes = { - operation: observation.operation, - outcome: observation.outcome, - source_kind: observation.sourceKind ?? "unknown", - } - const effects: Effect.Effect[] = [] - const count = observation.count ?? 1 - if (Number.isFinite(count) && count > 0) - effects.push(Metric.update(Metric.withAttributes(operations, attributes), count)) - if (observation.durationMs !== undefined && Number.isFinite(observation.durationMs)) - effects.push( - Metric.update( - Metric.withAttributes(durations, attributes), - Math.max(0, observation.durationMs), - ), - ) - if (observation.lag !== undefined && Number.isSafeInteger(observation.lag)) - effects.push( - Metric.update(Metric.withAttributes(consumerLag, attributes), Math.max(0, observation.lag)), - ) - if (effects.length > 0) run(Effect.all(effects, { discard: true })) - }, -}) +/** Records one bounded observation into the ambient metric registry. */ +export const observeEventing = (observation: EventingTelemetryObservation): Effect.Effect => { + const attributes = { + operation: observation.operation, + outcome: observation.outcome, + source_kind: observation.sourceKind ?? "unknown", + } + const effects: Effect.Effect[] = [] + const count = observation.count ?? 1 + if (Number.isFinite(count) && count > 0) + effects.push(Metric.update(Metric.withAttributes(operations, attributes), count)) + if (observation.durationMs !== undefined && Number.isFinite(observation.durationMs)) + effects.push( + Metric.update(Metric.withAttributes(durations, attributes), Math.max(0, observation.durationMs)), + ) + if (observation.lag !== undefined && Number.isSafeInteger(observation.lag)) + effects.push( + Metric.update(Metric.withAttributes(consumerLag, attributes), Math.max(0, observation.lag)), + ) + return Effect.all(effects, { discard: true }) +} diff --git a/apps/cli/src/server/local-token.ts b/apps/cli/src/server/local-token.ts index af9c4c928..62f4e0118 100644 --- a/apps/cli/src/server/local-token.ts +++ b/apps/cli/src/server/local-token.ts @@ -3,9 +3,16 @@ import { lstatSync, readFileSync } from "node:fs" import { Result, Schema } from "effect" import { durableWrite } from "./durable-files" +/** A credential or ledger path that is a symlink or a non-file; refused rather than followed. */ +export class LocalFileNotReal extends Schema.TaggedError()("@maple/cli/LocalFileNotReal", { + message: Schema.String, + path: Schema.String, +}) {} + export const readRealFile = (path: string, label: string): string => { const stat = lstatSync(path) - if (stat.isSymbolicLink() || !stat.isFile()) throw new Error(`${label} is not a real file: ${path}`) + if (stat.isSymbolicLink() || !stat.isFile()) + throw new LocalFileNotReal({ message: `${label} is not a real file: ${path}`, path }) return readFileSync(path, "utf8") } diff --git a/apps/cli/src/server/otlp/encode.ts b/apps/cli/src/server/otlp/encode.ts index 72c936505..fd576478c 100644 --- a/apps/cli/src/server/otlp/encode.ts +++ b/apps/cli/src/server/otlp/encode.ts @@ -19,6 +19,8 @@ * - attributes are arrays of `{ key, value: AnyValue }` */ +import { Schema } from "effect" + export interface EncodedBatch { datasource: string rowCount: number @@ -96,14 +98,10 @@ function base64ToBytes(b64: string): Uint8Array { * 400 from the ingest handler rather than the generic 500 an encoder crash * would produce. */ -// The pure encoder throws this sentinel for its HTTP adapter to translate into a 400 response. -// oxlint-disable-next-line effecttsgo/extends-native-error -export class OtlpFieldError extends Error { - constructor(message: string) { - super(message) - this.name = "OtlpFieldError" - } -} +// The pure encoder throws this for its HTTP adapter; eventing normalization fails with it. +export class OtlpFieldError extends Schema.TaggedError()("@maple/cli/OtlpFieldError", { + message: Schema.String, +}) {} const HEX_ONLY = /^[0-9a-fA-F]+$/ const ALL_ZERO_HEX = /^0+$/ @@ -151,9 +149,9 @@ function idHex(value: string | undefined, byteLength: number, field: string): st // noise ("deadbeef" is 4 hex bytes, but decodes as 6 base64 bytes). const base64Chars = Math.ceil(byteLength / 3) * 4 const shown = value.length > 80 ? `${value.slice(0, 80)}…` : value - throw new OtlpFieldError( - `${field} must be ${byteLength} bytes: expected ${byteLength * 2} hex chars (OTLP/JSON) or ${base64Chars} base64 chars (OTLP/protobuf), got ${JSON.stringify(shown)} (${value.length} chars)`, - ) + throw new OtlpFieldError({ + message: `${field} must be ${byteLength} bytes: expected ${byteLength * 2} hex chars (OTLP/JSON) or ${base64Chars} base64 chars (OTLP/protobuf), got ${JSON.stringify(shown)} (${value.length} chars)`, + }) } return hexFromBytes(bytes) } diff --git a/apps/cli/src/server/serve.ts b/apps/cli/src/server/serve.ts index b8b4978e8..5364098e4 100644 --- a/apps/cli/src/server/serve.ts +++ b/apps/cli/src/server/serve.ts @@ -3,7 +3,7 @@ // SPA, all on one port, backed by an embedded chDB. Replaces the Rust // `apps/ingest/src/bin/local.rs`. `maple start` calls `startServer`. -import { Effect, Predicate, Result, Schema, type Scope } from "effect" +import { Effect, Layer, Predicate, Result, Schema, type Scope } from "effect" import * as ManagedRuntime from "effect/ManagedRuntime" import { gunzipSync } from "node:zlib" import { TelemetryLayer } from "../core/telemetry" @@ -20,22 +20,14 @@ import { import { buildInsertStatements } from "./inserts" import { eventingControlSnapshotPath, - EventConsumerConflictError, - EventConsumerLeaseError, - EventConsumerDeliveryGapError, - OutboxAdministrationInvalid, - EventConsumerInputError, - EventConsumerNotFoundError, + type EventConsumerFailure, + type EventingControlStoreError, + LocalEventingControlConfig, LocalEventingControlStore, + writeControlSnapshot, } from "./eventing/control-store" import { ensureEventConsumerToken, eventConsumerTokenMatches } from "./eventing/consumer-auth" -import { - LocalEventingRuntime, - ProjectionActivationConflict, - ProjectionActivationInvalid, - SourceOccurrenceCollision, -} from "./eventing/runtime" -import { makeEffectEventingTelemetry } from "./eventing/telemetry" +import { LocalEventingRuntime, type LocalEventingRuntimeApi } from "./eventing/runtime" import { encodeLogs, encodeMetrics, encodeTraces, type EncodedBatch, OtlpFieldError } from "./otlp/encode" import { decodeLogsRequest, @@ -234,147 +226,143 @@ interface IngestResult { readonly requestBytes: number } +/** An ingest step that ends the request early; the carried result is the response. */ +class IngestStopped extends Schema.TaggedError()("@maple/cli/IngestStopped", { + message: Schema.String, + response: Schema.instanceOf(Response), + accepted: Schema.Number, + requestBytes: Schema.Number, +}) {} + +const stopIngest = (response: Response, requestBytes: number, accepted = 0): IngestStopped => + new IngestStopped({ message: `HTTP ${response.status}`, response, accepted, requestBytes }) + /** * A malformed field or an in-batch identity collision fails identically on every * retry, so it is a 400; OTLP exporters retry 503 and would resend the batch forever. */ -const projectionFailureStatus = (error: unknown): 400 | 503 => - error instanceof OtlpFieldError || error instanceof SourceOccurrenceCollision ? 400 : 503 +const projectionFailure = + (signal: Signal, requestBytes: number, status: 400 | 503) => + (error: { readonly message: string }): Effect.Effect => + Effect.fail(stopIngest(text(`event projection ${signal}: ${error.message}`, status), requestBytes)) -async function ingest( +const ingest = ( db: Pick, authority: RetiredDayAuthority, - eventing: LocalEventingRuntime, signal: Signal, req: Request, -): Promise { - let raw: Uint8Array - try { - raw = new Uint8Array(await req.arrayBuffer()) - } catch (error) { - // A client that hangs up mid-body rejects here. Unguarded, this became an - // Effect *defect* — no `error.type`, no 4xx suppression, and a span reading - // only "The connection was closed." It is a caller outcome, so 400 it. - return { - response: text(`read ${signal} body: ${describeThrown(error)}`, 400), - accepted: 0, - requestBytes: 0, - } - } - const requestBytes = raw.length - const contentType = req.headers.get("content-type") ?? "" - const contentEncoding = req.headers.get("content-encoding") - let decoded: unknown - try { - decoded = decodeOtlp(signal, raw, contentType, contentEncoding) - } catch (error) { - return { - response: text(`decode ${signal}: ${describeThrown(error)}`, 400), - accepted: 0, - requestBytes, - } - } - let evaluation: ReturnType - try { - evaluation = eventing.evaluateOtlp(signal, decoded, (rangeDate) => authority.isRetired(rangeDate)) - } catch (error) { - return { - response: text( - `event projection ${signal}: ${describeThrown(error)}`, - projectionFailureStatus(error), - ), - accepted: 0, - requestBytes, - } - } - let batches: EncodedBatch[] - try { - batches = encodeFor(signal, decoded) - } catch (error) { - // A malformed field is the sender's fault, not ours — reject the batch - // with a 400 naming the field instead of silently storing a bad value. - const status = error instanceof OtlpFieldError ? 400 : 500 - const stage = status === 400 ? "decode" : "encode" - return { - response: text(`${stage} ${signal}: ${describeThrown(error)}`, status), - accepted: 0, - requestBytes, - } - } - let stagedEventIds: readonly string[] = [] - let droppedEvents = 0 - try { - eventing.persistFailures(evaluation.failures) - if (evaluation.events.length > 0) { - const staged = eventing.stage(evaluation.events, evaluation.eventSourceFingerprints) - stagedEventIds = staged.eventIds - droppedEvents = staged.dropped - } - } catch (error) { - return { - response: text( - `event projection ${signal}: ${describeThrown(error)}`, - projectionFailureStatus(error), +): Effect.Effect => + Effect.gen(function* () { + const eventing = yield* LocalEventingRuntime + // A client that hangs up mid-body rejects here. It is a caller outcome, so 400 it. + const raw = yield* Effect.tryPromise({ + try: async () => new Uint8Array(await req.arrayBuffer()), + catch: (error) => stopIngest(text(`read ${signal} body: ${describeThrown(error)}`, 400), 0), + }) + const requestBytes = raw.length + const contentType = req.headers.get("content-type") ?? "" + const contentEncoding = req.headers.get("content-encoding") + const decoded = yield* Effect.try({ + try: () => decodeOtlp(signal, raw, contentType, contentEncoding), + catch: (error) => + stopIngest(text(`decode ${signal}: ${describeThrown(error)}`, 400), requestBytes), + }) + const rejectProjection = (status: 400 | 503) => projectionFailure(signal, requestBytes, status) + const evaluation = yield* eventing + .evaluateOtlp(signal, decoded, (rangeDate) => authority.isRetired(rangeDate)) + .pipe( + Effect.catchTags({ + "@maple/cli/OtlpFieldError": rejectProjection(400), + "@maple/cli/eventing/SourceOccurrenceCollision": rejectProjection(400), + "@maple/cli/eventing/StagedOccurrenceUnrecoverable": rejectProjection(503), + "@maple/cli/eventing/SourceOccurrenceInvalid": rejectProjection(503), + "@maple/eventing-core/ProjectionInvalid": rejectProjection(503), + "@maple/cli/eventing/ControlStoreFailed": rejectProjection(503), + }), + Effect.catchDefect((defect) => rejectProjection(503)({ message: describeThrown(defect) })), + ) + // A malformed field is the sender's fault, not ours: reject the batch with a + // 400 naming the field instead of silently storing a bad value. + let batches = yield* Effect.try({ + try: () => encodeFor(signal, decoded), + catch: (error) => + error instanceof OtlpFieldError + ? stopIngest(text(`decode ${signal}: ${error.message}`, 400), requestBytes) + : stopIngest(text(`encode ${signal}: ${describeThrown(error)}`, 500), requestBytes), + }) + const staged = yield* eventing.persistFailures(evaluation.failures).pipe( + Effect.andThen( + evaluation.events.length > 0 + ? eventing.stage(evaluation.events, evaluation.eventSourceFingerprints) + : Effect.succeed({ inserted: 0, deduplicated: 0, dropped: 0, eventIds: [] }), ), - accepted: 0, - requestBytes, - } - } - let rejected = 0 - batches = batches.map((batch) => { - const filtered = authority.filterBatch(batch.datasource, batch.ndjson) - rejected += filtered.rejected - return { ...batch, ndjson: filtered.ndjson, rowCount: filtered.accepted } - }) - let accepted = 0 - for (const batch of batches) { - if (batch.rowCount === 0) continue - for (const statement of buildInsertStatements(batch.datasource, batch.ndjson)) { - try { - db.exec(statement.sql) - } catch (error) { - return { - response: text(`chDB insert (${batch.datasource}): ${describeThrown(error)}`, 500), - accepted, - requestBytes, - } + Effect.catchTag("@maple/cli/eventing/ControlStoreFailed", rejectProjection(503)), + Effect.catchDefect((defect) => rejectProjection(503)({ message: describeThrown(defect) })), + ) + let rejected = 0 + batches = batches.map((batch) => { + const filtered = authority.filterBatch(batch.datasource, batch.ndjson) + rejected += filtered.rejected + return { ...batch, ndjson: filtered.ndjson, rowCount: filtered.accepted } + }) + let accepted = 0 + for (const batch of batches) { + if (batch.rowCount === 0) continue + for (const statement of buildInsertStatements(batch.datasource, batch.ndjson)) { + const inserted = Result.try(() => db.exec(statement.sql)) + if (Result.isFailure(inserted)) + return yield* stopIngest( + text(`chDB insert (${batch.datasource}): ${describeThrown(inserted.failure)}`, 500), + requestBytes, + accepted, + ) + accepted += statement.rowCount } - accepted += statement.rowCount } - } - try { - const readyEventIds = [...evaluation.recoveredEventIds, ...stagedEventIds] - if (readyEventIds.length > 0) eventing.markReady(readyEventIds) - } catch (error) { - return { - response: text(`event outbox readiness ${signal}: ${describeThrown(error)}`, 503), - accepted, - requestBytes, + const readyEventIds = [...evaluation.recoveredEventIds, ...staged.eventIds] + const readinessFailed = (message: string) => + Effect.fail( + stopIngest(text(`event outbox readiness ${signal}: ${message}`, 503), requestBytes, accepted), + ) + if (readyEventIds.length > 0) + yield* eventing.markReady(readyEventIds).pipe( + Effect.catchTag("@maple/cli/eventing/ControlStoreFailed", (error) => + readinessFailed(error.message), + ), + Effect.catchDefect((defect) => readinessFailed(describeThrown(defect))), + ) + const droppedEvents = staged.dropped + const errorMessage = rejected > 0 ? "telemetry from permanently retired UTC days was rejected" : "" + if (contentType.includes("json")) { + const rejectedField = + signal === "traces" + ? { rejectedSpans: rejected } + : signal === "logs" + ? { rejectedLogRecords: rejected } + : { rejectedDataPoints: rejected } + const response = json(rejected > 0 ? { partialSuccess: { ...rejectedField, errorMessage } } : {}) + if (droppedEvents > 0) response.headers.set("x-maple-eventing-dropped", String(droppedEvents)) + return { response, accepted, requestBytes } } - } - const errorMessage = rejected > 0 ? "telemetry from permanently retired UTC days was rejected" : "" - if (contentType.includes("json")) { - const rejectedField = - signal === "traces" - ? { rejectedSpans: rejected } - : signal === "logs" - ? { rejectedLogRecords: rejected } - : { rejectedDataPoints: rejected } - const response = json(rejected > 0 ? { partialSuccess: { ...rejectedField, errorMessage } } : {}) + const response = new Response(encodeExportResponse(signal, rejected, errorMessage), { + status: 200, + headers: { "content-type": "application/x-protobuf" }, + }) if (droppedEvents > 0) response.headers.set("x-maple-eventing-dropped", String(droppedEvents)) - return { - response, - accepted, - requestBytes, - } - } - const response = new Response(encodeExportResponse(signal, rejected, errorMessage), { - status: 200, - headers: { "content-type": "application/x-protobuf" }, - }) - if (droppedEvents > 0) response.headers.set("x-maple-eventing-dropped", String(droppedEvents)) - return { response, accepted, requestBytes } -} + return { response, accepted, requestBytes } + }).pipe( + Effect.catchTag("@maple/cli/IngestStopped", ({ response, accepted, requestBytes }) => + Effect.succeed({ response, accepted, requestBytes }), + ), + // Anything thrown outside a mapped step keeps its old meaning: a 500 with a real message. + Effect.catchDefect((defect) => + Effect.succeed({ + response: text(`ingest ${signal}: ${describeThrown(defect)}`, 500), + accepted: 0, + requestBytes: 0, + }), + ), + ) /** * Strip a trailing `FORMAT ` clause (optionally followed by `;`) and @@ -468,9 +456,12 @@ function serveAsset(assets: AssetResolver, pathname: string): Response { const MAX_DB_QUERY_TEXT = 16 * 1024 const truncateSql = (sql: string) => (sql.length > MAX_DB_QUERY_TEXT ? sql.slice(0, MAX_DB_QUERY_TEXT) : sql) -/** Runs a request's span effect on the server's tracing runtime (see - * `startServer`). The effect always succeeds with a `Response`. */ -type SpanRunner = (effect: Effect.Effect) => Promise +/** The services every request effect may use; `startServer` builds them into one runtime. */ +type RequestServices = LocalEventingRuntime | LocalEventingControlStore + +/** Runs a request's effect on the server's tracing runtime (see `startServer`). + * The effect always succeeds with a `Response`. */ +type SpanRunner = (effect: Effect.Effect) => Promise // A rejected 5xx ingest/query response, surfaced through the Effect error // channel. `message` carries the handler's descriptive body so the span records @@ -478,12 +469,6 @@ type SpanRunner = (effect: Effect.Effect) => Promise // hand back to the client in `recoverResponse`. (Failing with a bare `Response` // recorded an empty `{}` — a `Response` has no enumerable own fields — which lost // the cause entirely and bucketed every failure under one "Error" fingerprint.) -/** A rejection out of `ingest`, carried as a typed failure so it becomes a 500 - * with a real message rather than an untyped defect. */ -class IngestFailed extends Schema.TaggedError()("@maple/cli/IngestFailed", { - message: Schema.String, -}) {} - class IngestRejected extends Schema.TaggedError()("@maple/cli/IngestRejected", { response: Schema.instanceOf(Response), status: Schema.Number, @@ -508,87 +493,60 @@ const recordServerResponse = (response: Response): Effect.Effect): Effect.Effect => +const recoverResponse = ( + self: Effect.Effect, +): Effect.Effect => Effect.match(self, { onFailure: (error) => error.response, onSuccess: (response) => response }) /** OTLP-ingest request as a `Server`-kind span, mirroring the Rust gateway * (`apps/ingest`): `maple.signal`, item count, request size, HTTP semconv. */ const ingestSpan = ( - runSpan: SpanRunner, db: Chdb, authority: RetiredDayAuthority, - eventing: LocalEventingRuntime, signal: Signal, req: Request, -): Promise => - runSpan( - recoverResponse( - Effect.gen(function* () { - // `Effect.promise` is for promises that cannot reject, and `ingest` - // can: it awaits the request body and drives chDB. A rejection there - // became a DEFECT, which `recoverResponse`'s `Effect.match` does not - // catch — so it escaped as an untyped, unlabelled span error instead of - // the 500 the caller should have received. - const { response, accepted, requestBytes } = yield* Effect.tryPromise({ - try: () => ingest(db, authority, eventing, signal, req), - catch: (error): IngestFailed => new IngestFailed({ message: describeThrown(error) }), - }).pipe( - Effect.catchTag("@maple/cli/IngestFailed", (error) => - Effect.succeed({ - response: text(`ingest ${signal}: ${error.message}`, 500), - accepted: 0, - requestBytes: 0, - }), - ), - ) - yield* Effect.annotateCurrentSpan({ - "http.request.body.size": requestBytes, - "maple.ingest.item_count": accepted, - "http.response.status_code": response.status, - }) - return yield* recordServerResponse(response) - }).pipe( - Effect.withSpan(`POST /v1/${signal}`, { - kind: "server", - attributes: { - "maple.signal": signal, - "http.request.method": "POST", - "http.route": `/v1/${signal}`, - }, - }), - ), +): Effect.Effect => + recoverResponse( + Effect.gen(function* () { + const { response, accepted, requestBytes } = yield* ingest(db, authority, signal, req) + yield* Effect.annotateCurrentSpan({ + "http.request.body.size": requestBytes, + "maple.ingest.item_count": accepted, + "http.response.status_code": response.status, + }) + return yield* recordServerResponse(response) + }).pipe( + Effect.withSpan(`POST /v1/${signal}`, { + kind: "server", + attributes: { + "maple.signal": signal, + "http.request.method": "POST", + "http.route": `/v1/${signal}`, + }, + }), ), ) /** `/local/query` request as a `Server`-kind span with the canonical DB attrs. */ -const querySpan = ( - runSpan: SpanRunner, - db: Chdb, - authority: RetiredDayAuthority, - req: Request, -): Promise => - runSpan( - recoverResponse( - Effect.gen(function* () { - const { response, rowCount, durationMs, sql } = yield* Effect.promise(() => - handleQuery(db, authority, req), - ) - yield* Effect.annotateCurrentSpan({ - "db.system.name": "clickhouse", - "db.duration_ms": durationMs, - "result.rowCount": rowCount, - "http.response.status_code": response.status, - ...(sql - ? { "db.query.text": truncateSql(sql), "db.query.length": sql.length } - : undefined), - }) - return yield* recordServerResponse(response) - }).pipe( - Effect.withSpan("POST /local/query", { - kind: "server", - attributes: { "http.request.method": "POST", "http.route": "/local/query" }, - }), - ), +const querySpan = (db: Chdb, authority: RetiredDayAuthority, req: Request): Effect.Effect => + recoverResponse( + Effect.gen(function* () { + const { response, rowCount, durationMs, sql } = yield* Effect.promise(() => + handleQuery(db, authority, req), + ) + yield* Effect.annotateCurrentSpan({ + "db.system.name": "clickhouse", + "db.duration_ms": durationMs, + "result.rowCount": rowCount, + "http.response.status_code": response.status, + ...(sql ? { "db.query.text": truncateSql(sql), "db.query.length": sql.length } : undefined), + }) + return yield* recordServerResponse(response) + }).pipe( + Effect.withSpan("POST /local/query", { + kind: "server", + attributes: { "http.request.method": "POST", "http.route": "/local/query" }, + }), ), ) @@ -613,16 +571,40 @@ export class RequestQuiescenceGate { } } + #drain(): Promise { + return this.#active > 0 + ? new Promise((resolve) => this.#drained.push(resolve)) + : Promise.resolve() + } + async exclusive(work: () => Promise): Promise { if (this.#closed) throw MaintenanceInProgressError.create() this.#closed = true try { - if (this.#active > 0) await new Promise((resolve) => this.#drained.push(resolve)) + await this.#drain() return await work() } finally { this.#closed = false } } + + /** `exclusive` for Effect work; closing is uninterruptible and reopening always runs. */ + exclusiveEffect( + work: Effect.Effect, + ): Effect.Effect { + return Effect.acquireUseRelease( + Effect.suspend(() => { + if (this.#closed) return Effect.fail(MaintenanceInProgressError.create()) + this.#closed = true + return Effect.void + }), + () => Effect.promise(() => this.#drain()).pipe(Effect.andThen(work)), + () => + Effect.sync(() => { + this.#closed = false + }), + ) + } } class MaintenanceInProgressError extends Schema.TaggedError()( @@ -646,57 +628,83 @@ class RequestBodyTooLargeError extends Schema.TaggedError => { - const contentLength = req.headers.get("content-length") - if (contentLength !== null && /^[0-9]+$/.test(contentLength)) { - const declared = Number(contentLength) - if (!Number.isSafeInteger(declared) || declared > maximumBytes) - throw RequestBodyTooLargeError.create(maximumBytes) - } - if (req.body === null) return Schema.decodeUnknownSync(Schema.fromJsonString(Schema.Unknown))("") - const reader = req.body.getReader() - const chunks: Uint8Array[] = [] - let total = 0 - try { - while (true) { - const { done, value } = await reader.read() - if (done) break - total += value.byteLength - if (total > maximumBytes) { - await reader.cancel() - throw RequestBodyTooLargeError.create(maximumBytes) - } - chunks.push(value) +/** The body could not be read or is not JSON. */ +class RequestBodyInvalidError extends Schema.TaggedError()( + "@maple/cli/RequestBodyInvalid", + { message: Schema.String, cause: Schema.Defect() }, +) {} + +const bodyInvalid = (cause: unknown): RequestBodyInvalidError => + new RequestBodyInvalidError({ message: describeThrown(cause), cause }) + +const decodeJsonBody = (body: string): Effect.Effect => + Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Unknown))(body).pipe(Effect.mapError(bodyInvalid)) + +const readBoundedJson = ( + req: Request, + maximumBytes: number, +): Effect.Effect => + Effect.gen(function* () { + const contentLength = req.headers.get("content-length") + if (contentLength !== null && /^[0-9]+$/.test(contentLength)) { + const declared = Number(contentLength) + if (!Number.isSafeInteger(declared) || declared > maximumBytes) + return yield* RequestBodyTooLargeError.create(maximumBytes) } - } finally { - reader.releaseLock() - } - const bytes = new Uint8Array(total) - let offset = 0 - for (const chunk of chunks) { - bytes.set(chunk, offset) - offset += chunk.byteLength - } - return Schema.decodeUnknownSync(Schema.fromJsonString(Schema.Unknown))(new TextDecoder().decode(bytes)) -} + const body = req.body + if (body === null) return yield* decodeJsonBody("") + const bytes = yield* Effect.acquireUseRelease( + Effect.sync(() => body.getReader()), + (reader) => + Effect.gen(function* () { + const chunks: Uint8Array[] = [] + let total = 0 + while (true) { + const { done, value } = yield* Effect.tryPromise({ + try: () => reader.read(), + catch: bodyInvalid, + }) + if (done) break + total += value.byteLength + if (total > maximumBytes) { + yield* Effect.tryPromise({ try: () => reader.cancel(), catch: bodyInvalid }) + return yield* RequestBodyTooLargeError.create(maximumBytes) + } + chunks.push(value) + } + const bytes = new Uint8Array(total) + let offset = 0 + for (const chunk of chunks) { + bytes.set(chunk, offset) + offset += chunk.byteLength + } + return bytes + }), + (reader) => Effect.sync(() => reader.releaseLock()), + ) + return yield* decodeJsonBody(new TextDecoder().decode(bytes)) + }) -const recoverMaintenanceError = (error: unknown, fallback: Response): Response => { - if (error instanceof MaintenanceInProgressError) return text(error.message, 409) - if (error instanceof RequestBodyTooLargeError) return text(error.message, 413) - return fallback -} -const invalidJsonResponse = (error: unknown): Response => - recoverMaintenanceError(error, text("invalid JSON body", 400)) +/** The 409 and 413 a maintenance request reports on purpose; any other body failure is a 400. */ +const recoverBodyFailure = ( + self: Effect.Effect, +): Effect.Effect => + self.pipe( + Effect.catchTags({ + "@maple/cli/RequestBodyTooLarge": (error) => Effect.succeed(text(error.message, 413)), + "@maple/cli/RequestBodyInvalid": () => Effect.succeed(text("invalid JSON body", 400)), + }), + ) -const admitted = async (gate: RequestQuiescenceGate, work: () => Promise): Promise => { - const leave = gate.enter() - if (!leave) return text("server maintenance in progress", 503) - try { - return await work() - } finally { - leave() - } -} +const admitted = ( + gate: RequestQuiescenceGate, + work: Effect.Effect, +): Effect.Effect => + Effect.acquireUseRelease( + Effect.sync(() => gate.enter()), + (leave) => (leave ? work : Effect.succeed(text("server maintenance in progress", 503))), + (leave) => Effect.sync(() => leave?.()), + ) const handleRetirement = async ( db: Chdb, @@ -749,47 +757,64 @@ const MAX_CHECKPOINT_BODY_BYTES = 4 * 1024 const MAX_PROJECTION_BODY_BYTES = 512 * 1024 const MAX_CONSUMER_BODY_BYTES = 16 * 1024 +/** The chDB half of a checkpoint backup failed. */ +class CheckpointBackupFailed extends Schema.TaggedError()( + "@maple/cli/CheckpointBackupFailed", + { message: Schema.String, cause: Schema.Defect() }, +) {} + /** Typed, authenticated replacement for sending BACKUP through /local/query. */ -const handleCheckpointBackup = async ( +const handleCheckpointBackup = ( db: Chdb, - controlStore: LocalEventingControlStore, dataDir: string, gate: RequestQuiescenceGate, token: string, req: Request, -): Promise => { +): Effect.Effect => { if (!maintenanceTokenMatches(token, req.headers.get("x-maple-maintenance-token"))) - return text("maintenance authorization required", 403) - let body: unknown - try { - body = await readBoundedJson(req, MAX_CHECKPOINT_BODY_BYTES) - } catch (error) { - return invalidJsonResponse(error) - } - const decoded = Schema.decodeUnknownResult( - Schema.Struct({ - checkpointId: Schema.String.check(Schema.isPattern(CHECKPOINT_ID)), - }), - { onExcessProperty: "error" }, - )(body) - if (Result.isFailure(decoded)) return text("invalid checkpoint fields", 400) - const record = decoded.success - try { - const checkpointId = record.checkpointId.toLowerCase() - const controlBytes = await gate.exclusive(async () => { - // Both captures are synchronous: no request can mutate either database between them. - const bytes = controlStore.captureSnapshot() - db.exec(`BACKUP DATABASE default TO Disk('default', 'backups/snapshots/${checkpointId}/backup')`) - return bytes - }) - const control = await LocalEventingControlStore.writeSnapshot( + return Effect.succeed(text("maintenance authorization required", 403)) + return Effect.gen(function* () { + const body = yield* readBoundedJson(req, MAX_CHECKPOINT_BODY_BYTES) + const decoded = Schema.decodeUnknownResult( + Schema.Struct({ + checkpointId: Schema.String.check(Schema.isPattern(CHECKPOINT_ID)), + }), + { onExcessProperty: "error" }, + )(body) + if (Result.isFailure(decoded)) return text("invalid checkpoint fields", 400) + const checkpointId = decoded.success.checkpointId.toLowerCase() + const controlStore = yield* LocalEventingControlStore + // The gate has drained every admitted request, so neither database changes between captures. + const controlBytes = yield* gate.exclusiveEffect( + controlStore.captureSnapshot.pipe( + Effect.tap(() => + Effect.try({ + try: () => + db.exec( + `BACKUP DATABASE default TO Disk('default', 'backups/snapshots/${checkpointId}/backup')`, + ), + catch: (cause) => + new CheckpointBackupFailed({ message: describeThrown(cause), cause }), + }), + ), + ), + ) + const control = yield* writeControlSnapshot( eventingControlSnapshotPath(dataDir, checkpointId), controlBytes, ) return json({ checkpointId, control }) - } catch (error) { - return recoverMaintenanceError(error, text(`checkpoint backup failed: ${describeThrown(error)}`, 400)) - } + }).pipe( + Effect.catchTags({ + "@maple/cli/MaintenanceInProgress": (error) => Effect.succeed(text(error.message, 409)), + "@maple/cli/RequestBodyTooLarge": (error) => Effect.succeed(text(error.message, 413)), + "@maple/cli/RequestBodyInvalid": () => Effect.succeed(text("invalid JSON body", 400)), + "@maple/cli/CheckpointBackupFailed": (error) => + Effect.succeed(text(`checkpoint backup failed: ${error.message}`, 400)), + "@maple/cli/eventing/ControlStoreFailed": (error) => + Effect.succeed(text(`checkpoint backup failed: ${error.message}`, 400)), + }), + ) } const eventingAuthorized = (token: string, req: Request): Response | null => @@ -797,233 +822,235 @@ const eventingAuthorized = (token: string, req: Request): Response | null => ? null : text("maintenance authorization required", 403) -const handleProjectionActivation = async ( - eventing: LocalEventingRuntime, +const handleProjectionActivation = ( gate: RequestQuiescenceGate, token: string, req: Request, -): Promise => { +): Effect.Effect => { const unauthorized = eventingAuthorized(token, req) - if (unauthorized) return unauthorized - let body: unknown - try { - body = await readBoundedJson(req, MAX_PROJECTION_BODY_BYTES) - } catch (error) { - return invalidJsonResponse(error) - } - let activation - try { + if (unauthorized) return Effect.succeed(unauthorized) + return Effect.gen(function* () { + const eventing = yield* LocalEventingRuntime + const body = yield* readBoundedJson(req, MAX_PROJECTION_BODY_BYTES) // Recursive schema validation and full registry compilation happen while // normal ingest/query admission remains open. - activation = eventing.prepareActivation(body) - } catch (error) { - return text(`invalid event projection: ${describeThrown(error)}`, 400) - } - try { - await gate.exclusive(async () => eventing.commitActivation(activation)) - return json({ active: eventing.listActive() }) - } catch (error) { - if (error instanceof ProjectionActivationConflict) return text(error.message, 409) - if (error instanceof ProjectionActivationInvalid) return text(error.message, 400) - return recoverMaintenanceError( - error, - text(`event projection activation failed: ${describeThrown(error)}`, 500), + const activation = yield* eventing.prepareActivation(body).pipe(Effect.result) + if (Result.isFailure(activation)) + return text(`invalid event projection: ${activation.failure.message}`, 400) + return yield* gate.exclusiveEffect(eventing.commitActivation(activation.success)).pipe( + Effect.andThen(eventing.listActive), + Effect.map((active) => json({ active })), + Effect.catchTags({ + "@maple/cli/eventing/ProjectionActivationConflict": (error) => + Effect.succeed(text(error.message, 409)), + "@maple/cli/MaintenanceInProgress": (error) => Effect.succeed(text(error.message, 409)), + "@maple/cli/eventing/ControlStoreFailed": (error) => + Effect.succeed(text(`event projection activation failed: ${error.message}`, 500)), + }), + Effect.catchDefect((defect) => + Effect.succeed(text(`event projection activation failed: ${describeThrown(defect)}`, 500)), + ), ) - } + }).pipe(recoverBodyFailure) } -const eventConsumerErrorResponse = (error: unknown): Response => { - if (error instanceof EventConsumerDeliveryGapError) - return json( - { - error: error._tag, - message: error.message, - consumerId: error.consumerId, - generation: error.generation, - droppedEvents: error.droppedEvents, - }, - 409, - ) - if (error instanceof OutboxAdministrationInvalid || error instanceof EventConsumerInputError) - return text(error.message, 400) - if (error instanceof EventConsumerNotFoundError) return text(error.message, 404) - if (error instanceof EventConsumerConflictError || error instanceof EventConsumerLeaseError) - return text(error.message, 409) - return text(`event consumer operation failed: ${describeThrown(error)}`, 500) -} +const recoverEventConsumerFailure = ( + self: Effect.Effect, +): Effect.Effect => + self.pipe( + Effect.catchTags({ + "@maple/cli/eventing/EventConsumerDeliveryGap": (error) => + Effect.succeed( + json( + { + error: error._tag, + message: error.message, + consumerId: error.consumerId, + generation: error.generation, + droppedEvents: error.droppedEvents, + }, + 409, + ), + ), + "@maple/cli/eventing/EventConsumerInputInvalid": (error) => + Effect.succeed(text(error.message, 400)), + "@maple/cli/eventing/EventConsumerNotFound": (error) => Effect.succeed(text(error.message, 404)), + "@maple/cli/eventing/EventConsumerConflict": (error) => Effect.succeed(text(error.message, 409)), + "@maple/cli/eventing/EventConsumerLeaseConflict": (error) => + Effect.succeed(text(error.message, 409)), + "@maple/cli/eventing/ControlStoreFailed": (error) => + Effect.succeed(text(`event consumer operation failed: ${error.message}`, 500)), + }), + Effect.catchDefect((defect) => + Effect.succeed(text(`event consumer operation failed: ${describeThrown(defect)}`, 500)), + ), + ) const ConsumerIdSchema = Schema.String.check(Schema.isPattern(/^[a-z][a-z0-9._-]{0,63}$/)) -const handleConsumerRegistration = async ( - eventing: LocalEventingRuntime, +/** Decodes a bounded consumer body, then runs the store call under ordinary admission. */ +const consumerRequest = ( + gate: RequestQuiescenceGate, + req: Request, + schema: S, + invalidMessage: string, + run: (eventing: LocalEventingRuntimeApi, decoded: S["Type"]) => Effect.Effect, + status = 200, +): Effect.Effect => + Effect.gen(function* () { + const eventing = yield* LocalEventingRuntime + const body = yield* readBoundedJson(req, MAX_CONSUMER_BODY_BYTES) + const decoded = Schema.decodeUnknownResult(schema, { onExcessProperty: "error" })(body) + if (Result.isFailure(decoded)) return text(invalidMessage, 400) + return yield* admitted( + gate, + run(eventing, decoded.success).pipe( + Effect.map((result) => json(result, status)), + recoverEventConsumerFailure, + ), + ) + }).pipe(recoverBodyFailure) + +const handleConsumerRegistration = ( gate: RequestQuiescenceGate, maintenanceToken: string, req: Request, -): Promise => { - const unauthorized = eventingAuthorized(maintenanceToken, req) - if (unauthorized) return unauthorized - let body: unknown - try { - body = await readBoundedJson(req, MAX_CONSUMER_BODY_BYTES) - } catch (error) { - return invalidJsonResponse(error) - } - const decoded = Schema.decodeUnknownResult( - Schema.Struct({ consumerId: ConsumerIdSchema, startAt: Schema.Literals(["beginning", "latest"]) }), - { onExcessProperty: "error" }, - )(body) - if (Result.isFailure(decoded)) return text("invalid event consumer registration fields", 400) - const { consumerId, startAt } = decoded.success - return admitted(gate, async () => { - try { - return json(eventing.registerConsumer(consumerId, startAt), 201) - } catch (error) { - return eventConsumerErrorResponse(error) - } +): Effect.Effect => + Effect.suspend(() => { + const unauthorized = eventingAuthorized(maintenanceToken, req) + if (unauthorized) return Effect.succeed(unauthorized) + return consumerRequest( + gate, + req, + Schema.Struct({ + consumerId: ConsumerIdSchema, + startAt: Schema.Literals(["beginning", "latest"]), + }), + "invalid event consumer registration fields", + (eventing, { consumerId, startAt }) => eventing.registerConsumer(consumerId, startAt), + 201, + ) }) -} -const handleConsumerDisable = async ( - eventing: LocalEventingRuntime, +const handleConsumerDisable = ( gate: RequestQuiescenceGate, maintenanceToken: string, req: Request, -): Promise => { - const unauthorized = eventingAuthorized(maintenanceToken, req) - if (unauthorized) return unauthorized - let body: unknown - try { - body = await readBoundedJson(req, MAX_CONSUMER_BODY_BYTES) - } catch (error) { - return invalidJsonResponse(error) - } - const decoded = Schema.decodeUnknownResult(Schema.Struct({ consumerId: ConsumerIdSchema }), { - onExcessProperty: "error", - })(body) - if (Result.isFailure(decoded)) return text("invalid event consumer disable fields", 400) - const { consumerId } = decoded.success - return admitted(gate, async () => { - try { - return json(eventing.disableConsumer(consumerId)) - } catch (error) { - return eventConsumerErrorResponse(error) - } +): Effect.Effect => + Effect.suspend(() => { + const unauthorized = eventingAuthorized(maintenanceToken, req) + if (unauthorized) return Effect.succeed(unauthorized) + return consumerRequest( + gate, + req, + Schema.Struct({ consumerId: ConsumerIdSchema }), + "invalid event consumer disable fields", + (eventing, { consumerId }) => eventing.disableConsumer(consumerId), + ) }) -} -const handleConsumerClaim = async ( - eventing: Pick, +const consumerUnauthorized = (consumerToken: string, req: Request): Response | null => + eventConsumerTokenMatches(consumerToken, req.headers.get("x-maple-event-consumer-token")) + ? null + : text("event consumer authorization required", 403) + +const handleConsumerClaim = ( gate: RequestQuiescenceGate, consumerToken: string, req: Request, -): Promise => { - if (!eventConsumerTokenMatches(consumerToken, req.headers.get("x-maple-event-consumer-token"))) - return text("event consumer authorization required", 403) - let body: unknown - try { - body = await readBoundedJson(req, MAX_CONSUMER_BODY_BYTES) - } catch (error) { - return invalidJsonResponse(error) - } - const decoded = Schema.decodeUnknownResult( - Schema.Struct({ - consumerId: ConsumerIdSchema, - limit: Schema.Int.check(Schema.isBetween({ minimum: 1, maximum: 1000 })), - leaseSeconds: Schema.Int.check(Schema.isBetween({ minimum: 5, maximum: 300 })), - }), - { onExcessProperty: "error" }, - )(body) - if (Result.isFailure(decoded)) return text("invalid event consumer claim fields", 400) - const { consumerId, limit, leaseSeconds } = decoded.success - return admitted(gate, async () => { - try { - return json(eventing.claimReady(consumerId, limit, leaseSeconds)) - } catch (error) { - return eventConsumerErrorResponse(error) - } +): Effect.Effect => + Effect.suspend(() => { + const unauthorized = consumerUnauthorized(consumerToken, req) + if (unauthorized) return Effect.succeed(unauthorized) + return consumerRequest( + gate, + req, + Schema.Struct({ + consumerId: ConsumerIdSchema, + limit: Schema.Int.check(Schema.isBetween({ minimum: 1, maximum: 1000 })), + leaseSeconds: Schema.Int.check(Schema.isBetween({ minimum: 5, maximum: 300 })), + }), + "invalid event consumer claim fields", + (eventing, { consumerId, limit, leaseSeconds }) => + eventing.claimReady(consumerId, limit, leaseSeconds), + ) }) -} -const handleConsumerAcknowledgement = async ( - eventing: LocalEventingRuntime, +const handleConsumerAcknowledgement = ( gate: RequestQuiescenceGate, consumerToken: string, req: Request, -): Promise => { - if (!eventConsumerTokenMatches(consumerToken, req.headers.get("x-maple-event-consumer-token"))) - return text("event consumer authorization required", 403) - let body: unknown - try { - body = await readBoundedJson(req, MAX_CONSUMER_BODY_BYTES) - } catch (error) { - return invalidJsonResponse(error) - } - const decoded = Schema.decodeUnknownResult( - Schema.Struct({ - consumerId: ConsumerIdSchema, - leaseToken: Schema.String.check(Schema.isPattern(/^[0-9a-f]{64}$/)), - throughSequence: Schema.Int.check(Schema.isGreaterThanOrEqualTo(1)), - }), - { onExcessProperty: "error" }, - )(body) - if (Result.isFailure(decoded)) return text("invalid event consumer acknowledgement fields", 400) - const { consumerId, leaseToken, throughSequence } = decoded.success - return admitted(gate, async () => { - try { - return json(eventing.acknowledgeClaim(consumerId, leaseToken, throughSequence)) - } catch (error) { - return eventConsumerErrorResponse(error) - } +): Effect.Effect => + Effect.suspend(() => { + const unauthorized = consumerUnauthorized(consumerToken, req) + if (unauthorized) return Effect.succeed(unauthorized) + return consumerRequest( + gate, + req, + Schema.Struct({ + consumerId: ConsumerIdSchema, + leaseToken: Schema.String.check(Schema.isPattern(/^[0-9a-f]{64}$/)), + throughSequence: Schema.Int.check(Schema.isGreaterThanOrEqualTo(1)), + }), + "invalid event consumer acknowledgement fields", + (eventing, { consumerId, leaseToken, throughSequence }) => + eventing.acknowledgeClaim(consumerId, leaseToken, throughSequence), + ) }) -} -const handleOutboxAdministration = async ( - eventing: Pick, +const handleOutboxAdministration = ( gate: RequestQuiescenceGate, token: string, req: Request, action: "abandon" | "accept-gap", -): Promise => { +): Effect.Effect => { const unauthorized = eventingAuthorized(token, req) - if (unauthorized) return unauthorized - let body: unknown - try { - body = await readBoundedJson(req, MAX_PROJECTION_BODY_BYTES) - } catch (error) { - return invalidJsonResponse(error) - } - if (action === "abandon") { + if (unauthorized) return Effect.succeed(unauthorized) + return Effect.gen(function* () { + const eventing = yield* LocalEventingRuntime + const body = yield* readBoundedJson(req, MAX_PROJECTION_BODY_BYTES) + if (action === "abandon") { + const decoded = Schema.decodeUnknownResult( + Schema.Struct({ + eventIds: Schema.Array(Schema.NonEmptyString.check(Schema.isMaxLength(256))).check( + Schema.isMinLength(1), + Schema.isMaxLength(1000), + ), + }), + { onExcessProperty: "error" }, + )(body) + if (Result.isFailure(decoded)) return text("invalid outbox abandonment fields", 400) + return yield* gate.exclusiveEffect(eventing.abandonEvents(decoded.success.eventIds)).pipe( + Effect.map((result) => json(result)), + Effect.catchTags({ + "@maple/cli/MaintenanceInProgress": (error) => Effect.succeed(text(error.message, 409)), + "@maple/cli/eventing/OutboxAdministrationInvalid": (error) => + Effect.succeed(text(error.message, 400)), + "@maple/cli/eventing/ControlStoreFailed": (error) => + Effect.succeed(text(`event consumer operation failed: ${error.message}`, 500)), + }), + Effect.catchDefect((defect) => + Effect.succeed(text(`event consumer operation failed: ${describeThrown(defect)}`, 500)), + ), + ) + } const decoded = Schema.decodeUnknownResult( Schema.Struct({ - eventIds: Schema.Array(Schema.NonEmptyString.check(Schema.isMaxLength(256))).check( - Schema.isMinLength(1), - Schema.isMaxLength(1000), - ), + consumerId: ConsumerIdSchema, + generation: Schema.Int.check(Schema.isGreaterThan(0)), }), { onExcessProperty: "error" }, )(body) - if (Result.isFailure(decoded)) return text("invalid outbox abandonment fields", 400) - try { - return json(await gate.exclusive(async () => eventing.abandonEvents(decoded.success.eventIds))) - } catch (error) { - return recoverMaintenanceError(error, eventConsumerErrorResponse(error)) - } - } - const decoded = Schema.decodeUnknownResult( - Schema.Struct({ - consumerId: ConsumerIdSchema, - generation: Schema.Int.check(Schema.isGreaterThan(0)), - }), - { onExcessProperty: "error" }, - )(body) - if (Result.isFailure(decoded)) return text("invalid delivery gap acknowledgement fields", 400) - return admitted(gate, async () => { - try { - return json(eventing.acceptDeliveryGap(decoded.success.consumerId, decoded.success.generation)) - } catch (error) { - return eventConsumerErrorResponse(error) - } - }) + if (Result.isFailure(decoded)) return text("invalid delivery gap acknowledgement fields", 400) + const { consumerId, generation } = decoded.success + return yield* admitted( + gate, + eventing.acceptDeliveryGap(consumerId, generation).pipe( + Effect.map((gap) => json(gap)), + recoverEventConsumerFailure, + ), + ) + }).pipe(recoverBodyFailure) } const decodeOutboxQuery = Schema.decodeUnknownResult( @@ -1039,30 +1066,44 @@ const decodeOutboxQuery = Schema.decodeUnknownResult( { onExcessProperty: "error" }, ) +/** A read the store could not serve is a 500 naming the read. */ +const readFailed = + (label: string) => + (self: Effect.Effect): Effect.Effect => + self.pipe( + Effect.catchTag("@maple/cli/eventing/ControlStoreFailed", (error) => + Effect.succeed(text(`${label} failed: ${error.message}`, 500)), + ), + Effect.catchDefect((defect) => + Effect.succeed(text(`${label} failed: ${describeThrown(defect)}`, 500)), + ), + ) + const handleEventingRead = ( - eventing: LocalEventingRuntime, token: string, req: Request, url: URL, -): Response => { +): Effect.Effect => { const unauthorized = eventingAuthorized(token, req) - if (unauthorized) return unauthorized - if (url.pathname === "/local/eventing/health") return json(eventing.health()) - if (url.pathname === "/local/eventing/projections") return json(eventing.listActive()) - if (url.pathname === "/local/eventing/consumers") return json(eventing.listConsumers()) - if (url.pathname === "/local/eventing/outbox") { - const query = decodeOutboxQuery(Object.fromEntries(url.searchParams)) - if (Result.isFailure(query)) return text(`invalid outbox query: ${query.failure.message}`, 400) - const { state = "ready", limit = 100, after = 0 } = query.success - try { - return json( - state === "ready" ? eventing.listReady(limit, after) : eventing.listStaged(limit, after), - ) - } catch (error) { - return text(`outbox read failed: ${describeThrown(error)}`, 500) + if (unauthorized) return Effect.succeed(unauthorized) + return Effect.gen(function* () { + const eventing = yield* LocalEventingRuntime + if (url.pathname === "/local/eventing/health") + return yield* eventing.health.pipe(Effect.map(json), readFailed("eventing health read")) + if (url.pathname === "/local/eventing/projections") + return yield* eventing.listActive.pipe(Effect.map(json), readFailed("projection read")) + if (url.pathname === "/local/eventing/consumers") + return yield* eventing.listConsumers.pipe(Effect.map(json), readFailed("consumer read")) + if (url.pathname === "/local/eventing/outbox") { + const query = decodeOutboxQuery(Object.fromEntries(url.searchParams)) + if (Result.isFailure(query)) return text(`invalid outbox query: ${query.failure.message}`, 400) + const { state = "ready", limit = 100, after = 0 } = query.success + return yield* ( + state === "ready" ? eventing.listReady(limit, after) : eventing.listStaged(limit, after) + ).pipe(Effect.map(json), readFailed("outbox read")) } - } - return text("not found", 404) + return text("not found", 404) + }) } /** The `Bun.serve` fetch handler, closed over the chDB connection. Each ingest @@ -1077,8 +1118,6 @@ const makeFetch = gate: RequestQuiescenceGate, maintenanceToken: string, consumerToken: string, - controlStore: LocalEventingControlStore, - eventing: LocalEventingRuntime, ) => async (req: Request): Promise => { const url = new URL(req.url) @@ -1092,53 +1131,40 @@ const makeFetch = if (url.pathname === "/health") return respond(text("OK")) if (req.method === "POST") { if (url.pathname === "/v1/traces") - return respond( - await admitted(gate, () => ingestSpan(runSpan, db, authority, eventing, "traces", req)), - ) + return respond(await runSpan(admitted(gate, ingestSpan(db, authority, "traces", req)))) if (url.pathname === "/v1/logs") - return respond( - await admitted(gate, () => ingestSpan(runSpan, db, authority, eventing, "logs", req)), - ) + return respond(await runSpan(admitted(gate, ingestSpan(db, authority, "logs", req)))) if (url.pathname === "/v1/metrics") - return respond( - await admitted(gate, () => ingestSpan(runSpan, db, authority, eventing, "metrics", req)), - ) + return respond(await runSpan(admitted(gate, ingestSpan(db, authority, "metrics", req)))) if (url.pathname === "/local/query") - return respond(await admitted(gate, () => querySpan(runSpan, db, authority, req))) + return respond(await runSpan(admitted(gate, querySpan(db, authority, req)))) if (url.pathname === "/local/eventing/outbox/abandon") return respond( - await handleOutboxAdministration(eventing, gate, maintenanceToken, req, "abandon"), + await runSpan(handleOutboxAdministration(gate, maintenanceToken, req, "abandon")), ) if (url.pathname === "/local/eventing/consumers/accept-gap") return respond( - await handleOutboxAdministration(eventing, gate, maintenanceToken, req, "accept-gap"), + await runSpan(handleOutboxAdministration(gate, maintenanceToken, req, "accept-gap")), ) if (url.pathname === "/local/checkpoint/backup") return respond( - await handleCheckpointBackup( - db, - controlStore, - options.dataDir, - gate, - maintenanceToken, - req, - ), + await runSpan(handleCheckpointBackup(db, options.dataDir, gate, maintenanceToken, req)), ) if (url.pathname === "/local/eventing/projections") - return respond(await handleProjectionActivation(eventing, gate, maintenanceToken, req)) + return respond(await runSpan(handleProjectionActivation(gate, maintenanceToken, req))) if (url.pathname === "/local/eventing/consumers") - return respond(await handleConsumerRegistration(eventing, gate, maintenanceToken, req)) + return respond(await runSpan(handleConsumerRegistration(gate, maintenanceToken, req))) if (url.pathname === "/local/eventing/consumers/disable") - return respond(await handleConsumerDisable(eventing, gate, maintenanceToken, req)) + return respond(await runSpan(handleConsumerDisable(gate, maintenanceToken, req))) if (url.pathname === "/local/eventing/claims") - return respond(await handleConsumerClaim(eventing, gate, consumerToken, req)) + return respond(await runSpan(handleConsumerClaim(gate, consumerToken, req))) if (url.pathname === "/local/eventing/acks") - return respond(await handleConsumerAcknowledgement(eventing, gate, consumerToken, req)) + return respond(await runSpan(handleConsumerAcknowledgement(gate, consumerToken, req))) if (url.pathname === "/local/retention/retire") return respond(await handleRetirement(db, authority, gate, maintenanceToken, req)) } if (req.method === "GET" && url.pathname.startsWith("/local/eventing/")) - return respond(handleEventingRead(eventing, maintenanceToken, req, url)) + return respond(await runSpan(handleEventingRead(maintenanceToken, req, url))) if (req.method === "GET" && options.assets) return respond(serveAsset(options.assets, url.pathname)) return respond(text("not found", 404)) } @@ -1178,44 +1204,28 @@ export const startServer = ( configFile: options.configFile, rawTelemetryRetentionDays: retention.effective, }) - // The request handler and synchronous eventing store share one telemetry - // runtime; eventing observations contain only bounded operation labels. - const telemetry = yield* Effect.acquireRelease( - Effect.sync(() => ManagedRuntime.make(TelemetryLayer)), + // Request spans, eventing metrics, and the eventing services share one runtime; + // disposing it closes the control store (WAL checkpoint) and flushes telemetry. + const eventingLayer = Layer.mergeAll( + LocalEventingRuntime.layer, + LocalEventingControlStore.layer, + ).pipe(Layer.provide(Layer.succeed(LocalEventingControlConfig, { dataDir: options.dataDir }))) + const runtime = yield* Effect.acquireRelease( + Effect.sync(() => ManagedRuntime.make(Layer.mergeAll(TelemetryLayer, eventingLayer))), (rt) => Effect.promise(() => rt.dispose()), ) - const eventingTelemetry = makeEffectEventingTelemetry((effect) => { - telemetry.runFork(effect) - }) - const controlStore = yield* Effect.acquireRelease( - Effect.tryPromise({ - try: () => LocalEventingControlStore.open(options.dataDir, undefined, eventingTelemetry), - catch: (error) => + yield* runtime.contextEffect.pipe( + Effect.mapError( + (error) => new EventingStartupError({ cause: error, - message: `failed to open local eventing control store: ${describeThrown(error)}`, + message: + error._tag === "@maple/cli/eventing/ControlStoreFailed" + ? `failed to open local eventing control store: ${error.message}` + : `failed to compile local event projections: ${error.message}`, }), - }), - (store) => - Effect.try({ - try: () => store.close(), - catch: (cause) => - new EventingStartupError({ - message: "failed to close eventing control store", - cause, - }), - }).pipe( - Effect.catchTag("@maple/cli/eventing/StartupFailed", (error) => Effect.logError(error)), - ), + ), ) - const eventing = yield* Effect.try({ - try: () => new LocalEventingRuntime(controlStore, eventingTelemetry), - catch: (error) => - new EventingStartupError({ - cause: error, - message: `failed to compile local event projections: ${describeThrown(error)}`, - }), - }) // `CREATE ... IF NOT EXISTS` does not repair a table whose physical // definition was altered out of band. Inspect the opened store before the // listener is bound; a mismatch fails startup rather than allowing new @@ -1274,16 +1284,17 @@ export const startServer = ( message: `failed to load maintenance token: ${error instanceof Error ? error.message : String(error)}`, }), }) - const consumerToken = yield* Effect.tryPromise({ - try: () => ensureEventConsumerToken(options.dataDir), - catch: (error) => - new EventingStartupError({ - cause: error, - message: `failed to load event consumer token: ${describeThrown(error)}`, - }), - }) + const consumerToken = yield* ensureEventConsumerToken(options.dataDir).pipe( + Effect.mapError( + (error) => + new EventingStartupError({ + cause: error, + message: `failed to load event consumer token: ${error.message}`, + }), + ), + ) const gate = new RequestQuiescenceGate() - const runSpan: SpanRunner = (effect) => telemetry.runPromise(effect) + const runSpan: SpanRunner = (effect) => runtime.runPromise(effect) const server = yield* Effect.acquireRelease( Effect.try({ try: () => @@ -1298,8 +1309,6 @@ export const startServer = ( gate, maintenanceToken, consumerToken, - controlStore, - eventing, ), }), catch: (error) => diff --git a/apps/cli/test/checkpoints.test.ts b/apps/cli/test/checkpoints.test.ts index 49e739bd1..6ee4dd8c1 100644 --- a/apps/cli/test/checkpoints.test.ts +++ b/apps/cli/test/checkpoints.test.ts @@ -60,7 +60,8 @@ import { import { SCHEMA_FINGERPRINT } from "../src/server/schema-identity" import { storeMarkerPath, storeOpenMarkerPath } from "../src/server/store-version" import { CHDB_VERSION, MAPLE_VERSION } from "../src/version" -import { eventingControlSnapshotPath, LocalEventingControlStore } from "../src/server/eventing/control-store" +import { eventingControlSnapshotPath } from "../src/server/eventing/control-store" +import { openStore, runAsync } from "./eventing-test-support" const withDataDir = async (run: (dataDir: string) => Promise | void): Promise => { const parent = mkdtempSync(join(tmpdir(), "maple-checkpoint-test-")) @@ -356,10 +357,10 @@ describe("checkpoint state resolution", () => { mkdirSync(join(snapshot, "backup"), { recursive: true }) writeFileSync(join(snapshot, "backup", "data.bin"), "backup") - const store = await LocalEventingControlStore.open(dataDir) + const store = await openStore(dataDir) const controlPath = eventingControlSnapshotPath(dataDir, checkpointId) - const controlValidation = await store.backupTo(controlPath) - store.close() + const controlValidation = await runAsync(store.backupTo(controlPath)) + await store.close() const controlBytes = readFileSync(controlPath) writeFileSync( join(snapshot, "manifest.json"), diff --git a/apps/cli/test/eventing-test-support.ts b/apps/cli/test/eventing-test-support.ts new file mode 100644 index 000000000..5c49ae5bf --- /dev/null +++ b/apps/cli/test/eventing-test-support.ts @@ -0,0 +1,82 @@ +import { Context, Effect, Exit, Metric, Scope } from "effect" +import type { ProjectorRegistry } from "@maple/eventing-core" +import { + LocalEventingControlStore, + openControlStore, + type LocalEventingControlLimits, + type LocalEventingControlStoreApi, +} from "../src/server/eventing/control-store" +import { + LocalEventingProjectors, + LocalEventingRuntime, + type LocalEventingRuntimeApi, +} from "../src/server/eventing/runtime" + +/** Runs a synchronous store or runtime step, throwing its typed failure like the old API did. */ +export const run = (effect: Effect.Effect): A => Effect.runSync(effect) + +export const runAsync = (effect: Effect.Effect): Promise => Effect.runPromise(effect) + +export interface OpenedStore extends LocalEventingControlStoreApi { + /** Closes the store's scope: the WAL checkpoint and database close. */ + readonly close: () => Promise +} + +export const openStore = async ( + dataDir: string, + limits?: LocalEventingControlLimits, +): Promise => { + const scope = Effect.runSync(Scope.make()) + const store = await Effect.runPromise( + openControlStore(dataDir, limits).pipe( + Scope.provide(scope), + Effect.onError(() => Scope.close(scope, Exit.void)), + ), + ) + return { ...store, close: () => Effect.runPromise(Scope.close(scope, Exit.void)) } +} + +export const makeRuntime = ( + store: LocalEventingControlStoreApi, + projectors?: ProjectorRegistry, +): LocalEventingRuntimeApi => { + const make = LocalEventingRuntime.make.pipe(Effect.provideService(LocalEventingControlStore, store)) + return Effect.runSync( + projectors === undefined + ? make + : make.pipe(Effect.provideService(LocalEventingProjectors, projectors)), + ) +} + +/** An isolated metric registry; `observe` runs an effect against it. */ +export const metricRecorder = () => { + const registry = new Map>() + const context = Context.make(Metric.MetricRegistry, registry) + return { + observe: (effect: Effect.Effect): Effect.Effect => + Effect.provideService(effect, Metric.MetricRegistry, registry), + /** `operation:outcome` pairs of every eventing counter with a non-zero count. */ + operationOutcomes: (): string[] => + Metric.snapshotUnsafe(context).flatMap((snapshot) => + snapshot.id === "maple.eventing.operations_total" && + snapshot.type === "Counter" && + Number(snapshot.state.count) > 0 + ? [`${snapshot.attributes?.operation}:${snapshot.attributes?.outcome}`] + : [], + ), + attributes: (): string => + JSON.stringify(Metric.snapshotUnsafe(context).map(({ attributes }) => attributes)), + } +} + +/** Runs a serve.ts request effect against a runtime (real or a partial stub). */ +export const serveWith = ( + eventing: LocalEventingRuntimeApi, + effect: Effect.Effect, +): Promise => Effect.runPromise(Effect.provideService(effect, LocalEventingRuntime, eventing)) + +/** Runs the checkpoint backup handler against a control store (real or a stub). */ +export const serveCheckpointWith = ( + store: LocalEventingControlStoreApi, + effect: Effect.Effect, +): Promise => Effect.runPromise(Effect.provideService(effect, LocalEventingControlStore, store)) diff --git a/apps/cli/test/local-eventing-consumer-auth.test.ts b/apps/cli/test/local-eventing-consumer-auth.test.ts index 6e264bdcd..24fe47fef 100644 --- a/apps/cli/test/local-eventing-consumer-auth.test.ts +++ b/apps/cli/test/local-eventing-consumer-auth.test.ts @@ -2,6 +2,7 @@ import { strictEqual } from "node:assert" import { mkdirSync, mkdtempSync, rmSync, statSync, symlinkSync } from "node:fs" import { tmpdir } from "node:os" import { join } from "node:path" +import { Effect } from "effect" import { describe, it } from "vitest" import { ensureEventConsumerToken, @@ -15,8 +16,8 @@ describe("local event consumer authorization", () => { const dataDir = join(parent, "data") mkdirSync(dataDir) try { - const first = await ensureEventConsumerToken(dataDir) - const second = await ensureEventConsumerToken(dataDir) + const first = await Effect.runPromise(ensureEventConsumerToken(dataDir)) + const second = await Effect.runPromise(ensureEventConsumerToken(dataDir)) strictEqual(first.length, 64) strictEqual(second, first) strictEqual(statSync(eventConsumerTokenPath(dataDir)).mode & 0o777, 0o600) @@ -36,7 +37,7 @@ describe("local event consumer authorization", () => { symlinkSync(join(parent, "target"), eventConsumerTokenPath(dataDir)) let message = "" try { - await ensureEventConsumerToken(dataDir) + await Effect.runPromise(ensureEventConsumerToken(dataDir)) } catch (error) { message = error instanceof Error ? error.message : String(error) } diff --git a/apps/cli/test/local-eventing-control-store.test.ts b/apps/cli/test/local-eventing-control-store.test.ts index 359701909..3523e191d 100644 --- a/apps/cli/test/local-eventing-control-store.test.ts +++ b/apps/cli/test/local-eventing-control-store.test.ts @@ -14,8 +14,14 @@ import { tmpdir } from "node:os" import { join } from "node:path" import { describe, it } from "vitest" import type { MapleCloudEvent, SignalProjectionSpec } from "@maple/eventing-core" -import { eventingControlPath, LocalEventingControlStore } from "../src/server/eventing/control-store" -import type { EventingTelemetryObservation } from "../src/server/eventing/telemetry" +import { Effect } from "effect" +import { + eventingControlPath, + restoreControlSnapshot, + validateControlSnapshot, + writeControlSnapshot, +} from "../src/server/eventing/control-store" +import { metricRecorder, openStore, run, runAsync } from "./eventing-test-support" const withDataDir = async (run: (dataDir: string) => Promise): Promise => { const parent = mkdtempSync(join(tmpdir(), "maple-eventing-control-")) @@ -67,44 +73,45 @@ const SOURCE_FINGERPRINT = `sha256:${"a".repeat(64)}` describe("LocalEventingControlStore", () => { it("records bounded outbox and consumer telemetry without identifiers or payloads", async () => withDataDir(async (dataDir) => { - const observations: EventingTelemetryObservation[] = [] - const store = await LocalEventingControlStore.open(dataDir, undefined, { - record: (observation) => observations.push(observation), - }) + const metrics = metricRecorder() + const observed = (effect: Effect.Effect): A => run(metrics.observe(effect)) + const store = await openStore(dataDir) try { const sensitiveEvent = event({ data: { recordId: 42, label: "PAYLOAD-MUST-NOT-BE-METRIC-DATA" }, }) - store.stageEvents([sensitiveEvent, sensitiveEvent]) - throws(() => store.stageEvents([event({ data: { recordId: 43 } })]), /collision/) - throws(() => store.markReady(["unknown-event-identifier"]), /unknown event/) - store.markReady([sensitiveEvent.id]) - store.registerConsumer("tenant-a", "private-consumer-identifier", "beginning") - const claim = store.claimReady("tenant-a", "private-consumer-identifier", 10, 30) + observed(store.stageEvents([sensitiveEvent, sensitiveEvent])) + throws(() => observed(store.stageEvents([event({ data: { recordId: 43 } })])), /collision/) + throws(() => observed(store.markReady(["unknown-event-identifier"])), /unknown event/) + observed(store.markReady([sensitiveEvent.id])) + observed(store.registerConsumer("tenant-a", "private-consumer-identifier", "beginning")) + const claim = observed(store.claimReady("tenant-a", "private-consumer-identifier", 10, 30)) throws( - () => store.claimReady("tenant-a", "private-consumer-identifier", 10, 30), + () => observed(store.claimReady("tenant-a", "private-consumer-identifier", 10, 30)), /active lease/, ) throws( () => - store.acknowledgeClaim( - "tenant-a", - "private-consumer-identifier", - "incorrect-private-token", - claim.throughSequence!, + observed( + store.acknowledgeClaim( + "tenant-a", + "private-consumer-identifier", + "incorrect-private-token", + claim.throughSequence!, + ), ), /token does not match/, ) - store.acknowledgeClaim( - "tenant-a", - "private-consumer-identifier", - claim.leaseToken!, - claim.throughSequence!, + observed( + store.acknowledgeClaim( + "tenant-a", + "private-consumer-identifier", + claim.leaseToken!, + claim.throughSequence!, + ), ) - const operationOutcomes = observations.map( - ({ operation, outcome }) => `${operation}:${outcome}`, - ) + const operationOutcomes = metrics.operationOutcomes() for (const expected of [ "outbox_stage:success", "outbox_stage:failure", @@ -120,7 +127,7 @@ describe("LocalEventingControlStore", () => { ]) ok(operationOutcomes.includes(expected), `missing telemetry observation ${expected}`) - const serialized = JSON.stringify(observations) + const serialized = metrics.attributes() for (const forbidden of [ "PAYLOAD-MUST-NOT-BE-METRIC-DATA", "private-consumer-identifier", @@ -130,37 +137,39 @@ describe("LocalEventingControlStore", () => { ]) strictEqual(serialized.includes(forbidden), false) } finally { - store.close() + await store.close() } })) it("stores immutable sequential revisions and only loads the active revision", async () => withDataDir(async (dataDir) => { - const store = await LocalEventingControlStore.open(dataDir) + const store = await openStore(dataDir) try { - store.saveProjection(projection()) - deepStrictEqual(store.loadEnabledProjections("tenant-a"), [projection()]) + run(store.saveProjection(projection())) + deepStrictEqual(run(store.loadEnabledProjections("tenant-a")), [projection()]) throws( () => - store.saveProjection( - projection({ projector: { id: "changed", version: 1, config: {} } }), + run( + store.saveProjection( + projection({ projector: { id: "changed", version: 1, config: {} } }), + ), ), /immutable/, ) - throws(() => store.saveProjection(projection({ revision: 3 })), /must be 2/) + throws(() => run(store.saveProjection(projection({ revision: 3 }))), /must be 2/) - store.saveProjection(projection({ revision: 2, enabled: false })) - deepStrictEqual(store.loadEnabledProjections("tenant-a"), []) - store.saveProjection(projection({ revision: 3 })) - deepStrictEqual(store.loadEnabledProjections("tenant-a"), [projection({ revision: 3 })]) + run(store.saveProjection(projection({ revision: 2, enabled: false }))) + deepStrictEqual(run(store.loadEnabledProjections("tenant-a")), []) + run(store.saveProjection(projection({ revision: 3 }))) + deepStrictEqual(run(store.loadEnabledProjections("tenant-a")), [projection({ revision: 3 })]) throws( - () => store.saveProjection(projection({ revision: 2, enabled: false })), + () => run(store.saveProjection(projection({ revision: 2, enabled: false }))), /stale projection revision/, ) - throws(() => store.saveProjection(projection()), /stale projection revision/) - store.saveProjection(projection({ revision: 3 })) - deepStrictEqual(store.loadEnabledProjections("tenant-a"), [projection({ revision: 3 })]) - deepStrictEqual(store.validate(), { + throws(() => run(store.saveProjection(projection())), /stale projection revision/) + run(store.saveProjection(projection({ revision: 3 }))) + deepStrictEqual(run(store.loadEnabledProjections("tenant-a")), [projection({ revision: 3 })]) + deepStrictEqual(run(store.validate), { schemaVersion: 1, projectionRevisions: 3, projectionFailures: 0, @@ -168,93 +177,99 @@ describe("LocalEventingControlStore", () => { readyEvents: 0, }) } finally { - store.close() + await store.close() } })) it("deduplicates staged events, rejects collisions, and preserves ready order", async () => withDataDir(async (dataDir) => { - const store = await LocalEventingControlStore.open(dataDir) + const store = await openStore(dataDir) try { - deepStrictEqual(store.stageEvents([event(), event()]), { + deepStrictEqual(run(store.stageEvents([event(), event()])), { inserted: 1, deduplicated: 1, dropped: 0, eventIds: [event().id, event().id], }) - throws(() => store.stageEvents([event({ data: { recordId: 43 } })]), /collision/) - throws(() => store.markReady(["unknown"]), /unknown event/) - store.markReady([event().id]) - store.markReady([event().id]) - deepStrictEqual(store.listStaged().events, []) + throws(() => run(store.stageEvents([event({ data: { recordId: 43 } })])), /collision/) + throws(() => run(store.markReady(["unknown"])), /unknown event/) + run(store.markReady([event().id])) + run(store.markReady([event().id])) + deepStrictEqual(run(store.listStaged()).events, []) deepStrictEqual( - store.listReady().events.map(({ event }) => event), + run(store.listReady()).events.map(({ event }) => event), [event()], ) } finally { - store.close() + await store.close() } })) it("binds staged source recovery to the normalized occurrence fingerprint", async () => withDataDir(async (dataDir) => { - const store = await LocalEventingControlStore.open(dataDir) + const store = await openStore(dataDir) try { - store.saveProjection(projection()) + run(store.saveProjection(projection())) const sourced = event({ sourceoccurrenceid: "record-42" }) - throws(() => store.stageEvents([sourced]), /requires a source fingerprint/) - store.stageEvents([sourced], new Map([[sourced.id, SOURCE_FINGERPRINT]])) + throws(() => run(store.stageEvents([sourced])), /requires a source fingerprint/) + run(store.stageEvents([sourced], new Map([[sourced.id, SOURCE_FINGERPRINT]]))) deepStrictEqual( - store.stagedEventIdsForOccurrence( - sourced.tenantid, - "otel.log", - sourced.source, - sourced.sourceoccurrenceid!, - SOURCE_FINGERPRINT, - ), - [sourced.id], - ) - throws( - () => + run( store.stagedEventIdsForOccurrence( sourced.tenantid, "otel.log", sourced.source, sourced.sourceoccurrenceid!, - `sha256:${"b".repeat(64)}`, + SOURCE_FINGERPRINT, + ), + ), + [sourced.id], + ) + throws( + () => + run( + store.stagedEventIdsForOccurrence( + sourced.tenantid, + "otel.log", + sourced.source, + sourced.sourceoccurrenceid!, + `sha256:${"b".repeat(64)}`, + ), ), /staged source occurrence collision/, ) - strictEqual(store.listStaged().events.length, 1) + strictEqual(run(store.listStaged()).events.length, 1) } finally { - store.close() + await store.close() } })) it("survives restart and round-trips through a validated standalone snapshot", async () => withDataDir(async (dataDir) => { - let store = await LocalEventingControlStore.open(dataDir) - store.saveProjection(projection()) - store.stageEvents([event()]) - store.markReady([event().id]) - store.recordProjectionFailures("tenant-a", [ - { - projectionId: "example-record-observed", - projectionRevision: 1, - occurrenceId: "record-42", - message: "test failure", - }, - ]) - store.close() + let store = await openStore(dataDir) + run(store.saveProjection(projection())) + run(store.stageEvents([event()])) + run(store.markReady([event().id])) + run( + store.recordProjectionFailures("tenant-a", [ + { + projectionId: "example-record-observed", + projectionRevision: 1, + occurrenceId: "record-42", + message: "test failure", + }, + ]), + ) + await store.close() - store = await LocalEventingControlStore.open(dataDir) - deepStrictEqual(store.loadEnabledProjections("tenant-a"), [projection()]) + store = await openStore(dataDir) + deepStrictEqual(run(store.loadEnabledProjections("tenant-a")), [projection()]) deepStrictEqual( - store.listReady().events.map(({ event }) => event), + run(store.listReady()).events.map(({ event }) => event), [event()], ) const snapshot = join(dataDir, "backups", "snapshot", "control.sqlite") - const validation = await store.backupTo(snapshot) + const validation = await runAsync(store.backupTo(snapshot)) deepStrictEqual(validation, { schemaVersion: 1, projectionRevisions: 1, @@ -262,168 +277,167 @@ describe("LocalEventingControlStore", () => { stagedEvents: 0, readyEvents: 1, }) - store.close() + await store.close() const restored = join(dataDir, "restored") - await LocalEventingControlStore.restoreSnapshot(snapshot, restored) - deepStrictEqual( - LocalEventingControlStore.validateSnapshot(eventingControlPath(restored)), - validation, - ) - const restoredStore = await LocalEventingControlStore.open(restored) + await runAsync(restoreControlSnapshot(snapshot, restored)) + deepStrictEqual(run(validateControlSnapshot(eventingControlPath(restored))), validation) + const restoredStore = await openStore(restored) try { - deepStrictEqual(restoredStore.loadEnabledProjections("tenant-a"), [projection()]) + deepStrictEqual(run(restoredStore.loadEnabledProjections("tenant-a")), [projection()]) deepStrictEqual( - restoredStore.listReady().events.map(({ event }) => event), + run(restoredStore.listReady()).events.map(({ event }) => event), [event()], ) } finally { - restoredStore.close() + await restoredStore.close() } })) it("writes the captured SQLite state even when the live store changes before archive I/O", async () => withDataDir(async (dataDir) => { - const store = await LocalEventingControlStore.open(dataDir) + const store = await openStore(dataDir) try { - store.saveProjection(projection()) - store.stageEvents([event()]) - const bytes = store.captureSnapshot() - store.markReady([event().id]) - store.saveProjection(projection({ revision: 2, enabled: false })) + run(store.saveProjection(projection())) + run(store.stageEvents([event()])) + const bytes = run(store.captureSnapshot) + run(store.markReady([event().id])) + run(store.saveProjection(projection({ revision: 2, enabled: false }))) const snapshot = join(dataDir, "backups", "captured", "control.sqlite") - const validation = await LocalEventingControlStore.writeSnapshot(snapshot, bytes) + const validation = await runAsync(writeControlSnapshot(snapshot, bytes)) strictEqual(validation.stagedEvents, 1) strictEqual(validation.readyEvents, 0) strictEqual(validation.projectionRevisions, 1) - strictEqual(store.validate().readyEvents, 1) - strictEqual(store.validate().projectionRevisions, 2) + strictEqual(run(store.validate).readyEvents, 1) + strictEqual(run(store.validate).projectionRevisions, 2) const restored = join(dataDir, "restored-capture") - await LocalEventingControlStore.restoreSnapshot(snapshot, restored) - const recovered = await LocalEventingControlStore.open(restored) + await runAsync(restoreControlSnapshot(snapshot, restored)) + const recovered = await openStore(restored) try { - deepStrictEqual(recovered.loadEnabledProjections("tenant-a"), [projection()]) + deepStrictEqual(run(recovered.loadEnabledProjections("tenant-a")), [projection()]) deepStrictEqual( - recovered.listStaged().events.map((row) => row.event), + run(recovered.listStaged()).events.map((row) => row.event), [event()], ) } finally { - recovered.close() + await recovered.close() } } finally { - store.close() + await store.close() } })) it("checkpoints committed live WAL state before serializing", async () => withDataDir(async (dataDir) => { - const store = await LocalEventingControlStore.open(dataDir) + const store = await openStore(dataDir) try { - store.saveProjection(projection()) - store.stageEvents([event()]) - store.markReady([event().id]) + run(store.saveProjection(projection())) + run(store.stageEvents([event()])) + run(store.markReady([event().id])) const walPath = `${eventingControlPath(dataDir)}-wal` ok(existsSync(walPath)) ok(statSync(walPath).size > 0, "test requires uncheckpointed WAL frames") const snapshot = join(dataDir, "backups", "live-wal", "control.sqlite") - await store.backupTo(snapshot) + await runAsync(store.backupTo(snapshot)) strictEqual(statSync(walPath).size, 0) const restored = join(dataDir, "restored-live-wal") - await LocalEventingControlStore.restoreSnapshot(snapshot, restored) - const restoredStore = await LocalEventingControlStore.open(restored) + await runAsync(restoreControlSnapshot(snapshot, restored)) + const restoredStore = await openStore(restored) try { - deepStrictEqual(restoredStore.loadEnabledProjections("tenant-a"), [projection()]) + deepStrictEqual(run(restoredStore.loadEnabledProjections("tenant-a")), [projection()]) deepStrictEqual( - restoredStore.listReady().events.map(({ event }) => event), + run(restoredStore.listReady()).events.map(({ event }) => event), [event()], ) } finally { - restoredStore.close() + await restoredStore.close() } } finally { - store.close() + await store.close() } })) it("paginates every ready event and reports bounded outbox overflow", async () => withDataDir(async (dataDir) => { - const store = await LocalEventingControlStore.open(dataDir, { + const store = await openStore(dataDir, { maxOutboxEvents: 2, maxOutboxBytes: 1024 * 1024, }) try { const second = event({ id: "event-2", data: { recordId: 43, label: "Second" } }) const third = event({ id: "event-3", data: { recordId: 44, label: "Third" } }) - const staged = store.stageEvents([event(), second]) - store.markReady(staged.eventIds) + const staged = run(store.stageEvents([event(), second])) + run(store.markReady(staged.eventIds)) - const firstPage = store.listReady(1) + const firstPage = run(store.listReady(1)) strictEqual(firstPage.events.length, 1) strictEqual(firstPage.nextCursor, firstPage.events[0]?.sequence) - const secondPage = store.listReady(1, firstPage.nextCursor!) + const secondPage = run(store.listReady(1, firstPage.nextCursor!)) deepStrictEqual( [...firstPage.events, ...secondPage.events].map(({ event }) => event.id), [event().id, second.id], ) strictEqual(secondPage.nextCursor, null) - deepStrictEqual(store.stageEvents([event()]).deduplicated, 1) - deepStrictEqual(store.stageEvents([third]), { + deepStrictEqual(run(store.stageEvents([event()])).deduplicated, 1) + deepStrictEqual(run(store.stageEvents([third])), { inserted: 0, deduplicated: 0, dropped: 1, eventIds: [], }) - strictEqual(store.deliveryGap("tenant-a").generation, 1) + strictEqual(run(store.deliveryGap("tenant-a")).generation, 1) } finally { - store.close() + await store.close() } })) it("pages recovered events by first readiness transition instead of staging order", async () => withDataDir(async (dataDir) => { - const store = await LocalEventingControlStore.open(dataDir) + const store = await openStore(dataDir) try { const first = event({ id: "event-a" }) const second = event({ id: "event-b" }) - store.stageEvents([first]) - store.stageEvents([second]) - store.markReady([second.id]) + run(store.stageEvents([first])) + run(store.stageEvents([second])) + run(store.markReady([second.id])) - const initialPage = store.listReady(1) + const initialPage = run(store.listReady(1)) deepStrictEqual( initialPage.events.map(({ event }) => event.id), [second.id], ) const cursor = initialPage.events[0]!.sequence - store.markReady([first.id]) - const recoveredPage = store.listReady(1, cursor) + run(store.markReady([first.id])) + const recoveredPage = run(store.listReady(1, cursor)) deepStrictEqual( recoveredPage.events.map(({ event }) => event.id), [first.id], ) strictEqual(recoveredPage.events[0]!.sequence > cursor, true) } finally { - store.close() + await store.close() } })) it("rejects invalid schema-1 staged fingerprints during snapshot validation", async () => withDataDir(async (dataDir) => { - const store = await LocalEventingControlStore.open(dataDir) - store.saveProjection(projection()) + const store = await openStore(dataDir) + run(store.saveProjection(projection())) const missing = event({ id: "event-missing-fingerprint", sourceoccurrenceid: "record-1" }) const malformed = event({ id: "event-malformed-fingerprint", sourceoccurrenceid: "record-2" }) - store.stageEvents( - [missing, malformed], - new Map([ - [missing.id, SOURCE_FINGERPRINT], - [malformed.id, SOURCE_FINGERPRINT], - ]), + run( + store.stageEvents( + [missing, malformed], + new Map([ + [missing.id, SOURCE_FINGERPRINT], + [malformed.id, SOURCE_FINGERPRINT], + ]), + ), ) - store.close() + await store.close() const database = new Database(eventingControlPath(dataDir), { readwrite: true, @@ -440,15 +454,15 @@ describe("LocalEventingControlStore", () => { database.close(true) throws( - () => LocalEventingControlStore.validateSnapshot(eventingControlPath(dataDir)), + () => run(validateControlSnapshot(eventingControlPath(dataDir))), /invalid staged source fingerprint/, ) - await rejects(() => LocalEventingControlStore.open(dataDir), /invalid staged source fingerprint/) + await rejects(() => openStore(dataDir), /invalid staged source fingerprint/) })) it("leases whole batches, redelivers after expiry, and rejects stale acknowledgements", async () => withDataDir(async (dataDir) => { - const store = await LocalEventingControlStore.open(dataDir, { + const store = await openStore(dataDir, { maxOutboxEvents: 10, maxOutboxBytes: 1024 * 1024, retainAcknowledgedReadyEvents: 0, @@ -456,16 +470,12 @@ describe("LocalEventingControlStore", () => { try { const second = event({ id: "event-2" }) const third = event({ id: "event-3" }) - const staged = store.stageEvents([event(), second, third]) - store.markReady(staged.eventIds) - store.registerConsumer("tenant-a", "automation", "beginning", "2026-08-13T12:00:00.000Z") + const staged = run(store.stageEvents([event(), second, third])) + run(store.markReady(staged.eventIds)) + run(store.registerConsumer("tenant-a", "automation", "beginning", "2026-08-13T12:00:00.000Z")) - const firstClaim = store.claimReady( - "tenant-a", - "automation", - 2, - 10, - "2026-08-13T12:00:01.000Z", + const firstClaim = run( + store.claimReady("tenant-a", "automation", 2, 10, "2026-08-13T12:00:01.000Z"), ) strictEqual(firstClaim.leaseToken?.length, 64) deepStrictEqual( @@ -473,45 +483,53 @@ describe("LocalEventingControlStore", () => { [event().id, second.id], ) throws( - () => store.claimReady("tenant-a", "automation", 2, 10, "2026-08-13T12:00:02.000Z"), + () => run(store.claimReady("tenant-a", "automation", 2, 10, "2026-08-13T12:00:02.000Z")), /active lease/, ) throws( () => - store.acknowledgeClaim( - "tenant-a", - "automation", - "0".repeat(64), - firstClaim.throughSequence!, - "2026-08-13T12:00:03.000Z", + run( + store.acknowledgeClaim( + "tenant-a", + "automation", + "0".repeat(64), + firstClaim.throughSequence!, + "2026-08-13T12:00:03.000Z", + ), ), /token does not match/, ) throws( () => - store.acknowledgeClaim( - "tenant-a", - "automation", - firstClaim.leaseToken!, - firstClaim.events[0]!.sequence, - "2026-08-13T12:00:03.000Z", + run( + store.acknowledgeClaim( + "tenant-a", + "automation", + firstClaim.leaseToken!, + firstClaim.events[0]!.sequence, + "2026-08-13T12:00:03.000Z", + ), ), /complete claimed batch/, ) - const retry = store.claimReady("tenant-a", "automation", 2, 10, "2026-08-13T12:00:12.000Z") + const retry = run( + store.claimReady("tenant-a", "automation", 2, 10, "2026-08-13T12:00:12.000Z"), + ) deepStrictEqual( retry.events.map(({ event }) => event.id), [event().id, second.id], ) strictEqual(retry.leaseToken === firstClaim.leaseToken, false) deepStrictEqual( - store.acknowledgeClaim( - "tenant-a", - "automation", - retry.leaseToken!, - retry.throughSequence!, - "2026-08-13T12:00:13.000Z", + run( + store.acknowledgeClaim( + "tenant-a", + "automation", + retry.leaseToken!, + retry.throughSequence!, + "2026-08-13T12:00:13.000Z", + ), ), { consumerId: "automation", @@ -520,28 +538,30 @@ describe("LocalEventingControlStore", () => { }, ) deepStrictEqual( - store.listReady().events.map(({ event }) => event.id), + run(store.listReady()).events.map(({ event }) => event.id), [third.id], ) throws( () => - store.acknowledgeClaim( - "tenant-a", - "automation", - firstClaim.leaseToken!, - firstClaim.throughSequence!, - "2026-08-13T12:00:14.000Z", + run( + store.acknowledgeClaim( + "tenant-a", + "automation", + firstClaim.leaseToken!, + firstClaim.throughSequence!, + "2026-08-13T12:00:14.000Z", + ), ), /no active lease/, ) } finally { - store.close() + await store.close() } })) it("prunes only after every active consumer advances and never prunes staged events", async () => withDataDir(async (dataDir) => { - const store = await LocalEventingControlStore.open(dataDir, { + const store = await openStore(dataDir, { maxOutboxEvents: 10, maxOutboxBytes: 1024 * 1024, retainAcknowledgedReadyEvents: 0, @@ -550,107 +570,110 @@ describe("LocalEventingControlStore", () => { const second = event({ id: "event-2" }) const third = event({ id: "event-3" }) const stranded = event({ id: "event-staged" }) - const ready = store.stageEvents([event(), second, third]) - store.markReady(ready.eventIds) - store.stageEvents([stranded]) - store.registerConsumer("tenant-a", "automation-a", "beginning") - store.registerConsumer("tenant-a", "automation-b", "beginning") + const ready = run(store.stageEvents([event(), second, third])) + run(store.markReady(ready.eventIds)) + run(store.stageEvents([stranded])) + run(store.registerConsumer("tenant-a", "automation-a", "beginning")) + run(store.registerConsumer("tenant-a", "automation-b", "beginning")) - const fast = store.claimReady("tenant-a", "automation-a", 3, 30) + const fast = run(store.claimReady("tenant-a", "automation-a", 3, 30)) strictEqual( - store.acknowledgeClaim( - "tenant-a", - "automation-a", - fast.leaseToken!, - fast.throughSequence!, + run( + store.acknowledgeClaim( + "tenant-a", + "automation-a", + fast.leaseToken!, + fast.throughSequence!, + ), ).prunedEvents, 0, ) - const slow = store.claimReady("tenant-a", "automation-b", 2, 30) + const slow = run(store.claimReady("tenant-a", "automation-b", 2, 30)) strictEqual( - store.acknowledgeClaim( - "tenant-a", - "automation-b", - slow.leaseToken!, - slow.throughSequence!, + run( + store.acknowledgeClaim( + "tenant-a", + "automation-b", + slow.leaseToken!, + slow.throughSequence!, + ), ).prunedEvents, 2, ) deepStrictEqual( - store.listReady().events.map(({ event }) => event.id), + run(store.listReady()).events.map(({ event }) => event.id), [third.id], ) - store.disableConsumer("tenant-a", "automation-b") - deepStrictEqual(store.listReady().events, []) + run(store.disableConsumer("tenant-a", "automation-b")) + deepStrictEqual(run(store.listReady()).events, []) deepStrictEqual( - store.listStaged().events.map(({ event }) => event.id), + run(store.listStaged()).events.map(({ event }) => event.id), [stranded.id], ) } finally { - store.close() + await store.close() } })) it("starts latest consumers after backlog and checkpoints active leases", async () => withDataDir(async (dataDir) => { - let store = await LocalEventingControlStore.open(dataDir) - store.stageEvents([event()]) - store.markReady([event().id]) - const registered = store.registerConsumer( - "tenant-a", - "automation", - "latest", - "2099-01-01T00:00:00.000Z", + let store = await openStore(dataDir) + run(store.stageEvents([event()])) + run(store.markReady([event().id])) + const registered = run( + store.registerConsumer("tenant-a", "automation", "latest", "2099-01-01T00:00:00.000Z"), ) - strictEqual(registered.lastAcknowledgedSequence, store.listReady().events[0]!.sequence) + strictEqual(registered.lastAcknowledgedSequence, run(store.listReady()).events[0]!.sequence) deepStrictEqual( - store.claimReady("tenant-a", "automation", 10, 300, "2099-01-01T00:00:01.000Z").events, + run(store.claimReady("tenant-a", "automation", 10, 300, "2099-01-01T00:00:01.000Z")).events, [], ) const second = event({ id: "event-2" }) - store.stageEvents([second]) - store.markReady([second.id]) - const claim = store.claimReady("tenant-a", "automation", 10, 300, "2099-01-01T00:00:02.000Z") + run(store.stageEvents([second])) + run(store.markReady([second.id])) + const claim = run(store.claimReady("tenant-a", "automation", 10, 300, "2099-01-01T00:00:02.000Z")) const snapshot = join(dataDir, "backups", "consumer", "control.sqlite") - await store.backupTo(snapshot) - store.close() + await runAsync(store.backupTo(snapshot)) + await store.close() const restored = join(dataDir, "restored-consumer") - await LocalEventingControlStore.restoreSnapshot(snapshot, restored) - store = await LocalEventingControlStore.open(restored) + await runAsync(restoreControlSnapshot(snapshot, restored)) + store = await openStore(restored) try { deepStrictEqual( - store.listConsumers("tenant-a")[0]?.claimedThroughSequence, + run(store.listConsumers("tenant-a"))[0]?.claimedThroughSequence, claim.throughSequence, ) strictEqual( - store.acknowledgeClaim( - "tenant-a", - "automation", - claim.leaseToken!, - claim.throughSequence!, - "2099-01-01T00:00:03.000Z", + run( + store.acknowledgeClaim( + "tenant-a", + "automation", + claim.leaseToken!, + claim.throughSequence!, + "2099-01-01T00:00:03.000Z", + ), ).acknowledgedThrough, claim.throughSequence, ) } finally { - store.close() + await store.close() } })) it("rejects a corrupted durable outbox counter at reopen", async () => withDataDir(async (dataDir) => { - const store = await LocalEventingControlStore.open(dataDir) - store.stageEvents([event()]) - store.close() + const store = await openStore(dataDir) + run(store.stageEvents([event()])) + await store.close() const db = new Database(eventingControlPath(dataDir)) try { db.run("UPDATE outbox_usage SET bytes = bytes + 1 WHERE singleton = 1") } finally { db.close() } - await rejects(() => LocalEventingControlStore.open(dataDir), /accounting is inconsistent/) + await rejects(() => openStore(dataDir), /accounting is inconsistent/) })) it("refuses a symlink in place of the database", async () => @@ -658,7 +681,7 @@ describe("LocalEventingControlStore", () => { const controlPath = eventingControlPath(dataDir) mkdirSync(join(dataDir, "control"), { recursive: true }) symlinkSync(join(dataDir, "target.sqlite"), controlPath) - await rejects(() => LocalEventingControlStore.open(dataDir), /not a real file/) + await rejects(() => openStore(dataDir), /not a real file/) strictEqual(controlPath.endsWith("control/eventing.sqlite"), true) })) }) diff --git a/apps/cli/test/local-eventing-ingest.test.ts b/apps/cli/test/local-eventing-ingest.test.ts index a1fef70cf..df782c22c 100644 --- a/apps/cli/test/local-eventing-ingest.test.ts +++ b/apps/cli/test/local-eventing-ingest.test.ts @@ -1,36 +1,45 @@ import { deepStrictEqual, ok, rejects, strictEqual } from "node:assert" +import { Effect } from "effect" import { describe, it } from "vitest" +import { EventingControlStoreError } from "../src/server/eventing/control-store" import { normalizeOtlpLogs } from "../src/server/eventing/otlp" -import { ProjectionActivationConflict, SourceOccurrenceCollision } from "../src/server/eventing/runtime" +import { + type LocalEventingRuntimeApi, + ProjectionActivationConflict, + SourceOccurrenceCollision, +} from "../src/server/eventing/runtime" import { __testables } from "../src/server/serve" +import { run, serveCheckpointWith, serveWith } from "./eventing-test-support" describe("Local eventing ingest seam", () => { it("requires maintenance authorization and exposes staged records only when requested", async () => { const eventing = { - health: () => ({ activeProjections: 1 }), - listActive: () => [], - listReady: () => ({ events: [{ sequence: 1, event: { id: "ready" } }], nextCursor: null }), - listStaged: (_limit: number, after: number) => ({ - events: [{ sequence: after + 1, event: { id: "staged" } }], - nextCursor: null, - }), + health: Effect.succeed({ activeProjections: 1 }), + listActive: Effect.succeed([]), + listReady: () => + Effect.succeed({ events: [{ sequence: 1, event: { id: "ready" } }], nextCursor: null }), + listStaged: (_limit: number, after: number) => + Effect.succeed({ + events: [{ sequence: after + 1, event: { id: "staged" } }], + nextCursor: null, + }), } - const unauthorized = __testables.handleEventingRead( + const unauthorized = await serveWith( eventing as never, - "maintenance-secret", - new Request("http://127.0.0.1/local/eventing/outbox?state=staged"), - new URL("http://127.0.0.1/local/eventing/outbox?state=staged"), + __testables.handleEventingRead( + "maintenance-secret", + new Request("http://127.0.0.1/local/eventing/outbox?state=staged"), + new URL("http://127.0.0.1/local/eventing/outbox?state=staged"), + ), ) strictEqual(unauthorized.status, 403) const request = new Request("http://127.0.0.1/local/eventing/outbox?state=staged&after=41", { headers: { "x-maple-maintenance-token": "maintenance-secret" }, }) - const authorized = __testables.handleEventingRead( + const authorized = await serveWith( eventing as never, - "maintenance-secret", - request, - new URL(request.url), + __testables.handleEventingRead("maintenance-secret", request, new URL(request.url)), ) strictEqual(authorized.status, 200) deepStrictEqual(await authorized.json(), { @@ -42,27 +51,24 @@ describe("Local eventing ingest seam", () => { it("authenticates and reads activation bodies before closing admission", async () => { const gate = new __testables.RequestQuiescenceGate() const neverClosed = new ReadableStream() - const unauthorized = await __testables.handleProjectionActivation( + const unauthorized = await serveWith( {} as never, - gate, - "maintenance-secret", - { + __testables.handleProjectionActivation(gate, "maintenance-secret", { headers: new Headers(), body: neverClosed, - } as Request, + } as Request), ) strictEqual(unauthorized.status, 403) const afterUnauthorized = gate.enter() ok(afterUnauthorized, "invalid authorization must not close admission") afterUnauthorized() - const checkpointUnauthorized = await __testables.handleCheckpointBackup( - {} as never, + const checkpointUnauthorized = await serveCheckpointWith( {} as never, - "/unused", - gate, - "maintenance-secret", - { headers: new Headers(), body: neverClosed } as Request, + __testables.handleCheckpointBackup({} as never, "/unused", gate, "maintenance-secret", { + headers: new Headers(), + body: neverClosed, + } as Request), ) strictEqual(checkpointUnauthorized.status, 403) const afterCheckpointUnauthorized = gate.enter() @@ -76,20 +82,19 @@ describe("Local eventing ingest seam", () => { }, }) let committed = false - const pending = __testables.handleProjectionActivation( + const pending = serveWith( { - prepareActivation: (body: unknown) => ({ body }), - commitActivation: () => { - committed = true - }, - listActive: () => [], + prepareActivation: (body: unknown) => Effect.succeed({ body }), + commitActivation: () => + Effect.sync(() => { + committed = true + }), + listActive: Effect.succeed([]), } as never, - gate, - "maintenance-secret", - { + __testables.handleProjectionActivation(gate, "maintenance-secret", { headers: new Headers({ "x-maple-maintenance-token": "maintenance-secret" }), body: slowBody, - } as Request, + } as Request), ) await Promise.resolve() const whileReading = gate.enter() @@ -106,7 +111,7 @@ describe("Local eventing ingest seam", () => { method: "POST", body: "123456789", }) - await rejects(() => __testables.readBoundedJson(oversized, 8), /exceeds 8 bytes/) + await rejects(() => Effect.runPromise(__testables.readBoundedJson(oversized, 8)), /exceeds 8 bytes/) const gate = new __testables.RequestQuiescenceGate() let releaseMaintenance!: () => void @@ -117,22 +122,24 @@ describe("Local eventing ingest seam", () => { }), ) await Promise.resolve() - const response = await __testables.handleProjectionActivation( + const response = await serveWith( { - prepareActivation: () => ({}), - commitActivation: () => undefined, - listActive: () => [], + prepareActivation: () => Effect.succeed({}), + commitActivation: () => Effect.void, + listActive: Effect.succeed([]), } as never, - gate, - "maintenance-secret", - new Request("http://127.0.0.1/local/eventing/projections", { - method: "POST", - headers: { - "content-type": "application/json", - "x-maple-maintenance-token": "maintenance-secret", - }, - body: "{}", - }), + __testables.handleProjectionActivation( + gate, + "maintenance-secret", + new Request("http://127.0.0.1/local/eventing/projections", { + method: "POST", + headers: { + "content-type": "application/json", + "x-maple-maintenance-token": "maintenance-secret", + }, + body: "{}", + }), + ), ) strictEqual(response.status, 409) releaseMaintenance() @@ -143,91 +150,103 @@ describe("Local eventing ingest seam", () => { const gate = new __testables.RequestQuiescenceGate() const calls: string[] = [] const eventing = { - registerConsumer: (consumerId: string, startAt: string) => { - calls.push(`register:${consumerId}:${startAt}`) - return { consumerId, active: true } - }, - disableConsumer: (consumerId: string) => { - calls.push(`disable:${consumerId}`) - return { consumerId, active: false } - }, - claimReady: (consumerId: string, limit: number, leaseSeconds: number) => { - calls.push(`claim:${consumerId}:${limit}:${leaseSeconds}`) - return { - consumerId, - leaseToken: "a".repeat(64), - throughSequence: 7, - events: [{ sequence: 7, event: { id: "event-7" } }], - } - }, - acknowledgeClaim: (consumerId: string, _leaseToken: string, throughSequence: number) => { - calls.push(`ack:${consumerId}:${throughSequence}`) - return { consumerId, acknowledgedThrough: throughSequence, prunedEvents: 0 } - }, + registerConsumer: (consumerId: string, startAt: string) => + Effect.sync(() => { + calls.push(`register:${consumerId}:${startAt}`) + return { consumerId, active: true } + }), + disableConsumer: (consumerId: string) => + Effect.sync(() => { + calls.push(`disable:${consumerId}`) + return { consumerId, active: false } + }), + claimReady: (consumerId: string, limit: number, leaseSeconds: number) => + Effect.sync(() => { + calls.push(`claim:${consumerId}:${limit}:${leaseSeconds}`) + return { + consumerId, + leaseToken: "a".repeat(64), + throughSequence: 7, + events: [{ sequence: 7, event: { id: "event-7" } }], + } + }), + acknowledgeClaim: (consumerId: string, _leaseToken: string, throughSequence: number) => + Effect.sync(() => { + calls.push(`ack:${consumerId}:${throughSequence}`) + return { consumerId, acknowledgedThrough: throughSequence, prunedEvents: 0 } + }), } - const registration = await __testables.handleConsumerRegistration( + const registration = await serveWith( eventing as never, - gate, - "maintenance-secret", - new Request("http://127.0.0.1/local/eventing/consumers", { - method: "POST", - headers: { - "content-type": "application/json", - "x-maple-maintenance-token": "maintenance-secret", - }, - body: JSON.stringify({ consumerId: "automation", startAt: "beginning" }), - }), + __testables.handleConsumerRegistration( + gate, + "maintenance-secret", + new Request("http://127.0.0.1/local/eventing/consumers", { + method: "POST", + headers: { + "content-type": "application/json", + "x-maple-maintenance-token": "maintenance-secret", + }, + body: JSON.stringify({ consumerId: "automation", startAt: "beginning" }), + }), + ), ) strictEqual(registration.status, 201) - const wrongClaimCredential = await __testables.handleConsumerClaim( + const wrongClaimCredential = await serveWith( eventing as never, - gate, - "consumer-secret", - new Request("http://127.0.0.1/local/eventing/claims", { - method: "POST", - headers: { - "content-type": "application/json", - "x-maple-maintenance-token": "maintenance-secret", - }, - body: JSON.stringify({ consumerId: "automation", limit: 10, leaseSeconds: 30 }), - }), + __testables.handleConsumerClaim( + gate, + "consumer-secret", + new Request("http://127.0.0.1/local/eventing/claims", { + method: "POST", + headers: { + "content-type": "application/json", + "x-maple-maintenance-token": "maintenance-secret", + }, + body: JSON.stringify({ consumerId: "automation", limit: 10, leaseSeconds: 30 }), + }), + ), ) strictEqual(wrongClaimCredential.status, 403) - const claim = await __testables.handleConsumerClaim( + const claim = await serveWith( eventing as never, - gate, - "consumer-secret", - new Request("http://127.0.0.1/local/eventing/claims", { - method: "POST", - headers: { - "content-type": "application/json", - "x-maple-event-consumer-token": "consumer-secret", - }, - body: JSON.stringify({ consumerId: "automation", limit: 10, leaseSeconds: 30 }), - }), + __testables.handleConsumerClaim( + gate, + "consumer-secret", + new Request("http://127.0.0.1/local/eventing/claims", { + method: "POST", + headers: { + "content-type": "application/json", + "x-maple-event-consumer-token": "consumer-secret", + }, + body: JSON.stringify({ consumerId: "automation", limit: 10, leaseSeconds: 30 }), + }), + ), ) strictEqual(claim.status, 200) const claimed = (await claim.json()) as { leaseToken: string; throughSequence: number } - const acknowledgement = await __testables.handleConsumerAcknowledgement( + const acknowledgement = await serveWith( eventing as never, - gate, - "consumer-secret", - new Request("http://127.0.0.1/local/eventing/acks", { - method: "POST", - headers: { - "content-type": "application/json", - "x-maple-event-consumer-token": "consumer-secret", - }, - body: JSON.stringify({ - consumerId: "automation", - leaseToken: claimed.leaseToken, - throughSequence: claimed.throughSequence, + __testables.handleConsumerAcknowledgement( + gate, + "consumer-secret", + new Request("http://127.0.0.1/local/eventing/acks", { + method: "POST", + headers: { + "content-type": "application/json", + "x-maple-event-consumer-token": "consumer-secret", + }, + body: JSON.stringify({ + consumerId: "automation", + leaseToken: claimed.leaseToken, + throughSequence: claimed.throughSequence, + }), }), - }), + ), ) strictEqual(acknowledgement.status, 200) deepStrictEqual(calls, [ @@ -244,18 +263,20 @@ describe("Local eventing ingest seam", () => { }), ) await Promise.resolve() - const blockedClaim = await __testables.handleConsumerClaim( + const blockedClaim = await serveWith( eventing as never, - gate, - "consumer-secret", - new Request("http://127.0.0.1/local/eventing/claims", { - method: "POST", - headers: { - "content-type": "application/json", - "x-maple-event-consumer-token": "consumer-secret", - }, - body: JSON.stringify({ consumerId: "automation", limit: 10, leaseSeconds: 30 }), - }), + __testables.handleConsumerClaim( + gate, + "consumer-secret", + new Request("http://127.0.0.1/local/eventing/claims", { + method: "POST", + headers: { + "content-type": "application/json", + "x-maple-event-consumer-token": "consumer-secret", + }, + body: JSON.stringify({ consumerId: "automation", limit: 10, leaseSeconds: 30 }), + }), + ), ) strictEqual(blockedClaim.status, 503) releaseMaintenance() @@ -263,19 +284,19 @@ describe("Local eventing ingest seam", () => { }) it("rejects fractional consumer limits before calling the store", async () => { - const response = await __testables.handleConsumerClaim( + const response = await serveWith( { - claimReady: () => { - throw new Error("invalid input reached the store") - }, + claimReady: () => Effect.die(new Error("invalid input reached the store")), }, - new __testables.RequestQuiescenceGate(), - "secret", - new Request("http://127.0.0.1/local/eventing/claims", { - method: "POST", - headers: { "x-maple-event-consumer-token": "secret" }, - body: JSON.stringify({ consumerId: "automation", limit: 1.5, leaseSeconds: 30 }), - }), + __testables.handleConsumerClaim( + new __testables.RequestQuiescenceGate(), + "secret", + new Request("http://127.0.0.1/local/eventing/claims", { + method: "POST", + headers: { "x-maple-event-consumer-token": "secret" }, + body: JSON.stringify({ consumerId: "automation", limit: 1.5, leaseSeconds: 30 }), + }), + ), ) strictEqual(response.status, 400) }) @@ -296,28 +317,30 @@ describe("Local eventing ingest seam", () => { }, } const eventing = { - evaluateOtlp: () => { - order.push("evaluate") - return { - events: [event], - recoveredEventIds: [], - failures: [ - { - projectionId: "oversized-projector", - projectionRevision: 1, - occurrenceId: "occurrence-1", - message: "CloudEvent exceeds 262144 UTF-8 bytes", - }, - ], - typeMismatchFields: [], - } - }, - persistFailures: () => order.push("persist-failures"), - stage: () => { - order.push("stage") - return { inserted: 1, deduplicated: 0, dropped: 0, eventIds: [event.id] } - }, - markReady: () => order.push("ready"), + evaluateOtlp: () => + Effect.sync(() => { + order.push("evaluate") + return { + events: [event], + recoveredEventIds: [], + failures: [ + { + projectionId: "oversized-projector", + projectionRevision: 1, + occurrenceId: "occurrence-1", + message: "CloudEvent exceeds 262144 UTF-8 bytes", + }, + ], + typeMismatchFields: [], + } + }), + persistFailures: () => Effect.sync(() => void order.push("persist-failures")), + stage: () => + Effect.sync(() => { + order.push("stage") + return { inserted: 1, deduplicated: 0, dropped: 0, eventIds: [event.id] } + }), + markReady: () => Effect.sync(() => void order.push("ready")), } const request = new Request("http://127.0.0.1/v1/logs", { method: "POST", @@ -340,12 +363,9 @@ describe("Local eventing ingest seam", () => { }), }) - const result = await __testables.ingest( - db as never, - authority as never, + const result = await serveWith( eventing as never, - "logs", - request, + __testables.ingest(db as never, authority as never, "logs", request), ) strictEqual(result.response.status, 200) strictEqual(result.accepted, 1) @@ -368,35 +388,40 @@ describe("Local eventing ingest seam", () => { resourceLogs: [{ scopeLogs: [{ logRecords: [{ body: { stringValue: "one" } }] }] }], }), }) - const result = await __testables.ingest( - { - exec: () => { - throw new Error("write failed") - }, - } as never, + const result = await serveWith( { - isRetired: () => false, - filterBatch: (_datasource: string, ndjson: string) => ({ - ndjson, - accepted: 1, - rejected: 0, - }), - } as never, - { - evaluateOtlp: () => ({ - events: [{ id: "event-1" }], - recoveredEventIds: [], - failures: [], - typeMismatchFields: [], - }), - persistFailures: () => undefined, - stage: () => ({ inserted: 1, deduplicated: 0, dropped: 0, eventIds: ["event-1"] }), - markReady: () => { - markedReady = true - }, + evaluateOtlp: () => + Effect.succeed({ + events: [{ id: "event-1" }], + recoveredEventIds: [], + failures: [], + typeMismatchFields: [], + }), + persistFailures: () => Effect.void, + stage: () => + Effect.succeed({ inserted: 1, deduplicated: 0, dropped: 0, eventIds: ["event-1"] }), + markReady: () => + Effect.sync(() => { + markedReady = true + }), } as never, - "logs", - request, + __testables.ingest( + { + exec: () => { + throw new Error("write failed") + }, + } as never, + { + isRetired: () => false, + filterBatch: (_datasource: string, ndjson: string) => ({ + ndjson, + accepted: 1, + rejected: 0, + }), + } as never, + "logs", + request, + ), ) strictEqual(result.response.status, 500) strictEqual(markedReady, false) @@ -421,31 +446,35 @@ describe("Local eventing ingest seam", () => { ], }), }) - const result = await __testables.ingest( - { exec: () => undefined } as never, + const result = await serveWith( { - isRetired: () => false, - filterBatch: (_datasource: string, ndjson: string) => ({ - ndjson, - accepted: 1, - rejected: 0, - }), - } as never, - { - evaluateOtlp: () => ({ - events: [], - recoveredEventIds: ["revision-1-event"], - failures: [], - typeMismatchFields: [], - }), - persistFailures: () => undefined, - stage: () => ({ inserted: 0, deduplicated: 0, dropped: 0, eventIds: [] }), - markReady: (eventIds: readonly string[]) => { - readyIds = eventIds - }, + evaluateOtlp: () => + Effect.succeed({ + events: [], + recoveredEventIds: ["revision-1-event"], + failures: [], + typeMismatchFields: [], + }), + persistFailures: () => Effect.void, + stage: () => Effect.succeed({ inserted: 0, deduplicated: 0, dropped: 0, eventIds: [] }), + markReady: (eventIds: readonly string[]) => + Effect.sync(() => { + readyIds = eventIds + }), } as never, - "logs", - request, + __testables.ingest( + { exec: () => undefined } as never, + { + isRetired: () => false, + filterBatch: (_datasource: string, ndjson: string) => ({ + ndjson, + accepted: 1, + rejected: 0, + }), + } as never, + "logs", + request, + ), ) strictEqual(result.response.status, 200) deepStrictEqual(readyIds, ["revision-1-event"]) @@ -520,41 +549,44 @@ describe("Local eventing ingest seam", () => { }, ], } - const result = await __testables.ingest( - { exec: () => (inserted = true) } as never, - { - isRetired: () => false, - filterBatch: (_datasource: string, ndjson: string) => ({ - ndjson, - accepted: ndjson.trim().split("\n").length, - rejected: 0, - }), - } as never, + const result = await serveWith( { evaluateOtlp: (_signal: string, decoded: unknown) => { - const projected = normalizeOtlpLogs(decoded).filter( + const projected = run(normalizeOtlpLogs(decoded)).filter( (signal) => signal.fields.get("signal:event.name")?.value === "project.me", ) - return { + return Effect.succeed({ events: projected.map((_signal, index) => ({ id: `event-${index + 1}` })), recoveredEventIds: [], failures: [], typeMismatchFields: [], - } - }, - persistFailures: () => undefined, - stage: (events: readonly { readonly id: string }[]) => { - stagedIds = events.map(({ id }) => id) - return { inserted: events.length, deduplicated: 0, dropped: 0, eventIds: stagedIds } + }) }, - markReady: () => undefined, + persistFailures: () => Effect.void, + stage: (events: readonly { readonly id: string }[]) => + Effect.sync(() => { + stagedIds = events.map(({ id }) => id) + return { inserted: events.length, deduplicated: 0, dropped: 0, eventIds: stagedIds } + }), + markReady: () => Effect.void, } as never, - "logs", - new Request("http://127.0.0.1/v1/logs", { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify(body), - }), + __testables.ingest( + { exec: () => (inserted = true) } as never, + { + isRetired: () => false, + filterBatch: (_datasource: string, ndjson: string) => ({ + ndjson, + accepted: ndjson.trim().split("\n").length, + rejected: 0, + }), + } as never, + "logs", + new Request("http://127.0.0.1/v1/logs", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }), + ), ) strictEqual(result.response.status, 200) strictEqual(result.accepted, 5) @@ -563,29 +595,35 @@ describe("Local eventing ingest seam", () => { }) it("refuses an in-batch source collision with 400 so exporters do not resend it", async () => { - const ingestWith = (failure: unknown) => - __testables.ingest( - { exec: () => undefined } as never, - { isRetired: () => false } as never, + const ingestWith = (failure: Effect.Effect) => + serveWith( { - evaluateOtlp: () => { - throw failure - }, + evaluateOtlp: () => failure, } as never, - "logs", - new Request("http://127.0.0.1/v1/logs", { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ resourceLogs: [] }), - }), + __testables.ingest( + { exec: () => undefined } as never, + { isRetired: () => false } as never, + "logs", + new Request("http://127.0.0.1/v1/logs", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ resourceLogs: [] }), + }), + ), ) const collision = await ingestWith( - new SourceOccurrenceCollision({ message: "source occurrence collision", occurrenceId: "a" }), + Effect.fail( + new SourceOccurrenceCollision({ message: "source occurrence collision", occurrenceId: "a" }), + ), ) strictEqual(collision.response.status, 400) strictEqual(collision.accepted, 0) - const transient = await ingestWith(new Error("control store unavailable")) + const transient = await ingestWith( + Effect.fail(new EventingControlStoreError({ message: "control store unavailable" })), + ) strictEqual(transient.response.status, 503) + const unexpected = await ingestWith(Effect.die(new Error("projection crashed"))) + strictEqual(unexpected.response.status, 503) }) it("reports a concurrent activation as a retryable 409", async () => { @@ -594,51 +632,49 @@ describe("Local eventing ingest seam", () => { headers: { "x-maple-maintenance-token": "maintenance-secret" }, body: JSON.stringify({}), }) - const response = await __testables.handleProjectionActivation( + const response = await serveWith( { - prepareActivation: () => ({}), - commitActivation: () => { - throw new ProjectionActivationConflict({ message: "projection registry changed" }) - }, + prepareActivation: () => Effect.succeed({}), + commitActivation: () => + Effect.fail(new ProjectionActivationConflict({ message: "projection registry changed" })), } as never, - new __testables.RequestQuiescenceGate(), - "maintenance-secret", - request, + __testables.handleProjectionActivation( + new __testables.RequestQuiescenceGate(), + "maintenance-secret", + request, + ), ) strictEqual(response.status, 409) }) it("validates outbox query parameters at the boundary and keeps store failures a 500", async () => { - const read = (query: string, eventing: Pick) => { + const read = async (query: string, eventing: Pick) => { const request = new Request(`http://127.0.0.1/local/eventing/outbox${query}`, { headers: { "x-maple-maintenance-token": "maintenance-secret" }, }) - return __testables.handleEventingRead( + return serveWith( eventing as never, - "maintenance-secret", - request, - new URL(request.url), + __testables.handleEventingRead("maintenance-secret", request, new URL(request.url)), ) } const listed: Array = [] const eventing = { - listReady: (limit: number, after: number) => { - listed.push([limit, after]) - return { events: [], nextCursor: null } - }, + listReady: (limit?: number, after?: number) => + Effect.sync(() => { + listed.push([limit ?? 100, after ?? 0]) + return { events: [], nextCursor: null } + }), } - strictEqual(read("?limit=0", eventing).status, 400) - strictEqual(read("?limit=1.5", eventing).status, 400) - strictEqual(read("?after=-1", eventing).status, 400) - strictEqual(read("?state=acked", eventing).status, 400) - strictEqual(read("?cursor=1", eventing).status, 400) - strictEqual(read("?limit=25&after=7", eventing).status, 200) + strictEqual((await read("?limit=0", eventing)).status, 400) + strictEqual((await read("?limit=1.5", eventing)).status, 400) + strictEqual((await read("?after=-1", eventing)).status, 400) + strictEqual((await read("?state=acked", eventing)).status, 400) + strictEqual((await read("?cursor=1", eventing)).status, 400) + strictEqual((await read("?limit=25&after=7", eventing)).status, 200) deepStrictEqual(listed, [[25, 7]]) const failing = { - listReady: () => { - throw new Error("database is locked") - }, + listReady: () => Effect.fail(new EventingControlStoreError({ message: "database is locked" })), } - strictEqual(read("", failing).status, 500) + strictEqual((await read("", failing)).status, 500) }) }) diff --git a/apps/cli/test/local-eventing-overflow.test.ts b/apps/cli/test/local-eventing-overflow.test.ts index 088ba3406..694c114de 100644 --- a/apps/cli/test/local-eventing-overflow.test.ts +++ b/apps/cli/test/local-eventing-overflow.test.ts @@ -6,16 +6,15 @@ import { tmpdir } from "node:os" import { join } from "node:path" import { describe, it } from "vitest" import { RetiredDayAuthority } from "../src/server/archives/retention" -import { LocalEventingControlStore } from "../src/server/eventing/control-store" -import { LocalEventingRuntime } from "../src/server/eventing/runtime" import { __testables } from "../src/server/serve" +import { makeRuntime, openStore, run, serveWith } from "./eventing-test-support" describe("Durable outbox overflow", () => { it("keeps warehouse ingestion available and requires explicit gap recovery across reopen", async () => { const parent = mkdtempSync(join(tmpdir(), "maple-overflow-")) const dataDir = join(parent, "data") mkdirSync(dataDir) - const store = await LocalEventingControlStore.open(dataDir, { + const store = await openStore(dataDir, { maxOutboxEvents: 1, maxOutboxBytes: 1024 * 1024, }) @@ -32,17 +31,22 @@ describe("Durable outbox overflow", () => { project: () => ({ data: { observed: true } }), }), ) - const runtime = new LocalEventingRuntime(store, undefined, projectors) - runtime.activate({ - id: "observed", - revision: 1, - enabled: true, - tenantId: "local", - sourceKind: "otel.log", - selector: { op: "exists", field: { namespace: "signal", key: "event.name", type: "string" } }, - projector: { id: "example.observed", version: 1, config: {} }, - activeFrom: "1970-01-01T00:00:00Z", - }) + const runtime = makeRuntime(store, projectors) + run( + runtime.activate({ + id: "observed", + revision: 1, + enabled: true, + tenantId: "local", + sourceKind: "otel.log", + selector: { + op: "exists", + field: { namespace: "signal", key: "event.name", type: "string" }, + }, + projector: { id: "example.observed", version: 1, config: {} }, + activeFrom: "1970-01-01T00:00:00Z", + }), + ) const statements: string[] = [] const warehouse = { exec: (sql: string) => { @@ -51,34 +55,36 @@ describe("Durable outbox overflow", () => { } const authority = new RetiredDayAuthority(dataDir) const ingest = (id: string) => - __testables.ingest( - warehouse, - authority, + serveWith( runtime, - "logs", - new Request("http://localhost/v1/logs", { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ - resourceLogs: [ - { - scopeLogs: [ - { - logRecords: [ - { - eventName: "example.observed", - timeUnixNano: "1786131720123456789", - attributes: [ - { key: "event.id", value: { stringValue: id } }, - ], - }, - ], - }, - ], - }, - ], + __testables.ingest( + warehouse, + authority, + "logs", + new Request("http://localhost/v1/logs", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + resourceLogs: [ + { + scopeLogs: [ + { + logRecords: [ + { + eventName: "example.observed", + timeUnixNano: "1786131720123456789", + attributes: [ + { key: "event.id", value: { stringValue: id } }, + ], + }, + ], + }, + ], + }, + ], + }), }), - }), + ), ) const first = await ingest("record-1") strictEqual(first.accepted, 1) @@ -88,69 +94,73 @@ describe("Durable outbox overflow", () => { strictEqual(second.response.status, 200) strictEqual(second.response.headers.get("x-maple-eventing-dropped"), "1") strictEqual(statements.length, 2) - const retained = store.listReady(10).events + const retained = run(store.listReady(10)).events strictEqual(retained.length, 1) const eventId = retained[0]?.event.id if (eventId === undefined) throw new Error("missing retained event") - store.registerConsumer("local", "consumer", "beginning") - throws(() => store.claimReady("local", "consumer", 10, 60), /delivery has a gap/) - throws(() => store.acceptDeliveryGap("local", "consumer", 2), /generation changed/) - store.acceptDeliveryGap("local", "consumer", 1) - strictEqual(store.claimReady("local", "consumer", 10, 60).events.length, 1) - throws(() => store.abandonEvents("other-tenant", [eventId]), /unknown event ID/) - throws(() => store.abandonEvents("local", [eventId, "missing"]), /unknown event ID/) - strictEqual(store.listReady(10).events.length, 1) - strictEqual(store.deliveryGap("local").generation, 1) - const unauthorized = await __testables.handleOutboxAdministration( + run(store.registerConsumer("local", "consumer", "beginning")) + throws(() => run(store.claimReady("local", "consumer", 10, 60)), /delivery has a gap/) + throws(() => run(store.acceptDeliveryGap("local", "consumer", 2)), /generation changed/) + run(store.acceptDeliveryGap("local", "consumer", 1)) + strictEqual(run(store.claimReady("local", "consumer", 10, 60)).events.length, 1) + throws(() => run(store.abandonEvents("other-tenant", [eventId])), /unknown event ID/) + throws(() => run(store.abandonEvents("local", [eventId, "missing"])), /unknown event ID/) + strictEqual(run(store.listReady(10)).events.length, 1) + strictEqual(run(store.deliveryGap("local")).generation, 1) + const unauthorized = await serveWith( runtime, - new __testables.RequestQuiescenceGate(), - "secret", - new Request("http://localhost/local/eventing/outbox/abandon", { - method: "POST", - body: JSON.stringify({ eventIds: [eventId] }), - }), - "abandon", + __testables.handleOutboxAdministration( + new __testables.RequestQuiescenceGate(), + "secret", + new Request("http://localhost/local/eventing/outbox/abandon", { + method: "POST", + body: JSON.stringify({ eventIds: [eventId] }), + }), + "abandon", + ), ) strictEqual(unauthorized.status, 403) - strictEqual(store.listReady(10).events.length, 1) + strictEqual(run(store.listReady(10)).events.length, 1) const gate = new __testables.RequestQuiescenceGate() const release = gate.enter() if (release === null) throw new Error("gate unexpectedly closed") - const pending = __testables.handleOutboxAdministration( + const pending = serveWith( runtime, - gate, - "secret", - new Request("http://localhost/local/eventing/outbox/abandon", { - method: "POST", - headers: { "x-maple-maintenance-token": "secret" }, - body: JSON.stringify({ eventIds: [eventId] }), - }), - "abandon", + __testables.handleOutboxAdministration( + gate, + "secret", + new Request("http://localhost/local/eventing/outbox/abandon", { + method: "POST", + headers: { "x-maple-maintenance-token": "secret" }, + body: JSON.stringify({ eventIds: [eventId] }), + }), + "abandon", + ), ) await Promise.resolve() - strictEqual(store.listReady(10).events.length, 1) + strictEqual(run(store.listReady(10)).events.length, 1) release() strictEqual((await pending).status, 200) - strictEqual(store.deliveryGap("local").generation, 2) - throws(() => store.claimReady("local", "consumer", 10, 60), /delivery has a gap/) - store.acceptDeliveryGap("local", "consumer", 2) + strictEqual(run(store.deliveryGap("local")).generation, 2) + throws(() => run(store.claimReady("local", "consumer", 10, 60)), /delivery has a gap/) + run(store.acceptDeliveryGap("local", "consumer", 2)) // Abandonment cleared the old lease and transactional counters free capacity. const third = await ingest("record-3") strictEqual(third.accepted, 1) strictEqual(third.response.headers.get("x-maple-eventing-dropped"), null) - strictEqual(store.claimReady("local", "consumer", 10, 60).events.length, 1) - store.validate() + strictEqual(run(store.claimReady("local", "consumer", 10, 60)).events.length, 1) + run(store.validate) } finally { - store.close() + await store.close() } try { - const reopened = await LocalEventingControlStore.open(dataDir) + const reopened = await openStore(dataDir) try { - strictEqual(reopened.deliveryGap("local").generation, 2) - strictEqual(reopened.deliveryGap("local").droppedEvents, 2) - strictEqual(reopened.listReady(10).events.length, 1) + strictEqual(run(reopened.deliveryGap("local")).generation, 2) + strictEqual(run(reopened.deliveryGap("local")).droppedEvents, 2) + strictEqual(run(reopened.listReady(10)).events.length, 1) } finally { - reopened.close() + await reopened.close() } } finally { rmSync(parent, { recursive: true, force: true }) diff --git a/apps/cli/test/local-eventing-runtime.test.ts b/apps/cli/test/local-eventing-runtime.test.ts index f4b0a454c..3d447f99d 100644 --- a/apps/cli/test/local-eventing-runtime.test.ts +++ b/apps/cli/test/local-eventing-runtime.test.ts @@ -1,4 +1,4 @@ -import { Result } from "effect" +import { Effect, Result } from "effect" import { deepStrictEqual, ok, strictEqual, throws } from "node:assert" import { mkdirSync, mkdtempSync, rmSync } from "node:fs" import { tmpdir } from "node:os" @@ -13,11 +13,10 @@ import { type SignalProjectionSpec, type SignalScalar, } from "@maple/eventing-core" -import { LocalEventingControlStore } from "../src/server/eventing/control-store" import { normalizeOtlpLogs, normalizeOtlpLogsWithDiagnostics } from "../src/server/eventing/otlp" -import { LocalEventingRuntime, sourceOccurrenceFingerprint } from "../src/server/eventing/runtime" -import type { EventingTelemetryObservation } from "../src/server/eventing/telemetry" +import { sourceOccurrenceFingerprint } from "../src/server/eventing/runtime" import { encodeLogs } from "../src/server/otlp/encode" +import { makeRuntime, metricRecorder, openStore, run } from "./eventing-test-support" const withDataDir = async (run: (dataDir: string) => Promise): Promise => { const parent = mkdtempSync(join(tmpdir(), "maple-eventing-runtime-")) @@ -192,22 +191,24 @@ describe("OTLP eventing input validation", () => { it("rejects non-string attribute keys before normalization", () => { throws( () => - normalizeOtlpLogs({ - resourceLogs: [ - { - scopeLogs: [ - { - logRecords: [ - { - timeUnixNano: "1786125600000000000", - attributes: [{ key: 123, value: { stringValue: "bad" } }], - }, - ], - }, - ], - }, - ], - }), + run( + normalizeOtlpLogs({ + resourceLogs: [ + { + scopeLogs: [ + { + logRecords: [ + { + timeUnixNano: "1786125600000000000", + attributes: [{ key: 123, value: { stringValue: "bad" } }], + }, + ], + }, + ], + }, + ], + }), + ), /invalid OTLP logs/, ) }) @@ -216,27 +217,25 @@ describe("OTLP eventing input validation", () => { describe("LocalEventingRuntime", () => { it("records bounded normalization and projection outcomes without signal data", async () => withDataDir(async (dataDir) => { - const observations: EventingTelemetryObservation[] = [] - const telemetry = { - record: (observation: EventingTelemetryObservation) => observations.push(observation), - } - const store = await LocalEventingControlStore.open(dataDir) + const metrics = metricRecorder() + const observed = (effect: Effect.Effect): A => run(metrics.observe(effect)) + const store = await openStore(dataDir) try { - const runtime = new LocalEventingRuntime(store, telemetry, exampleProjectors()) - runtime.activate(projection()) - strictEqual(runtime.evaluateOtlp("logs", exampleRecordObserved).events.length, 1) + const runtime = makeRuntime(store, exampleProjectors()) + run(runtime.activate(projection())) + strictEqual(observed(runtime.evaluateOtlp("logs", exampleRecordObserved)).events.length, 1) const malformed = structuredClone(exampleRecordObserved) firstLogRecord(malformed).attributes = firstLogRecord(malformed).attributes.filter( ({ key }) => key !== "example.collection.name", ) - strictEqual(runtime.evaluateOtlp("logs", malformed).failures.length, 1) + strictEqual(observed(runtime.evaluateOtlp("logs", malformed)).failures.length, 1) const mismatched = structuredClone(exampleRecordObserved) firstLogRecord(mismatched).attributes = firstLogRecord(mismatched).attributes.map((entry) => entry.key === "example.record.sequence" ? attr(entry.key, { stringValue: "42" }) : entry, ) - deepStrictEqual(runtime.evaluateOtlp("logs", mismatched).typeMismatchFields, [ + deepStrictEqual(observed(runtime.evaluateOtlp("logs", mismatched)).typeMismatchFields, [ "attribute:example.record.sequence", ]) @@ -246,28 +245,26 @@ describe("LocalEventingRuntime", () => { attr(`projection-only-${index}`, { stringValue: "warehouse-valid" }), ), ) - strictEqual(runtime.evaluateOtlp("logs", projectionBoundFailure).events.length, 0) + strictEqual(observed(runtime.evaluateOtlp("logs", projectionBoundFailure)).events.length, 0) - const operationOutcomes = observations.map( - ({ operation, outcome }) => `${operation}:${outcome}`, - ) + const operationOutcomes = metrics.operationOutcomes() ok(operationOutcomes.includes("normalization:success")) ok(operationOutcomes.includes("normalization:failure")) ok(operationOutcomes.includes("projection:success")) ok(operationOutcomes.includes("projection:failure")) ok(operationOutcomes.includes("selector_type_mismatch:observed")) - const serialized = JSON.stringify(observations) + const serialized = metrics.attributes() strictEqual(serialized.includes("Observe example events"), false) strictEqual(serialized.includes("01K20EXAMPLERECORD42"), false) strictEqual(serialized.includes("example-record-observed"), false) strictEqual(serialized.includes("example.record.sequence"), false) } finally { - store.close() + await store.close() } })) it("normalizes typed generic OTLP fields while preserving the existing warehouse encoding", () => { - const [signal] = normalizeOtlpLogs(exampleRecordObserved, "2026-08-07T20:00:00Z") + const [signal] = run(normalizeOtlpLogs(exampleRecordObserved, "2026-08-07T20:00:00Z")) strictEqual(signal?.occurrenceId, "01K20EXAMPLERECORD42") strictEqual(signal?.identityQuality, "source") strictEqual(signal?.source, "https://events.example.test") @@ -289,7 +286,7 @@ describe("LocalEventingRuntime", () => { attr("cloudevents.id", { stringValue: " cloud-event-42 " }), ...aliasedRecord.attributes.filter(({ key }) => !["event.id", "cloudevents.id"].includes(key)), ] - const [aliasedSignal] = normalizeOtlpLogs(aliased, "2026-08-07T20:00:00Z") + const [aliasedSignal] = run(normalizeOtlpLogs(aliased, "2026-08-07T20:00:00Z")) strictEqual(aliasedSignal?.occurrenceId, "cloud-event-42") strictEqual(aliasedSignal?.identityQuality, "source") @@ -302,8 +299,8 @@ describe("LocalEventingRuntime", () => { ) const derivedB = structuredClone(derivedA) firstLogRecord(derivedB).body = { stringValue: "A different record occurrence" } - const [signalA] = normalizeOtlpLogs(derivedA, "2026-08-07T20:00:00Z") - const [signalB] = normalizeOtlpLogs(derivedB, "2026-08-07T20:00:00Z") + const [signalA] = run(normalizeOtlpLogs(derivedA, "2026-08-07T20:00:00Z")) + const [signalB] = run(normalizeOtlpLogs(derivedB, "2026-08-07T20:00:00Z")) strictEqual(signalA?.identityQuality, "derived") strictEqual(signalB?.identityQuality, "derived") strictEqual(signalA?.occurrenceId?.startsWith("derived:sha256:"), true) @@ -311,8 +308,8 @@ describe("LocalEventingRuntime", () => { }) it("keeps projectable retries byte-identical and skips timestamp-less durable logs", () => { - const first = normalizeOtlpLogs(exampleRecordObserved, "2026-08-07T20:00:00Z") - const retry = normalizeOtlpLogs(exampleRecordObserved, "2026-08-08T20:00:00Z") + const first = run(normalizeOtlpLogs(exampleRecordObserved, "2026-08-07T20:00:00Z")) + const retry = run(normalizeOtlpLogs(exampleRecordObserved, "2026-08-08T20:00:00Z")) deepStrictEqual(first, retry) const timestampLess = structuredClone(exampleRecordObserved) @@ -322,9 +319,10 @@ describe("LocalEventingRuntime", () => { } delete timestampLessRecord.timeUnixNano delete timestampLessRecord.observedTimeUnixNano - deepStrictEqual(normalizeOtlpLogs(timestampLess, "2026-08-07T20:00:00Z"), []) + deepStrictEqual(run(normalizeOtlpLogs(timestampLess, "2026-08-07T20:00:00Z")), []) deepStrictEqual( - normalizeOtlpLogsWithDiagnostics(timestampLess, "2026-08-07T20:00:00Z").unprojectedIdentities, + run(normalizeOtlpLogsWithDiagnostics(timestampLess, "2026-08-07T20:00:00Z")) + .unprojectedIdentities, [ { sourceKind: "otel.log", @@ -338,15 +336,18 @@ describe("LocalEventingRuntime", () => { }) it("uses a locale-independent source-fingerprint field order", () => { - const [signal] = normalizeOtlpLogs(exampleRecordObserved, "2026-08-07T20:00:00Z") + const [signal] = run(normalizeOtlpLogs(exampleRecordObserved, "2026-08-07T20:00:00Z")) const fields = new Map(signal!.fields) fields.set("attribute:ä", { type: "string", value: "umlaut" }) fields.set("attribute:z", { type: "string", value: "ascii" }) const forward = { ...signal!, fields } const reverse = { ...signal!, fields: new Map([...fields].reverse()) } - strictEqual(sourceOccurrenceFingerprint(forward), sourceOccurrenceFingerprint(reverse)) strictEqual( - sourceOccurrenceFingerprint(forward), + Result.getOrThrow(sourceOccurrenceFingerprint(forward)), + Result.getOrThrow(sourceOccurrenceFingerprint(reverse)), + ) + strictEqual( + Result.getOrThrow(sourceOccurrenceFingerprint(forward)), "sha256:4ed4d210645f2df1959e5c56acb5b22140a01aa267fdf1fab8b62e56ea63e31e", ) }) @@ -361,7 +362,7 @@ describe("LocalEventingRuntime", () => { kvlistValue: { values: [attr("__proto__", { stringValue: "nested" })] }, }), ) - const [signal] = normalizeOtlpLogs(request, "2026-08-07T20:00:00Z") + const [signal] = run(normalizeOtlpLogs(request, "2026-08-07T20:00:00Z")) const record = (signal!.data as { record: { attributes: Record } }).record ok(Object.prototype.hasOwnProperty.call(record.attributes, "__proto__")) // Attribute maps are null-prototype on purpose, so a `__proto__` key stays data. @@ -377,43 +378,47 @@ describe("LocalEventingRuntime", () => { it("catalogs only the scalar body field that the OTLP adapter can populate", async () => withDataDir(async (dataDir) => { - const store = await LocalEventingControlStore.open(dataDir) + const store = await openStore(dataDir) try { - const runtime = new LocalEventingRuntime(store, undefined, exampleProjectors()) + const runtime = makeRuntime(store, exampleProjectors()) throws( () => - runtime.prepareActivation( - projection({ - selector: { - op: "exists", - field: { namespace: "body", key: "text", type: "string" }, - }, - }), + run( + runtime.prepareActivation( + projection({ + selector: { + op: "exists", + field: { namespace: "body", key: "text", type: "string" }, + }, + }), + ), ), /unknown field body:text/, ) - const activation = runtime.prepareActivation( - projection({ - selector: { - op: "exists", - field: { namespace: "body", key: "value", type: "boolean" }, - }, - }), + const activation = run( + runtime.prepareActivation( + projection({ + selector: { + op: "exists", + field: { namespace: "body", key: "value", type: "boolean" }, + }, + }), + ), ) strictEqual(activation.spec.selector.op, "exists") } finally { - store.close() + await store.close() } })) it("projects before storage, deduplicates retry delivery, and makes the event ready after commit", async () => withDataDir(async (dataDir) => { - const store = await LocalEventingControlStore.open(dataDir) + const store = await openStore(dataDir) try { - const runtime = new LocalEventingRuntime(store, undefined, exampleProjectors()) - strictEqual(runtime.hasActiveSource("otel.log"), false) - runtime.activate(projection()) - const first = runtime.evaluateOtlp("logs", exampleRecordObserved) + const runtime = makeRuntime(store, exampleProjectors()) + strictEqual(run(runtime.hasActiveSource("otel.log")), false) + run(runtime.activate(projection())) + const first = run(runtime.evaluateOtlp("logs", exampleRecordObserved)) strictEqual(first.failures.length, 0) strictEqual(first.events.length, 1) deepStrictEqual(first.events[0], { @@ -444,14 +449,14 @@ describe("LocalEventingRuntime", () => { serviceName: "example-service", }, }) - const staged = runtime.stage(first.events, first.eventSourceFingerprints) + const staged = run(runtime.stage(first.events, first.eventSourceFingerprints)) strictEqual(staged.inserted, 1) - strictEqual(runtime.listReady().events.length, 0) + strictEqual(run(runtime.listReady()).events.length, 0) deepStrictEqual( - runtime.listStaged().events.map(({ event }) => event), + run(runtime.listStaged()).events.map(({ event }) => event), first.events, ) - runtime.activate(projection({ revision: 2, enabled: false })) + run(runtime.activate(projection({ revision: 2, enabled: false }))) const projectionIneligibleRetry = structuredClone(exampleRecordObserved) firstLogRecord(projectionIneligibleRetry).attributes.push( ...Array.from({ length: 257 }, (_, index) => @@ -459,39 +464,39 @@ describe("LocalEventingRuntime", () => { ), ) throws( - () => runtime.evaluateOtlp("logs", projectionIneligibleRetry, () => true), + () => run(runtime.evaluateOtlp("logs", projectionIneligibleRetry, () => true)), /cannot safely recover staged source occurrence/, ) - strictEqual(runtime.listStaged().events.length, 1) - strictEqual(runtime.listReady().events.length, 0) + strictEqual(run(runtime.listStaged()).events.length, 1) + strictEqual(run(runtime.listReady()).events.length, 0) const changedRetry = structuredClone(exampleRecordObserved) firstLogRecord(changedRetry).body = { stringValue: "changed retry content" } throws( - () => runtime.evaluateOtlp("logs", changedRetry, () => true), + () => run(runtime.evaluateOtlp("logs", changedRetry, () => true)), /staged source occurrence collision/, ) - strictEqual(runtime.listStaged().events.length, 1) - strictEqual(runtime.listReady().events.length, 0) - const retry = runtime.evaluateOtlp("logs", exampleRecordObserved, () => true) + strictEqual(run(runtime.listStaged()).events.length, 1) + strictEqual(run(runtime.listReady()).events.length, 0) + const retry = run(runtime.evaluateOtlp("logs", exampleRecordObserved, () => true)) deepStrictEqual(retry.events, []) deepStrictEqual(retry.recoveredEventIds, staged.eventIds) - runtime.markReady(retry.recoveredEventIds) + run(runtime.markReady(retry.recoveredEventIds)) deepStrictEqual( - runtime.listReady().events.map(({ event }) => event), + run(runtime.listReady()).events.map(({ event }) => event), first.events, ) - deepStrictEqual(runtime.listStaged().events, []) + deepStrictEqual(run(runtime.listStaged()).events, []) } finally { - store.close() + await store.close() } })) it("rejects same event bytes with conflicting source content within one batch", async () => withDataDir(async (dataDir) => { - const store = await LocalEventingControlStore.open(dataDir) + const store = await openStore(dataDir) try { - const runtime = new LocalEventingRuntime(store, undefined, exampleProjectors()) - runtime.activate(projection()) + const runtime = makeRuntime(store, exampleProjectors()) + run(runtime.activate(projection())) const request = structuredClone(exampleRecordObserved) const first = firstLogRecord(request) first.attributes.push(attr("example.projector.ignored", { stringValue: "first" })) @@ -503,43 +508,43 @@ describe("LocalEventingRuntime", () => { ) request.resourceLogs[0]!.scopeLogs[0]!.logRecords.push(second) throws( - () => runtime.evaluateOtlp("logs", request), + () => run(runtime.evaluateOtlp("logs", request)), /source occurrence collision within one ingest batch/, ) - strictEqual(runtime.listStaged().events.length, 0) - strictEqual(runtime.listReady().events.length, 0) + strictEqual(run(runtime.listStaged()).events.length, 0) + strictEqual(run(runtime.listReady()).events.length, 0) } finally { - store.close() + await store.close() } })) it("rejects matching and nonmatching records that reuse one source occurrence", async () => withDataDir(async (dataDir) => { - const store = await LocalEventingControlStore.open(dataDir) + const store = await openStore(dataDir) try { - const runtime = new LocalEventingRuntime(store, undefined, exampleProjectors()) - runtime.activate(eventNameProjection("observed-only", "example.record.observed")) + const runtime = makeRuntime(store, exampleProjectors()) + run(runtime.activate(eventNameProjection("observed-only", "example.record.observed"))) const request = structuredClone(exampleRecordObserved) const sibling = structuredClone(firstLogRecord(request)) sibling.eventName = "example.record.ignored" request.resourceLogs[0]!.scopeLogs[0]!.logRecords.push(sibling) throws( - () => runtime.evaluateOtlp("logs", request), + () => run(runtime.evaluateOtlp("logs", request)), /source occurrence collision within one ingest batch/, ) - strictEqual(runtime.listStaged().events.length, 0) - strictEqual(runtime.listReady().events.length, 0) + strictEqual(run(runtime.listStaged()).events.length, 0) + strictEqual(run(runtime.listReady()).events.length, 0) } finally { - store.close() + await store.close() } })) it("rejects projectable and projection-ineligible records with one source occurrence", async () => withDataDir(async (dataDir) => { - const store = await LocalEventingControlStore.open(dataDir) + const store = await openStore(dataDir) try { - const runtime = new LocalEventingRuntime(store, undefined, exampleProjectors()) - runtime.activate(projection()) + const runtime = makeRuntime(store, exampleProjectors()) + run(runtime.activate(projection())) const request = structuredClone(exampleRecordObserved) const sibling = structuredClone(firstLogRecord(request)) sibling.attributes.push( @@ -549,73 +554,75 @@ describe("LocalEventingRuntime", () => { ) request.resourceLogs[0]!.scopeLogs[0]!.logRecords.push(sibling) throws( - () => runtime.evaluateOtlp("logs", request), + () => run(runtime.evaluateOtlp("logs", request)), /source occurrence collision with an unprojectable record within one ingest batch/, ) - strictEqual(runtime.listStaged().events.length, 0) - strictEqual(runtime.listReady().events.length, 0) + strictEqual(run(runtime.listStaged()).events.length, 0) + strictEqual(run(runtime.listReady()).events.length, 0) } finally { - store.close() + await store.close() } })) it("rejects disjoint projections over conflicting records with one source occurrence", async () => withDataDir(async (dataDir) => { - const store = await LocalEventingControlStore.open(dataDir) + const store = await openStore(dataDir) try { - const runtime = new LocalEventingRuntime(store, undefined, exampleProjectors()) - runtime.activate(eventNameProjection("observed-events", "example.record.observed")) - runtime.activate(eventNameProjection("alternate-events", "example.record.alternate")) + const runtime = makeRuntime(store, exampleProjectors()) + run(runtime.activate(eventNameProjection("observed-events", "example.record.observed"))) + run(runtime.activate(eventNameProjection("alternate-events", "example.record.alternate"))) const request = structuredClone(exampleRecordObserved) const sibling = structuredClone(firstLogRecord(request)) sibling.eventName = "example.record.alternate" request.resourceLogs[0]!.scopeLogs[0]!.logRecords.push(sibling) throws( - () => runtime.evaluateOtlp("logs", request), + () => run(runtime.evaluateOtlp("logs", request)), /source occurrence collision within one ingest batch/, ) - strictEqual(runtime.listStaged().events.length, 0) - strictEqual(runtime.listReady().events.length, 0) + strictEqual(run(runtime.listStaged()).events.length, 0) + strictEqual(run(runtime.listReady()).events.length, 0) } finally { - store.close() + await store.close() } })) it("activates a validated revision without restart and reloads it after restart", async () => withDataDir(async (dataDir) => { - let store = await LocalEventingControlStore.open(dataDir) - let runtime = new LocalEventingRuntime(store, undefined, exampleProjectors()) - runtime.activate(projection()) - strictEqual(runtime.evaluateOtlp("logs", exampleRecordObserved).events.length, 1) - runtime.activate( - projection({ - revision: 2, - selector: { - op: "eq", - field: { namespace: "signal", key: "event.name", type: "string" }, - value: { type: "string", value: "example.record.closed" }, - }, - }), + let store = await openStore(dataDir) + let runtime = makeRuntime(store, exampleProjectors()) + run(runtime.activate(projection())) + strictEqual(run(runtime.evaluateOtlp("logs", exampleRecordObserved)).events.length, 1) + run( + runtime.activate( + projection({ + revision: 2, + selector: { + op: "eq", + field: { namespace: "signal", key: "event.name", type: "string" }, + value: { type: "string", value: "example.record.closed" }, + }, + }), + ), ) - strictEqual(runtime.evaluateOtlp("logs", exampleRecordObserved).events.length, 0) - store.close() + strictEqual(run(runtime.evaluateOtlp("logs", exampleRecordObserved)).events.length, 0) + await store.close() - store = await LocalEventingControlStore.open(dataDir) + store = await openStore(dataDir) try { - runtime = new LocalEventingRuntime(store, undefined, exampleProjectors()) - strictEqual(runtime.listActive()[0]?.revision, 2) - strictEqual(runtime.evaluateOtlp("logs", exampleRecordObserved).events.length, 0) + runtime = makeRuntime(store, exampleProjectors()) + strictEqual(run(runtime.listActive)[0]?.revision, 2) + strictEqual(run(runtime.evaluateOtlp("logs", exampleRecordObserved)).events.length, 0) } finally { - store.close() + await store.close() } })) it("does no normalization or event work for a source with no active projection", async () => withDataDir(async (dataDir) => { - const store = await LocalEventingControlStore.open(dataDir) + const store = await openStore(dataDir) try { - const runtime = new LocalEventingRuntime(store) - deepStrictEqual(runtime.evaluateOtlp("logs", { malformed: Symbol("not decoded") }), { + const runtime = makeRuntime(store) + deepStrictEqual(run(runtime.evaluateOtlp("logs", { malformed: Symbol("not decoded") })), { events: [], eventSourceFingerprints: new Map(), recoveredEventIds: [], @@ -623,7 +630,7 @@ describe("LocalEventingRuntime", () => { typeMismatchFields: [], }) } finally { - store.close() + await store.close() } })) }) diff --git a/apps/cli/test/local-eventing-telemetry.test.ts b/apps/cli/test/local-eventing-telemetry.test.ts index 466128b7e..0d0eaa309 100644 --- a/apps/cli/test/local-eventing-telemetry.test.ts +++ b/apps/cli/test/local-eventing-telemetry.test.ts @@ -2,7 +2,7 @@ import { strictEqual, ok } from "node:assert" import { describe, it } from "vitest" import { Effect, ManagedRuntime } from "effect" import { Maple } from "@maple-dev/effect-sdk/server" -import { makeEffectEventingTelemetry } from "../src/server/eventing/telemetry" +import { observeEventing } from "../src/server/eventing/telemetry" describe("eventing metric export", () => { it("exports eventing counters through the CLI's server SDK layer", async () => { @@ -25,12 +25,9 @@ describe("eventing metric export", () => { }), ) try { - const pending: Promise[] = [] - const telemetry = makeEffectEventingTelemetry((effect) => - pending.push(runtime.runPromise(effect)), + await runtime.runPromise( + observeEventing({ operation: "outbox_stage", outcome: "success", count: 3 }), ) - telemetry.record({ operation: "outbox_stage", outcome: "success", count: 3 }) - await Promise.all(pending) await runtime.runPromise( Effect.repeat(Effect.sleep("10 millis"), { until: () => bodies.some((body) => body.includes("maple.eventing.operations_total")),