diff --git a/apps/cli/src/server/checkpoints.ts b/apps/cli/src/server/checkpoints.ts index 97fbf063d..78109c51a 100644 --- a/apps/cli/src/server/checkpoints.ts +++ b/apps/cli/src/server/checkpoints.ts @@ -22,7 +22,9 @@ import { } from "./durable-files" import { eventingControlSnapshotPath, - LocalEventingControlStore, + openControlStore, + restoreControlSnapshot, + validateControlSnapshot, type EventingControlSnapshotValidation, } from "./eventing/control-store" import { CURRENT_LOCAL_SCHEMA, SCHEMA_FINGERPRINT } from "./schema-identity" @@ -885,7 +887,7 @@ const resolveCheckpointById = async ( const controlSha256 = await sha256File(controlPath) if (controlSha256 !== manifest.controlSha256) throw new Error("checkpoint control-store digest mismatch") - const controlValidation = LocalEventingControlStore.validateSnapshot(controlPath) + const controlValidation = await Effect.runPromise(validateControlSnapshot(controlPath)) if (!controlValidationMatches(manifest.controlValidation, controlValidation)) throw new Error("checkpoint control-store validation does not match its manifest") } else if (existsSync(controlPath)) { @@ -933,13 +935,12 @@ const restoreResolvedInto = async ( "SETTINGS allow_different_database_def=1", ) if (resolvedCheckpoint.manifest.formatVersion === MANIFEST_FORMAT_VERSION) { - await LocalEventingControlStore.restoreSnapshot( - join(resolvedCheckpoint.snapshotDir, "control.sqlite"), - targetDataDir, + await Effect.runPromise( + restoreControlSnapshot(join(resolvedCheckpoint.snapshotDir, "control.sqlite"), targetDataDir), ) } else { - const controlStore = await LocalEventingControlStore.open(targetDataDir) - controlStore.close() + // A legacy checkpoint has no control snapshot: open and close to create an empty store. + await Effect.runPromise(Effect.scoped(Effect.asVoid(openControlStore(targetDataDir)))) } return { db, validation: validateRestoredDatabase(db) } } catch (error) { @@ -1687,15 +1688,22 @@ const createCheckpointTraced = Effect.fn("CheckpointService.create")(function* ( : createError(error), ), ) - return yield* Effect.tryPromise({ + const controlPath = eventingControlSnapshotPath(options.dataDir, checkpointId) + yield* Effect.tryPromise({ try: async () => { - const { oldState, snapshot, startedAt } = prepared - let { operation } = prepared await syncTree(snapshotBackupDir(options.dataDir, checkpointId)) - const controlPath = eventingControlSnapshotPath(options.dataDir, checkpointId) await assertNoSymlink(checkpointSnapshotsRoot(options.dataDir), controlPath) await assertRealFile(controlPath, "checkpoint eventing control snapshot") - const controlValidation = LocalEventingControlStore.validateSnapshot(controlPath) + }, + catch: createError, + }) + const controlValidation = yield* validateControlSnapshot(controlPath).pipe( + Effect.mapError(createError), + ) + return yield* Effect.tryPromise({ + try: async () => { + const { oldState, snapshot, startedAt } = prepared + let { operation } = prepared operation = { ...operation, phase: "backup-complete" } await writeOperation(options.dataDir, operation, options.faults) const provisionalManifest: CheckpointManifest = { diff --git a/apps/cli/src/server/eventing/consumer-auth.ts b/apps/cli/src/server/eventing/consumer-auth.ts index 3318a1b11..7b8889295 100644 --- a/apps/cli/src/server/eventing/consumer-auth.ts +++ b/apps/cli/src/server/eventing/consumer-auth.ts @@ -1,7 +1,20 @@ import { resolve } from "node:path" +import { Effect, Schema } from "effect" import { ensureLocalToken, localTokenMatches } from "../local-token" +export class EventConsumerTokenError extends Schema.TaggedError()( + "@maple/cli/eventing/EventConsumerTokenFailed", + { message: Schema.String, cause: Schema.Defect() }, +) {} + export const eventConsumerTokenPath = (dataDir: string): string => `${resolve(dataDir)}.event-consumer-token` -export const ensureEventConsumerToken = (dataDir: string): Promise => - ensureLocalToken(eventConsumerTokenPath(dataDir), "event consumer token") +export const ensureEventConsumerToken = (dataDir: string): Effect.Effect => + Effect.tryPromise({ + try: () => ensureLocalToken(eventConsumerTokenPath(dataDir), "event consumer token"), + catch: (cause) => + new EventConsumerTokenError({ + message: cause instanceof Error ? cause.message : String(cause), + cause, + }), + }) export const eventConsumerTokenMatches = localTokenMatches diff --git a/apps/cli/src/server/eventing/control-store.ts b/apps/cli/src/server/eventing/control-store.ts index 260b67459..2d1f7d9be 100644 --- a/apps/cli/src/server/eventing/control-store.ts +++ b/apps/cli/src/server/eventing/control-store.ts @@ -15,9 +15,9 @@ import { type ProjectionFailure, type SignalProjectionSpec, } from "@maple/eventing-core" -import { Result, Schema } from "effect" +import { Context, Effect, Layer, Result, Schema, type Scope } from "effect" import { durableWrite, ensurePrivateDirectory } from "../durable-files" -import { NOOP_EVENTING_TELEMETRY, type EventingTelemetry } from "./telemetry" +import { observeEventing } from "./telemetry" const CONTROL_DIRECTORY = "control" const CONTROL_DATABASE = "eventing.sqlite" @@ -241,9 +241,51 @@ interface DeliveryGapRow { readonly last_dropped_at: string } +/** Any other control-store failure: SQLite, an invariant, or a corrupt row. */ +export class EventingControlStoreError extends Schema.TaggedError()( + "@maple/cli/eventing/ControlStoreFailed", + { message: Schema.String, cause: Schema.optionalKey(Schema.Defect()) }, +) {} + +export type EventConsumerFailure = + | EventConsumerInputError + | EventConsumerNotFoundError + | EventConsumerConflictError + | EventConsumerLeaseError + | EventConsumerDeliveryGapError + | EventingControlStoreError + +const storeError = (message: string): EventingControlStoreError => new EventingControlStoreError({ message }) + +const isStoreError = Schema.is(EventingControlStoreError) +const isConsumerFailure = Schema.is( + Schema.Union([ + EventConsumerInputError, + EventConsumerNotFoundError, + EventConsumerConflictError, + EventConsumerLeaseError, + EventConsumerDeliveryGapError, + EventingControlStoreError, + ]), +) +const isOutboxAdministrationInvalid = Schema.is(OutboxAdministrationInvalid) + +/** Maps a failure thrown inside one synchronous SQLite step onto the typed channel. */ +const storeFailure = (error: unknown): EventingControlStoreError => + isStoreError(error) + ? error + : new EventingControlStoreError({ + message: error instanceof Error ? error.message : String(error), + cause: error, + }) +const consumerFailure = (error: unknown): EventConsumerFailure => + isConsumerFailure(error) ? error : storeFailure(error) +const administrationFailure = (error: unknown): OutboxAdministrationInvalid | EventingControlStoreError => + isOutboxAdministrationInvalid(error) ? error : storeFailure(error) + const asNumber = (value: number | bigint): number => { const number = Number(value) - if (!Number.isSafeInteger(number) || number < 0) throw new Error(`invalid SQLite integer: ${value}`) + if (!Number.isSafeInteger(number) || number < 0) throw storeError(`invalid SQLite integer: ${value}`) return number } @@ -251,21 +293,21 @@ const decodeProjection = (json: string): SignalProjectionSpec => decodeSignalProjectionSpec(Schema.decodeUnknownSync(Schema.fromJsonString(Schema.Unknown))(json)) const decodeEvent = (json: string): MapleCloudEvent => { - return Result.getOrThrow( - validateMapleCloudEvent(Schema.decodeUnknownSync(Schema.fromJsonString(Schema.Unknown))(json)), - ).event + const validated = validateMapleCloudEvent( + Schema.decodeUnknownSync(Schema.fromJsonString(Schema.Unknown))(json), + ) + if (Result.isFailure(validated)) throw validated.failure + return validated.success.event } const assertRealDatabaseFile = (path: string): void => { - let info - try { - info = lstatSync(path) - } catch (error) { - if (Schema.is(Schema.Struct({ code: Schema.Literal("ENOENT") }))(error)) return - throw error + const info = Result.try(() => lstatSync(path)) + if (Result.isFailure(info)) { + if (Schema.is(Schema.Struct({ code: Schema.Literal("ENOENT") }))(info.failure)) return + throw info.failure } - if (info.isSymbolicLink() || !info.isFile()) - throw new Error(`eventing control database is not a real file: ${path}`) + if (info.success.isSymbolicLink() || !info.success.isFile()) + throw storeError(`eventing control database is not a real file: ${path}`) } const configure = (db: Database): void => { @@ -276,25 +318,25 @@ const configure = (db: Database): void => { const checkpointWal = (db: Database): void => { const result = db.query("PRAGMA wal_checkpoint(TRUNCATE)").get() - if (!result) throw new Error("eventing control WAL checkpoint returned no result") + if (!result) throw storeError("eventing control WAL checkpoint returned no result") const busy = asNumber(result.busy) const log = asNumber(result.log) const checkpointed = asNumber(result.checkpointed) if (busy !== 0 || log !== 0) - throw new Error( + throw storeError( `eventing control WAL checkpoint incomplete (busy=${busy}, log=${log}, checkpointed=${checkpointed})`, ) } const validateLimits = (limits: LocalEventingControlLimits): ResolvedLocalEventingControlLimits => { if (!Number.isSafeInteger(limits.maxOutboxEvents) || limits.maxOutboxEvents < 1) - throw new Error("maxOutboxEvents must be a positive safe integer") + throw storeError("maxOutboxEvents must be a positive safe integer") if (!Number.isSafeInteger(limits.maxOutboxBytes) || limits.maxOutboxBytes < 1) - throw new Error("maxOutboxBytes must be a positive safe integer") + throw storeError("maxOutboxBytes must be a positive safe integer") const retainAcknowledgedReadyEvents = limits.retainAcknowledgedReadyEvents ?? DEFAULT_RETAIN_ACKNOWLEDGED_READY_EVENTS if (!Number.isSafeInteger(retainAcknowledgedReadyEvents) || retainAcknowledgedReadyEvents < 0) - throw new Error("retainAcknowledgedReadyEvents must be a non-negative safe integer") + throw storeError("retainAcknowledgedReadyEvents must be a non-negative safe integer") return { ...limits, retainAcknowledgedReadyEvents } } @@ -303,12 +345,12 @@ const validateOpenDatabase = ( acceptedSchemaVersions: readonly number[] = [CONTROL_SCHEMA_VERSION], ): EventingControlSnapshotValidation => { const quick = db.query("PRAGMA quick_check").get() - if (quick?.quick_check !== "ok") throw new Error(`eventing control database quick_check failed`) + if (quick?.quick_check !== "ok") throw storeError(`eventing control database quick_check failed`) const version = db.query("PRAGMA user_version").get() - if (!version) throw new Error("eventing control database has no schema version") + if (!version) throw storeError("eventing control database has no schema version") const schemaVersion = asNumber(version.user_version) if (!acceptedSchemaVersions.includes(schemaVersion)) - throw new Error( + throw storeError( `unsupported eventing control schema ${schemaVersion}; expected ${acceptedSchemaVersions.join(" or ")}`, ) // Full accounting verification belongs at open/restore, never on the ingest hot path. @@ -321,19 +363,19 @@ const validateOpenDatabase = ( try { const row = accounting.get() if (row === null || asNumber(row.count) !== 1) - throw new Error("eventing control outbox accounting is inconsistent") + throw storeError("eventing control outbox accounting is inconsistent") } finally { accounting.finalize() } const count = (where: string): number => { const row = db.query(`SELECT count(*) AS count FROM outbox_events ${where}`).get() - if (!row) throw new Error("eventing control count query returned no row") + if (!row) throw storeError("eventing control count query returned no row") return asNumber(row.count) } const revisions = db.query("SELECT count(*) AS count FROM projection_revisions").get() - if (!revisions) throw new Error("eventing projection count query returned no row") + if (!revisions) throw storeError("eventing projection count query returned no row") const failures = db.query("SELECT count(*) AS count FROM projection_failures").get() - if (!failures) throw new Error("eventing projection-failure count query returned no row") + if (!failures) throw storeError("eventing projection-failure count query returned no row") const invalidReadiness = db .query( `SELECT count(*) AS count @@ -346,9 +388,9 @@ const validateOpenDatabase = ( ))`, ) .get() - if (!invalidReadiness) throw new Error("eventing readiness validation query returned no row") + if (!invalidReadiness) throw storeError("eventing readiness validation query returned no row") if (asNumber(invalidReadiness.count) !== 0) - throw new Error("eventing control database has inconsistent outbox readiness state") + throw storeError("eventing control database has inconsistent outbox readiness state") { const consumers = db .query, []>( @@ -383,9 +425,9 @@ const validateOpenDatabase = ( statement.finalize() } if (invalidFingerprints === null) - throw new Error("eventing staged source-fingerprint validation returned no row") + throw storeError("eventing staged source-fingerprint validation returned no row") if (asNumber(invalidFingerprints.count) > 0) - throw new Error("eventing control database has an invalid staged source fingerprint") + throw storeError("eventing control database has an invalid staged source fingerprint") } return { schemaVersion, @@ -435,145 +477,424 @@ const decodeConsumer = (row: ConsumerRow): EventConsumer => ({ disabledAt: row.disabled_at, }) -export class LocalEventingControlStore { - readonly #db: Database - #stagedSourceKinds = new Set() - readonly #limits: ResolvedLocalEventingControlLimits - readonly #telemetry: EventingTelemetry - readonly path: string +const DEFAULT_LIMITS: LocalEventingControlLimits = { + maxOutboxEvents: DEFAULT_MAX_OUTBOX_EVENTS, + maxOutboxBytes: DEFAULT_MAX_OUTBOX_BYTES, + retainAcknowledgedReadyEvents: DEFAULT_RETAIN_ACKNOWLEDGED_READY_EVENTS, +} - private constructor( +/** Where the control store lives; supplied by the application root. */ +export class LocalEventingControlConfig extends Context.Service< + LocalEventingControlConfig, + { readonly dataDir: string; readonly limits?: LocalEventingControlLimits } +>()("@maple/cli/eventing/LocalEventingControlConfig") {} + +export type OutboxCapacity = LocalEventingControlLimits & { + readonly currentEvents: number + readonly currentBytes: number +} + +export interface LocalEventingControlStoreApi { + readonly path: string + readonly saveProjection: ( + spec: SignalProjectionSpec, + createdAt?: string, + ) => Effect.Effect + readonly loadEnabledProjections: ( + tenantId: string, + ) => Effect.Effect + readonly stageEvents: ( + events: readonly MapleCloudEvent[], + sourceFingerprints?: ReadonlyMap, + stagedAt?: string, + ) => Effect.Effect + readonly deliveryGap: (tenantId: string) => Effect.Effect + readonly acceptDeliveryGap: ( + tenantId: string, + consumerId: string, + generation: number, + ) => Effect.Effect + /** Operator-authorized loss; the HTTP caller drains admission before invoking this transaction. */ + readonly abandonEvents: ( + tenantId: string, + eventIds: readonly string[], + ) => Effect.Effect< + { readonly abandoned: number; readonly gap: DeliveryGap }, + OutboxAdministrationInvalid | EventingControlStoreError + > + readonly hasStagedSourceKind: (tenantId: string, sourceKind: string) => Effect.Effect + readonly hasStagedSourceOccurrence: ( + tenantId: string, + sourceKind: string, + source: string, + sourceOccurrenceId: string, + ) => Effect.Effect + readonly stagedEventIdsForOccurrence: ( + tenantId: string, + sourceKind: string, + source: string, + sourceOccurrenceId: string, + sourceFingerprint: string, + ) => Effect.Effect + readonly markReady: ( + eventIds: readonly string[], + readyAt?: string, + ) => Effect.Effect + readonly listReady: ( + limit?: number, + after?: number, + ) => Effect.Effect + readonly listStaged: ( + limit?: number, + after?: number, + ) => Effect.Effect + readonly listConsumers: ( + tenantId: string, + ) => Effect.Effect + readonly registerConsumer: ( + tenantId: string, + consumerId: string, + startAt: EventConsumerStart, + registeredAt?: string, + ) => Effect.Effect + readonly disableConsumer: ( + tenantId: string, + consumerId: string, + disabledAt?: string, + ) => Effect.Effect + readonly claimReady: ( + tenantId: string, + consumerId: string, + limit: number, + leaseSeconds: number, + now?: string, + ) => Effect.Effect + readonly acknowledgeClaim: ( + tenantId: string, + consumerId: string, + leaseToken: string, + throughSequence: number, + now?: string, + ) => Effect.Effect + readonly outboxCapacity: Effect.Effect + readonly recordProjectionFailures: ( + tenantId: string, + failures: readonly ProjectionFailure[], + createdAt?: string, + ) => Effect.Effect + readonly validate: Effect.Effect + /** Synchronous so a caller can pair it with another synchronous capture. */ + readonly captureSnapshot: Effect.Effect + readonly backupTo: ( path: string, - db: Database, - limits: ResolvedLocalEventingControlLimits, - telemetry: EventingTelemetry, - ) { - this.path = path - this.#db = db - this.#limits = limits - this.#telemetry = telemetry - this.#refreshStagedSourceKinds() - } + ) => Effect.Effect +} + +const now = (): string => new Date().toISOString() - static async open( - dataDir: string, - limits: LocalEventingControlLimits = { - maxOutboxEvents: DEFAULT_MAX_OUTBOX_EVENTS, - maxOutboxBytes: DEFAULT_MAX_OUTBOX_BYTES, - retainAcknowledgedReadyEvents: DEFAULT_RETAIN_ACKNOWLEDGED_READY_EVENTS, +/** Opens, migrates, and validates the database; a failure here closes the handle it opened. */ +const openDatabase = (path: string): Effect.Effect => + Effect.try({ + try: () => { + assertRealDatabaseFile(path) + return new Database(path, { create: true, readwrite: true, strict: true, safeIntegers: true }) }, - telemetry: EventingTelemetry = NOOP_EVENTING_TELEMETRY, - ): Promise { - const validatedLimits = validateLimits(limits) - const directory = eventingControlDirectory(dataDir) - await ensurePrivateDirectory(directory) - const path = eventingControlPath(dataDir) - assertRealDatabaseFile(path) - const db = new Database(path, { create: true, readwrite: true, strict: true, safeIntegers: true }) - try { - configure(db) - db.exec("PRAGMA journal_mode = WAL") - db.exec("PRAGMA synchronous = FULL") - const version = db.query("PRAGMA user_version").get() - if (!version) throw new Error("eventing control database has no schema version") - let schemaVersion = asNumber(version.user_version) - if (schemaVersion === 0) { - db.transaction(() => db.exec(CREATE_SCHEMA)).exclusive() - schemaVersion = CONTROL_SCHEMA_VERSION + catch: storeFailure, + }).pipe( + Effect.flatMap((db) => + Effect.try({ + try: () => { + configure(db) + db.exec("PRAGMA journal_mode = WAL") + db.exec("PRAGMA synchronous = FULL") + const version = db.query("PRAGMA user_version").get() + if (!version) throw storeError("eventing control database has no schema version") + let schemaVersion = asNumber(version.user_version) + if (schemaVersion === 0) { + db.transaction(() => db.exec(CREATE_SCHEMA)).exclusive() + schemaVersion = CONTROL_SCHEMA_VERSION + } + if (schemaVersion !== CONTROL_SCHEMA_VERSION) + throw storeError( + `unsupported eventing control schema ${schemaVersion}; expected ${CONTROL_SCHEMA_VERSION}`, + ) + chmodSync(path, 0o600) + validateOpenDatabase(db) + return db + }, + catch: storeFailure, + }).pipe(Effect.onError(() => Effect.sync(() => db.close()))), + ), + ) + +/** A clean close truncates the WAL so the next open and any file-level copy see one file. */ +const closeDatabase = (db: Database): Effect.Effect => + Effect.try({ + try: () => { + checkpointWal(db) + db.close(true) + }, + catch: (cause) => + new EventingControlStoreError({ message: "failed to close eventing control store", cause }), + }).pipe(Effect.catchTag("@maple/cli/eventing/ControlStoreFailed", (error) => Effect.logError(error))) + +export class LocalEventingControlStore extends Context.Service< + LocalEventingControlStore, + LocalEventingControlStoreApi +>()("@maple/cli/eventing/LocalEventingControlStore") { + static readonly make: Effect.Effect< + LocalEventingControlStoreApi, + EventingControlStoreError, + LocalEventingControlConfig | Scope.Scope + > = Effect.gen(function* () { + const config = yield* LocalEventingControlConfig + const limits = yield* Effect.try({ + try: () => validateLimits(config.limits ?? DEFAULT_LIMITS), + catch: storeFailure, + }) + yield* Effect.tryPromise({ + try: () => ensurePrivateDirectory(eventingControlDirectory(config.dataDir)), + catch: storeFailure, + }) + const path = eventingControlPath(config.dataDir) + const db = yield* Effect.acquireRelease(openDatabase(path), closeDatabase) + + const readStagedSourceKinds = (): Set => { + const statement = db.prepare<{ tenant_id: string; source_kind: string }, []>( + "SELECT DISTINCT tenant_id, source_kind FROM outbox_events WHERE state = 'staged' AND source_kind IS NOT NULL", + ) + try { + return new Set(statement.all().map((row) => JSON.stringify([row.tenant_id, row.source_kind]))) + } finally { + statement.finalize() } - if (schemaVersion !== CONTROL_SCHEMA_VERSION) - throw new Error( - `unsupported eventing control schema ${schemaVersion}; expected ${CONTROL_SCHEMA_VERSION}`, - ) - chmodSync(path, 0o600) - validateOpenDatabase(db) - return new LocalEventingControlStore(path, db, validatedLimits, telemetry) - } catch (error) { - db.close() - throw error } - } + // Read on every ingest request, so it is cached and refreshed after each committed write. + let stagedSourceKinds = yield* Effect.try({ try: readStagedSourceKinds, catch: storeFailure }) + const refreshStagedSourceKinds = Effect.try({ + try: () => { + stagedSourceKinds = readStagedSourceKinds() + }, + catch: storeFailure, + }) - close(): void { - checkpointWal(this.#db) - this.#db.close(true) - } + const readDeliveryGap = (tenantId: string): DeliveryGap => { + const statement = db.prepare( + "SELECT generation, dropped_events, last_dropped_at FROM delivery_gaps WHERE tenant_id = ?", + ) + try { + const row = statement.get(tenantId) + return row === null + ? { generation: 0, droppedEvents: 0, lastDroppedAt: null } + : { + generation: asNumber(row.generation), + droppedEvents: asNumber(row.dropped_events), + lastDroppedAt: row.last_dropped_at, + } + } finally { + statement.finalize() + } + } + const recordDeliveryGap = (tenantId: string, count: number, at: string): void => { + if (count === 0) return + db.run( + `INSERT INTO delivery_gaps (tenant_id, generation, dropped_events, last_dropped_at) VALUES (?, 1, ?, ?) + ON CONFLICT (tenant_id) DO UPDATE SET generation = generation + 1, dropped_events = dropped_events + excluded.dropped_events, last_dropped_at = excluded.last_dropped_at`, + [tenantId, count, at], + ) + } + const outboxUsage = (): OutboxUsageRow => { + const statement = db.prepare( + "SELECT count, bytes FROM outbox_usage WHERE singleton = 1", + ) + try { + const usage = statement.get() + if (usage === null) throw storeError("event outbox usage query returned no row") + return usage + } finally { + statement.finalize() + } + } + const consumerRow = (tenantId: string, consumerId: string): ConsumerRow | null => + db + .query( + `SELECT consumer_id, tenant_id, active, last_acked_sequence, accepted_gap_generation, lease_token_hash, + lease_expires_at, claimed_through_sequence, registered_at, disabled_at + FROM event_consumers + WHERE tenant_id = ? AND consumer_id = ?`, + ) + .get(tenantId, consumerId) + const consumerLag = (tenantId: string, lastAcknowledgedSequence: number): number => { + const latest = db + .query( + `SELECT max(readiness.sequence) AS sequence + FROM outbox_ready_events AS readiness + INNER JOIN outbox_events AS event ON event.event_id = readiness.event_id + WHERE event.tenant_id = ? AND event.state = 'ready'`, + ) + .get(tenantId) + return Math.max( + 0, + (latest?.sequence == null ? 0 : asNumber(latest.sequence)) - lastAcknowledgedSequence, + ) + } + const pruneAcknowledgedReady = (tenantId: string): number => { + const boundary = db + .query( + "SELECT min(last_acked_sequence) AS sequence FROM event_consumers WHERE tenant_id = ? AND active = 1", + ) + .get(tenantId) + if (boundary?.sequence == null) return 0 + const rows = db + .query( + `SELECT readiness.event_id + FROM outbox_ready_events AS readiness + INNER JOIN outbox_events AS event ON event.event_id = readiness.event_id + WHERE event.tenant_id = ? AND readiness.sequence <= ? + ORDER BY readiness.sequence`, + ) + .all(tenantId, asNumber(boundary.sequence)) + const pruneCount = Math.max(0, rows.length - limits.retainAcknowledgedReadyEvents) + for (const { event_id } of rows.slice(0, pruneCount)) { + db.run("DELETE FROM outbox_ready_events WHERE event_id = ?", [event_id]) + db.run("DELETE FROM outbox_events WHERE event_id = ? AND state = 'ready'", [event_id]) + } + return pruneCount + } + const listOutbox = ( + state: "ready" | "staged", + limit = 100, + after = 0, + ): Effect.Effect => + Effect.try({ + try: () => { + if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1_000) + throw storeError("outbox-event limit must be between 1 and 1000") + if (!Number.isSafeInteger(after) || after < 0) + throw storeError("outbox cursor must be a non-negative safe integer") + const rows = + state === "ready" + ? db + .query( + `SELECT readiness.sequence, event.event_json, event.staged_at, readiness.ready_at + FROM outbox_ready_events AS readiness + INNER JOIN outbox_events AS event ON event.event_id = readiness.event_id + WHERE event.state = 'ready' AND readiness.sequence > ? + ORDER BY readiness.sequence + LIMIT ?`, + ) + .all(after, limit + 1) + : db + .query( + `SELECT sequence, event_json, staged_at, ready_at + FROM outbox_events + WHERE state = 'staged' AND sequence > ? + ORDER BY sequence + LIMIT ?`, + ) + .all(after, limit + 1) + const hasMore = rows.length > limit + const pageRows = hasMore ? rows.slice(0, limit) : rows + const page = pageRows.map(({ sequence, event_json, staged_at, ready_at }) => ({ + sequence: asNumber(sequence), + event: decodeEvent(event_json), + stagedAt: staged_at, + readyAt: ready_at, + })) + return { + events: page, + nextCursor: hasMore ? (page.at(-1)?.sequence ?? null) : null, + } + }, + catch: storeFailure, + }) + const captureSnapshot = Effect.try({ + try: () => { + checkpointWal(db) + return db.serialize() + }, + catch: storeFailure, + }) - saveProjection(spec: SignalProjectionSpec, createdAt = new Date().toISOString()): void { - const decoded = decodeSignalProjectionSpec(spec) - if (!isJsonValue(decoded)) throw new Error("projection spec must be finite JSON") - const specJson = canonicalJson(decoded) - this.#db - .transaction(() => { - const latest = this.#db - .query( - "SELECT max(revision) AS revision FROM projection_revisions WHERE tenant_id = ? AND projection_id = ?", - ) - .get(decoded.tenantId, decoded.id) - const latestRevision = latest?.revision == null ? null : asNumber(latest.revision) - const existing = this.#db - .query( - "SELECT spec_json FROM projection_revisions WHERE tenant_id = ? AND projection_id = ? AND revision = ?", - ) - .get(decoded.tenantId, decoded.id, decoded.revision) - if (existing) { - if (existing.spec_json !== specJson) - throw new Error( - `projection revision is immutable: ${decoded.tenantId}:${decoded.id}@${decoded.revision}`, - ) - if (latestRevision !== decoded.revision) - throw new Error( - `stale projection revision: ${decoded.tenantId}:${decoded.id}@${decoded.revision}; latest is ${latestRevision}`, - ) - const active = this.#db - .query( - "SELECT revision FROM active_projections WHERE tenant_id = ? AND projection_id = ?", - ) - .get(decoded.tenantId, decoded.id) - const activeRevision = active === null ? null : asNumber(active.revision) - const expectedActiveRevision = decoded.enabled ? decoded.revision : null - if (activeRevision !== expectedActiveRevision) - throw new Error( - `projection active state conflicts with exact revision replay: ${decoded.tenantId}:${decoded.id}@${decoded.revision}`, - ) - return - } else { - const expected = latestRevision === null ? 1 : latestRevision + 1 - if (decoded.revision !== expected) - throw new Error( - `projection revision must be ${expected}: ${decoded.tenantId}:${decoded.id}@${decoded.revision}`, - ) - this.#db.run( - "INSERT INTO projection_revisions (tenant_id, projection_id, revision, enabled, spec_json, created_at) VALUES (?, ?, ?, ?, ?, ?)", - [ - decoded.tenantId, - decoded.id, - decoded.revision, - decoded.enabled ? 1 : 0, - specJson, - createdAt, - ], - ) - } + const saveProjection: LocalEventingControlStoreApi["saveProjection"] = (spec, createdAt = now()) => + Effect.try({ + try: () => { + const decoded = decodeSignalProjectionSpec(spec) + if (!isJsonValue(decoded)) throw storeError("projection spec must be finite JSON") + const specJson = canonicalJson(decoded) + db.transaction(() => { + const latest = db + .query( + "SELECT max(revision) AS revision FROM projection_revisions WHERE tenant_id = ? AND projection_id = ?", + ) + .get(decoded.tenantId, decoded.id) + const latestRevision = latest?.revision == null ? null : asNumber(latest.revision) + const existing = db + .query( + "SELECT spec_json FROM projection_revisions WHERE tenant_id = ? AND projection_id = ? AND revision = ?", + ) + .get(decoded.tenantId, decoded.id, decoded.revision) + if (existing) { + if (existing.spec_json !== specJson) + throw storeError( + `projection revision is immutable: ${decoded.tenantId}:${decoded.id}@${decoded.revision}`, + ) + if (latestRevision !== decoded.revision) + throw storeError( + `stale projection revision: ${decoded.tenantId}:${decoded.id}@${decoded.revision}; latest is ${latestRevision}`, + ) + const active = db + .query( + "SELECT revision FROM active_projections WHERE tenant_id = ? AND projection_id = ?", + ) + .get(decoded.tenantId, decoded.id) + const activeRevision = active === null ? null : asNumber(active.revision) + const expectedActiveRevision = decoded.enabled ? decoded.revision : null + if (activeRevision !== expectedActiveRevision) + throw storeError( + `projection active state conflicts with exact revision replay: ${decoded.tenantId}:${decoded.id}@${decoded.revision}`, + ) + return + } else { + const expected = latestRevision === null ? 1 : latestRevision + 1 + if (decoded.revision !== expected) + throw storeError( + `projection revision must be ${expected}: ${decoded.tenantId}:${decoded.id}@${decoded.revision}`, + ) + db.run( + "INSERT INTO projection_revisions (tenant_id, projection_id, revision, enabled, spec_json, created_at) VALUES (?, ?, ?, ?, ?, ?)", + [ + decoded.tenantId, + decoded.id, + decoded.revision, + decoded.enabled ? 1 : 0, + specJson, + createdAt, + ], + ) + } - if (decoded.enabled) - this.#db.run( - "INSERT INTO active_projections (tenant_id, projection_id, revision) VALUES (?, ?, ?) ON CONFLICT (tenant_id, projection_id) DO UPDATE SET revision = excluded.revision", - [decoded.tenantId, decoded.id, decoded.revision], - ) - else - this.#db.run("DELETE FROM active_projections WHERE tenant_id = ? AND projection_id = ?", [ - decoded.tenantId, - decoded.id, - ]) + if (decoded.enabled) + db.run( + "INSERT INTO active_projections (tenant_id, projection_id, revision) VALUES (?, ?, ?) ON CONFLICT (tenant_id, projection_id) DO UPDATE SET revision = excluded.revision", + [decoded.tenantId, decoded.id, decoded.revision], + ) + else + db.run( + "DELETE FROM active_projections WHERE tenant_id = ? AND projection_id = ?", + [decoded.tenantId, decoded.id], + ) + }).immediate() + }, + catch: storeFailure, }) - .immediate() - } - loadEnabledProjections(tenantId: string): readonly SignalProjectionSpec[] { - return this.#db - .query( - `SELECT r.spec_json + const loadEnabledProjections: LocalEventingControlStoreApi["loadEnabledProjections"] = (tenantId) => + Effect.try({ + try: () => + db + .query( + `SELECT r.spec_json FROM active_projections a JOIN projection_revisions r ON r.tenant_id = a.tenant_id @@ -581,795 +902,825 @@ export class LocalEventingControlStore { AND r.revision = a.revision WHERE a.tenant_id = ? ORDER BY a.projection_id`, - ) - .all(tenantId) - .map(({ spec_json }) => decodeProjection(spec_json)) - } + ) + .all(tenantId) + .map(({ spec_json }) => decodeProjection(spec_json)), + catch: storeFailure, + }) - stageEvents( - events: readonly MapleCloudEvent[], - sourceFingerprints: ReadonlyMap = new Map(), - stagedAt = new Date().toISOString(), - ): StageEventsResult { - let inserted = 0 - let deduplicated = 0 - const droppedByTenant = new Map() - let dropped = 0 - const eventIds: string[] = [] - try { - this.#db - .transaction(() => { - const usage = this.#outboxUsage() - if (!usage) throw new Error("event outbox usage query returned no row") - let outboxEvents = asNumber(usage.count) - let outboxBytes = asNumber(usage.bytes) - for (const candidate of events) { - const validated = Result.getOrThrow(validateMapleCloudEvent(candidate)) - const { event, canonicalJson: eventJson, byteLength: eventBytes } = validated - const sourceFingerprint = sourceFingerprints.get(event.id) ?? null - if (sourceFingerprint !== null && !/^sha256:[0-9a-f]{64}$/.test(sourceFingerprint)) - throw new Error(`event has invalid source fingerprint: ${event.id}`) - if (event.sourceoccurrenceid !== undefined && sourceFingerprint === null) - throw new Error( - `event with source occurrence ID requires a source fingerprint: ${event.id}`, - ) - let sourceKind: string | null = null - if (event.sourceoccurrenceid !== undefined) { - const projection = this.#db - .query( - "SELECT spec_json FROM projection_revisions WHERE tenant_id = ? AND projection_id = ? AND revision = ?", - ) - .get(event.tenantid, event.projectionid, event.projectionrevision) - if (projection === null) - throw new Error( - `event references unknown projection revision: ${event.tenantid}:${event.projectionid}@${event.projectionrevision}`, + const stageEvents: LocalEventingControlStoreApi["stageEvents"] = ( + events, + sourceFingerprints = new Map(), + stagedAt = now(), + ) => + Effect.try({ + try: () => + db + .transaction(() => { + let inserted = 0 + let deduplicated = 0 + const droppedByTenant = new Map() + let dropped = 0 + const eventIds: string[] = [] + const usage = outboxUsage() + let outboxEvents = asNumber(usage.count) + let outboxBytes = asNumber(usage.bytes) + for (const candidate of events) { + const validation = validateMapleCloudEvent(candidate) + if (Result.isFailure(validation)) throw validation.failure + const { + event, + canonicalJson: eventJson, + byteLength: eventBytes, + } = validation.success + const sourceFingerprint = sourceFingerprints.get(event.id) ?? null + if ( + sourceFingerprint !== null && + !/^sha256:[0-9a-f]{64}$/.test(sourceFingerprint) ) - sourceKind = decodeProjection(projection.spec_json).sourceKind - } - const existing = this.#db - .query( - "SELECT event_id, event_json, state, source_fingerprint FROM outbox_events WHERE event_id = ?", - ) - .get(event.id) - if (existing) { - if (existing.event_json !== eventJson) - throw new Error(`event ID collision with different payload: ${event.id}`) - if ( - sourceFingerprint !== null && - existing.source_fingerprint !== null && - existing.source_fingerprint !== sourceFingerprint - ) - throw new Error( - `event ID collision with different source occurrence: ${event.id}`, + throw storeError(`event has invalid source fingerprint: ${event.id}`) + if (event.sourceoccurrenceid !== undefined && sourceFingerprint === null) + throw storeError( + `event with source occurrence ID requires a source fingerprint: ${event.id}`, + ) + let sourceKind: string | null = null + if (event.sourceoccurrenceid !== undefined) { + const projection = db + .query( + "SELECT spec_json FROM projection_revisions WHERE tenant_id = ? AND projection_id = ? AND revision = ?", + ) + .get(event.tenantid, event.projectionid, event.projectionrevision) + if (projection === null) + throw storeError( + `event references unknown projection revision: ${event.tenantid}:${event.projectionid}@${event.projectionrevision}`, + ) + sourceKind = decodeProjection(projection.spec_json).sourceKind + } + const existing = db + .query( + "SELECT event_id, event_json, state, source_fingerprint FROM outbox_events WHERE event_id = ?", + ) + .get(event.id) + if (existing) { + if (existing.event_json !== eventJson) + throw storeError( + `event ID collision with different payload: ${event.id}`, + ) + if ( + sourceFingerprint !== null && + existing.source_fingerprint !== null && + existing.source_fingerprint !== sourceFingerprint + ) + throw storeError( + `event ID collision with different source occurrence: ${event.id}`, + ) + if ( + existing.state === "staged" && + sourceFingerprint !== null && + existing.source_fingerprint === null + ) + throw storeError( + `staged event has no recovery fingerprint: ${event.id}`, + ) + deduplicated += 1 + } else { + if ( + outboxEvents + 1 > limits.maxOutboxEvents || + outboxBytes + eventBytes > limits.maxOutboxBytes + ) { + dropped += 1 + droppedByTenant.set( + event.tenantid, + (droppedByTenant.get(event.tenantid) ?? 0) + 1, + ) + continue + } + db.run( + "INSERT INTO outbox_events (event_id, tenant_id, projection_id, projection_revision, source_kind, source, source_occurrence_id, source_fingerprint, state, event_json, staged_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'staged', ?, ?)", + [ + event.id, + event.tenantid, + event.projectionid, + event.projectionrevision, + sourceKind, + event.sourceoccurrenceid === undefined ? null : event.source, + event.sourceoccurrenceid ?? null, + sourceFingerprint, + eventJson, + stagedAt, + ], + ) + inserted += 1 + outboxEvents += 1 + outboxBytes += eventBytes + } + eventIds.push(event.id) + } + for (const [tenantId, count] of droppedByTenant) + recordDeliveryGap(tenantId, count, stagedAt) + return { inserted, deduplicated, dropped, eventIds } + }) + .immediate(), + catch: storeFailure, + }).pipe( + Effect.tapError(() => observeEventing({ operation: "outbox_stage", outcome: "failure" })), + Effect.tap(() => refreshStagedSourceKinds), + Effect.tap((result) => + Effect.all( + [ + observeEventing({ + operation: "outbox_stage", + outcome: "success", + count: result.inserted, + }), + observeEventing({ + operation: "outbox_dedup", + outcome: "success", + count: result.deduplicated, + }), + observeEventing({ + operation: "outbox_stage", + outcome: "dropped", + count: result.dropped, + }), + ], + { discard: true }, + ), + ), + ) + + const deliveryGap: LocalEventingControlStoreApi["deliveryGap"] = (tenantId) => + Effect.try({ try: () => readDeliveryGap(tenantId), catch: storeFailure }) + + const acceptDeliveryGap: LocalEventingControlStoreApi["acceptDeliveryGap"] = ( + tenantId, + consumerId, + generation, + ) => + Effect.try({ + try: () => { + validateConsumerId(consumerId) + return db + .transaction(() => { + const consumer = consumerRow(tenantId, consumerId) + if (consumer === null) + throw EventConsumerNotFoundError.create( + `event consumer not found: ${consumerId}`, + consumerId, ) + const gap = readDeliveryGap(tenantId) if ( - existing.state === "staged" && - sourceFingerprint !== null && - existing.source_fingerprint === null + !Number.isSafeInteger(generation) || + generation < 1 || + generation !== gap.generation ) - throw new Error(`staged event has no recovery fingerprint: ${event.id}`) - deduplicated += 1 - } else { - if ( - outboxEvents + 1 > this.#limits.maxOutboxEvents || - outboxBytes + eventBytes > this.#limits.maxOutboxBytes - ) { - dropped += 1 - droppedByTenant.set( - event.tenantid, - (droppedByTenant.get(event.tenantid) ?? 0) + 1, + throw EventConsumerConflictError.create( + "delivery gap generation changed; inspect current health before accepting", + consumerId, ) - continue - } - this.#db.run( - "INSERT INTO outbox_events (event_id, tenant_id, projection_id, projection_revision, source_kind, source, source_occurrence_id, source_fingerprint, state, event_json, staged_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'staged', ?, ?)", - [ - event.id, - event.tenantid, - event.projectionid, - event.projectionrevision, - sourceKind, - event.sourceoccurrenceid === undefined ? null : event.source, - event.sourceoccurrenceid ?? null, - sourceFingerprint, - eventJson, - stagedAt, - ], + db.run( + "UPDATE event_consumers SET accepted_gap_generation = ? WHERE tenant_id = ? AND consumer_id = ?", + [generation, tenantId, consumerId], ) - inserted += 1 - outboxEvents += 1 - outboxBytes += eventBytes - } - eventIds.push(event.id) - } - for (const [tenantId, count] of droppedByTenant) - this.#recordDeliveryGap(tenantId, count, stagedAt) - }) - .immediate() - } catch (error) { - this.#telemetry.record({ operation: "outbox_stage", outcome: "failure" }) - throw error - } - this.#refreshStagedSourceKinds() - this.#telemetry.record({ operation: "outbox_stage", outcome: "success", count: inserted }) - this.#telemetry.record({ operation: "outbox_dedup", outcome: "success", count: deduplicated }) - this.#telemetry.record({ operation: "outbox_stage", outcome: "dropped", count: dropped }) - return { inserted, deduplicated, dropped, eventIds } - } - - deliveryGap(tenantId: string): DeliveryGap { - const statement = this.#db.prepare( - "SELECT generation, dropped_events, last_dropped_at FROM delivery_gaps WHERE tenant_id = ?", - ) - try { - const row = statement.get(tenantId) - return row === null - ? { generation: 0, droppedEvents: 0, lastDroppedAt: null } - : { - generation: asNumber(row.generation), - droppedEvents: asNumber(row.dropped_events), - lastDroppedAt: row.last_dropped_at, - } - } finally { - statement.finalize() - } - } - #recordDeliveryGap(tenantId: string, count: number, at: string): void { - if (count === 0) return - this.#db.run( - `INSERT INTO delivery_gaps (tenant_id, generation, dropped_events, last_dropped_at) VALUES (?, 1, ?, ?) - ON CONFLICT (tenant_id) DO UPDATE SET generation = generation + 1, dropped_events = dropped_events + excluded.dropped_events, last_dropped_at = excluded.last_dropped_at`, - [tenantId, count, at], - ) - } - acceptDeliveryGap(tenantId: string, consumerId: string, generation: number): DeliveryGap { - validateConsumerId(consumerId) - return this.#db - .transaction(() => { - const consumer = this.#consumer(tenantId, consumerId) - if (consumer === null) - throw EventConsumerNotFoundError.create( - `event consumer not found: ${consumerId}`, - consumerId, - ) - const gap = this.deliveryGap(tenantId) - if (!Number.isSafeInteger(generation) || generation < 1 || generation !== gap.generation) - throw EventConsumerConflictError.create( - "delivery gap generation changed; inspect current health before accepting", - consumerId, - ) - this.#db.run( - "UPDATE event_consumers SET accepted_gap_generation = ? WHERE tenant_id = ? AND consumer_id = ?", - [generation, tenantId, consumerId], - ) - return gap - }) - .immediate() - } - /** Operator-authorized loss; the HTTP caller drains admission before invoking this transaction. */ - abandonEvents( - tenantId: string, - eventIds: readonly string[], - ): { readonly abandoned: number; readonly gap: DeliveryGap } { - if (eventIds.length < 1 || eventIds.length > 1000 || new Set(eventIds).size !== eventIds.length) - throw new OutboxAdministrationInvalid({ message: "abandon requires 1–1000 distinct event IDs" }) - const result = this.#db - .transaction(() => { - const lookup = this.#db.prepare( - "SELECT event_id FROM outbox_events WHERE tenant_id = ? AND event_id = ?", - ) - try { - for (const eventId of eventIds) - if (lookup.get(tenantId, eventId) === null) - throw new OutboxAdministrationInvalid({ - message: `unknown event ID for abandonment: ${eventId}`, - }) - } finally { - lookup.finalize() - } - for (const eventId of eventIds) { - this.#db.run("DELETE FROM outbox_ready_events WHERE event_id = ?", [eventId]) - this.#db.run("DELETE FROM outbox_events WHERE tenant_id = ? AND event_id = ?", [ - tenantId, - eventId, - ]) - } - this.#recordDeliveryGap(tenantId, eventIds.length, new Date().toISOString()) - this.#db.run( - "UPDATE event_consumers SET lease_token_hash = NULL, lease_expires_at = NULL, claimed_through_sequence = NULL WHERE tenant_id = ?", - [tenantId], - ) - return { abandoned: eventIds.length, gap: this.deliveryGap(tenantId) } + return gap + }) + .immediate() + }, + catch: consumerFailure, }) - .immediate() - this.#refreshStagedSourceKinds() - this.#telemetry.record({ operation: "outbox_abandon", outcome: "success", count: result.abandoned }) - return result - } - - #refreshStagedSourceKinds(): void { - const statement = this.#db.prepare<{ tenant_id: string; source_kind: string }, []>( - "SELECT DISTINCT tenant_id, source_kind FROM outbox_events WHERE state = 'staged' AND source_kind IS NOT NULL", - ) - try { - this.#stagedSourceKinds = new Set( - statement.all().map((row) => JSON.stringify([row.tenant_id, row.source_kind])), - ) - } finally { - statement.finalize() - } - } - #outboxUsage(): OutboxUsageRow { - const statement = this.#db.prepare( - "SELECT count, bytes FROM outbox_usage WHERE singleton = 1", - ) - try { - const usage = statement.get() - if (usage === null) throw new Error("event outbox usage query returned no row") - return usage - } finally { - statement.finalize() - } - } - hasStagedSourceKind(tenantId: string, sourceKind: string): boolean { - return this.#stagedSourceKinds.has(JSON.stringify([tenantId, sourceKind])) - } - hasStagedSourceOccurrence( - tenantId: string, - sourceKind: string, - source: string, - sourceOccurrenceId: string, - ): boolean { - const row = this.#db - .query( - "SELECT count(*) AS count FROM outbox_events WHERE tenant_id = ? AND source_kind = ? AND source = ? AND source_occurrence_id = ? AND state = 'staged'", - ) - .get(tenantId, sourceKind, source, sourceOccurrenceId) - if (row === null) throw new Error("staged source-occurrence query returned no row") - return asNumber(row.count) > 0 - } - - stagedEventIdsForOccurrence( - tenantId: string, - sourceKind: string, - source: string, - sourceOccurrenceId: string, - sourceFingerprint: string, - ): readonly string[] { - const rows = this.#db - .query( - "SELECT event_id, source_fingerprint FROM outbox_events WHERE tenant_id = ? AND source_kind = ? AND source = ? AND source_occurrence_id = ? AND state = 'staged' ORDER BY sequence", + const abandonEvents: LocalEventingControlStoreApi["abandonEvents"] = (tenantId, eventIds) => + Effect.try({ + try: () => { + if ( + eventIds.length < 1 || + eventIds.length > 1000 || + new Set(eventIds).size !== eventIds.length + ) + throw new OutboxAdministrationInvalid({ + message: "abandon requires 1–1000 distinct event IDs", + }) + return db + .transaction(() => { + const lookup = db.prepare( + "SELECT event_id FROM outbox_events WHERE tenant_id = ? AND event_id = ?", + ) + try { + for (const eventId of eventIds) + if (lookup.get(tenantId, eventId) === null) + throw new OutboxAdministrationInvalid({ + message: `unknown event ID for abandonment: ${eventId}`, + }) + } finally { + lookup.finalize() + } + for (const eventId of eventIds) { + db.run("DELETE FROM outbox_ready_events WHERE event_id = ?", [eventId]) + db.run("DELETE FROM outbox_events WHERE tenant_id = ? AND event_id = ?", [ + tenantId, + eventId, + ]) + } + recordDeliveryGap(tenantId, eventIds.length, now()) + db.run( + "UPDATE event_consumers SET lease_token_hash = NULL, lease_expires_at = NULL, claimed_through_sequence = NULL WHERE tenant_id = ?", + [tenantId], + ) + return { abandoned: eventIds.length, gap: readDeliveryGap(tenantId) } + }) + .immediate() + }, + catch: administrationFailure, + }).pipe( + Effect.tap(() => refreshStagedSourceKinds), + Effect.tap((result) => + observeEventing({ + operation: "outbox_abandon", + outcome: "success", + count: result.abandoned, + }), + ), ) - .all(tenantId, sourceKind, source, sourceOccurrenceId) - for (const row of rows) { - if (row.source_fingerprint === null) - throw new Error(`staged source occurrence has no recovery fingerprint: ${row.event_id}`) - if (row.source_fingerprint !== sourceFingerprint) - throw new Error(`staged source occurrence collision: ${sourceOccurrenceId}`) - } - return rows.map(({ event_id }) => event_id) - } - markReady(eventIds: readonly string[], readyAt = new Date().toISOString()): void { - let markedReady = 0 - try { - this.#db - .transaction(() => { - for (const eventId of eventIds) { - const row = this.#db - .query, [string]>( - "SELECT state FROM outbox_events WHERE event_id = ?", - ) - .get(eventId) - if (!row) throw new Error(`cannot mark unknown event ready: ${eventId}`) - if (row.state === "ready") continue - this.#db.run("INSERT INTO outbox_ready_events (event_id, ready_at) VALUES (?, ?)", [ - eventId, - readyAt, - ]) - this.#db.run( - "UPDATE outbox_events SET state = 'ready', ready_at = ? WHERE event_id = ? AND state = 'staged'", - [readyAt, eventId], + const hasStagedSourceKind: LocalEventingControlStoreApi["hasStagedSourceKind"] = ( + tenantId, + sourceKind, + ) => Effect.sync(() => stagedSourceKinds.has(JSON.stringify([tenantId, sourceKind]))) + + const hasStagedSourceOccurrence: LocalEventingControlStoreApi["hasStagedSourceOccurrence"] = ( + tenantId, + sourceKind, + source, + sourceOccurrenceId, + ) => + Effect.try({ + try: () => { + const row = db + .query( + "SELECT count(*) AS count FROM outbox_events WHERE tenant_id = ? AND source_kind = ? AND source = ? AND source_occurrence_id = ? AND state = 'staged'", ) - markedReady += 1 - } - }) - .immediate() - } catch (error) { - this.#telemetry.record({ operation: "outbox_ready", outcome: "failure" }) - throw error - } - this.#refreshStagedSourceKinds() - this.#telemetry.record({ operation: "outbox_ready", outcome: "success", count: markedReady }) - } + .get(tenantId, sourceKind, source, sourceOccurrenceId) + if (row === null) throw storeError("staged source-occurrence query returned no row") + return asNumber(row.count) > 0 + }, + catch: storeFailure, + }) - #listOutbox(state: "ready" | "staged", limit = 100, after = 0): EventingOutboxPage { - if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1_000) - throw new Error("outbox-event limit must be between 1 and 1000") - if (!Number.isSafeInteger(after) || after < 0) - throw new Error("outbox cursor must be a non-negative safe integer") - const rows = - state === "ready" - ? this.#db - .query( - `SELECT readiness.sequence, event.event_json, event.staged_at, readiness.ready_at - FROM outbox_ready_events AS readiness - INNER JOIN outbox_events AS event ON event.event_id = readiness.event_id - WHERE event.state = 'ready' AND readiness.sequence > ? - ORDER BY readiness.sequence - LIMIT ?`, - ) - .all(after, limit + 1) - : this.#db - .query( - `SELECT sequence, event_json, staged_at, ready_at - FROM outbox_events - WHERE state = 'staged' AND sequence > ? - ORDER BY sequence - LIMIT ?`, + const stagedEventIdsForOccurrence: LocalEventingControlStoreApi["stagedEventIdsForOccurrence"] = ( + tenantId, + sourceKind, + source, + sourceOccurrenceId, + sourceFingerprint, + ) => + Effect.try({ + try: () => { + const rows = db + .query( + "SELECT event_id, source_fingerprint FROM outbox_events WHERE tenant_id = ? AND source_kind = ? AND source = ? AND source_occurrence_id = ? AND state = 'staged' ORDER BY sequence", ) - .all(after, limit + 1) - const hasMore = rows.length > limit - const pageRows = hasMore ? rows.slice(0, limit) : rows - const page = pageRows.map(({ sequence, event_json, staged_at, ready_at }) => ({ - sequence: asNumber(sequence), - event: decodeEvent(event_json), - stagedAt: staged_at, - readyAt: ready_at, - })) - return { - events: page, - nextCursor: hasMore ? (page.at(-1)?.sequence ?? null) : null, - } - } - - listReady(limit = 100, after = 0): EventingOutboxPage { - return this.#listOutbox("ready", limit, after) - } + .all(tenantId, sourceKind, source, sourceOccurrenceId) + for (const row of rows) { + if (row.source_fingerprint === null) + throw storeError( + `staged source occurrence has no recovery fingerprint: ${row.event_id}`, + ) + if (row.source_fingerprint !== sourceFingerprint) + throw storeError(`staged source occurrence collision: ${sourceOccurrenceId}`) + } + return rows.map(({ event_id }) => event_id) + }, + catch: storeFailure, + }) - listStaged(limit = 100, after = 0): EventingOutboxPage { - return this.#listOutbox("staged", limit, after) - } + const markReady: LocalEventingControlStoreApi["markReady"] = (eventIds, readyAt = now()) => + Effect.try({ + try: () => + db + .transaction(() => { + let markedReady = 0 + for (const eventId of eventIds) { + const row = db + .query, [string]>( + "SELECT state FROM outbox_events WHERE event_id = ?", + ) + .get(eventId) + if (!row) throw storeError(`cannot mark unknown event ready: ${eventId}`) + if (row.state === "ready") continue + db.run("INSERT INTO outbox_ready_events (event_id, ready_at) VALUES (?, ?)", [ + eventId, + readyAt, + ]) + db.run( + "UPDATE outbox_events SET state = 'ready', ready_at = ? WHERE event_id = ? AND state = 'staged'", + [readyAt, eventId], + ) + markedReady += 1 + } + return markedReady + }) + .immediate(), + catch: storeFailure, + }).pipe( + Effect.tapError(() => observeEventing({ operation: "outbox_ready", outcome: "failure" })), + Effect.tap(() => refreshStagedSourceKinds), + Effect.flatMap((markedReady) => + observeEventing({ operation: "outbox_ready", outcome: "success", count: markedReady }), + ), + ) - listConsumers(tenantId: string): readonly EventConsumer[] { - return this.#db - .query( - `SELECT consumer_id, tenant_id, active, last_acked_sequence, accepted_gap_generation, lease_token_hash, + const listConsumers: LocalEventingControlStoreApi["listConsumers"] = (tenantId) => + Effect.try({ + try: () => + db + .query( + `SELECT consumer_id, tenant_id, active, last_acked_sequence, accepted_gap_generation, lease_token_hash, lease_expires_at, claimed_through_sequence, registered_at, disabled_at FROM event_consumers WHERE tenant_id = ? ORDER BY consumer_id`, - ) - .all(tenantId) - .map(decodeConsumer) - } + ) + .all(tenantId) + .map(decodeConsumer), + catch: storeFailure, + }) - registerConsumer( - tenantId: string, - consumerId: string, - startAt: EventConsumerStart, - registeredAt = new Date().toISOString(), - ): EventConsumer { - validateConsumerId(consumerId) - if (startAt !== "beginning" && startAt !== "latest") - throw EventConsumerInputError.create("startAt must be beginning or latest") - canonicalInstant(registeredAt, "event consumer registeredAt") - return this.#db - .transaction(() => { - const existing = this.#consumer(tenantId, consumerId) - if (existing) - throw EventConsumerConflictError.create( - `event consumer already exists: ${consumerId}`, - consumerId, - ) - const boundary = this.#db - .query( - startAt === "latest" - ? `SELECT max(readiness.sequence) AS sequence + const registerConsumer: LocalEventingControlStoreApi["registerConsumer"] = ( + tenantId, + consumerId, + startAt, + registeredAt = now(), + ) => + Effect.try({ + try: () => { + validateConsumerId(consumerId) + if (startAt !== "beginning" && startAt !== "latest") + throw EventConsumerInputError.create("startAt must be beginning or latest") + canonicalInstant(registeredAt, "event consumer registeredAt") + return db + .transaction(() => { + const existing = consumerRow(tenantId, consumerId) + if (existing) + throw EventConsumerConflictError.create( + `event consumer already exists: ${consumerId}`, + consumerId, + ) + const boundary = db + .query( + startAt === "latest" + ? `SELECT max(readiness.sequence) AS sequence FROM outbox_ready_events AS readiness INNER JOIN outbox_events AS event ON event.event_id = readiness.event_id WHERE event.tenant_id = ?` - : `SELECT min(readiness.sequence) AS sequence + : `SELECT min(readiness.sequence) AS sequence FROM outbox_ready_events AS readiness INNER JOIN outbox_events AS event ON event.event_id = readiness.event_id WHERE event.tenant_id = ?`, - ) - .get(tenantId) - const sequence = boundary?.sequence == null ? 0 : asNumber(boundary.sequence) - const lastAcknowledged = startAt === "beginning" ? Math.max(0, sequence - 1) : sequence - this.#db.run( - "INSERT INTO event_consumers (consumer_id, tenant_id, active, last_acked_sequence, registered_at) VALUES (?, ?, 1, ?, ?)", - [consumerId, tenantId, lastAcknowledged, registeredAt], - ) - if (startAt === "latest") - this.#db.run( - "UPDATE event_consumers SET accepted_gap_generation = ? WHERE tenant_id = ? AND consumer_id = ?", - [this.deliveryGap(tenantId).generation, tenantId, consumerId], - ) - const updated = this.#consumer(tenantId, consumerId) - if (updated === null) - throw EventConsumerNotFoundError.create( - `event consumer not found: ${consumerId}`, - consumerId, - ) - return decodeConsumer(updated) + ) + .get(tenantId) + const sequence = boundary?.sequence == null ? 0 : asNumber(boundary.sequence) + const lastAcknowledged = + startAt === "beginning" ? Math.max(0, sequence - 1) : sequence + db.run( + "INSERT INTO event_consumers (consumer_id, tenant_id, active, last_acked_sequence, registered_at) VALUES (?, ?, 1, ?, ?)", + [consumerId, tenantId, lastAcknowledged, registeredAt], + ) + if (startAt === "latest") + db.run( + "UPDATE event_consumers SET accepted_gap_generation = ? WHERE tenant_id = ? AND consumer_id = ?", + [readDeliveryGap(tenantId).generation, tenantId, consumerId], + ) + const updated = consumerRow(tenantId, consumerId) + if (updated === null) + throw EventConsumerNotFoundError.create( + `event consumer not found: ${consumerId}`, + consumerId, + ) + return decodeConsumer(updated) + }) + .immediate() + }, + catch: consumerFailure, }) - .immediate() - } - disableConsumer( - tenantId: string, - consumerId: string, - disabledAt = new Date().toISOString(), - ): EventConsumer { - validateConsumerId(consumerId) - canonicalInstant(disabledAt, "event consumer disabledAt") - return this.#db - .transaction(() => { - const existing = this.#consumer(tenantId, consumerId) - if (!existing) - throw EventConsumerNotFoundError.create( - `unknown event consumer: ${consumerId}`, - consumerId, - ) - if (asNumber(existing.active) === 0) return decodeConsumer(existing) - this.#db.run( - `UPDATE event_consumers + const disableConsumer: LocalEventingControlStoreApi["disableConsumer"] = ( + tenantId, + consumerId, + disabledAt = now(), + ) => + Effect.try({ + try: () => { + validateConsumerId(consumerId) + canonicalInstant(disabledAt, "event consumer disabledAt") + return db + .transaction(() => { + const existing = consumerRow(tenantId, consumerId) + if (!existing) + throw EventConsumerNotFoundError.create( + `unknown event consumer: ${consumerId}`, + consumerId, + ) + if (asNumber(existing.active) === 0) return decodeConsumer(existing) + db.run( + `UPDATE event_consumers SET active = 0, lease_token_hash = NULL, lease_expires_at = NULL, claimed_through_sequence = NULL, disabled_at = ? WHERE tenant_id = ? AND consumer_id = ?`, - [disabledAt, tenantId, consumerId], - ) - this.#pruneAcknowledgedReady(tenantId) - const updated = this.#consumer(tenantId, consumerId) - if (updated === null) - throw EventConsumerNotFoundError.create( - `event consumer not found: ${consumerId}`, - consumerId, - ) - return decodeConsumer(updated) + [disabledAt, tenantId, consumerId], + ) + pruneAcknowledgedReady(tenantId) + const updated = consumerRow(tenantId, consumerId) + if (updated === null) + throw EventConsumerNotFoundError.create( + `event consumer not found: ${consumerId}`, + consumerId, + ) + return decodeConsumer(updated) + }) + .immediate() + }, + catch: consumerFailure, }) - .immediate() - } - claimReady( - tenantId: string, - consumerId: string, - limit: number, - leaseSeconds: number, - now = new Date().toISOString(), - ): EventConsumerClaim { - let reclaimedExpiredLease = false - let lag = 0 - try { - validateConsumerId(consumerId) - if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1_000) - throw EventConsumerInputError.create("claim limit must be between 1 and 1000") - if (!Number.isSafeInteger(leaseSeconds) || leaseSeconds < 5 || leaseSeconds > 300) - throw EventConsumerInputError.create("leaseSeconds must be between 5 and 300") - const nowMilliseconds = canonicalInstant(now, "claim time") - const claim = this.#db - .transaction(() => { - const consumer = this.#consumer(tenantId, consumerId) - if (!consumer) - throw EventConsumerNotFoundError.create( - `unknown event consumer: ${consumerId}`, - consumerId, - ) - const gap = this.deliveryGap(tenantId) - if (gap.generation > asNumber(consumer.accepted_gap_generation)) - throw new EventConsumerDeliveryGapError({ - message: - "Event delivery has a gap; an operator must acknowledge the reported generation before claiming more events", - consumerId, - generation: gap.generation, - droppedEvents: gap.droppedEvents, - }) - if (asNumber(consumer.active) === 0) - throw EventConsumerConflictError.create( - `event consumer is disabled: ${consumerId}`, - consumerId, - ) - if ( - consumer.lease_expires_at !== null && - canonicalInstant(consumer.lease_expires_at, "event consumer leaseExpiresAt") > - nowMilliseconds - ) - throw EventConsumerLeaseError.create( - `event consumer already has an active lease: ${consumerId}`, - consumerId, - consumer.lease_expires_at, - ) - if (consumer.lease_expires_at !== null) reclaimedExpiredLease = true - lag = this.#consumerLag(tenantId, asNumber(consumer.last_acked_sequence)) + const observeConsumerFailure = ( + operation: "consumer_claim" | "consumer_ack", + error: EventConsumerFailure, + ) => + Effect.all( + [ + observeEventing({ operation, outcome: "failure" }), + error._tag === "@maple/cli/eventing/EventConsumerLeaseConflict" + ? observeEventing({ operation: "consumer_lease", outcome: "failure" }) + : Effect.void, + ], + { discard: true }, + ) + + const claimReady: LocalEventingControlStoreApi["claimReady"] = ( + tenantId, + consumerId, + limit, + leaseSeconds, + claimedAt = now(), + ) => + Effect.try({ + try: () => { + validateConsumerId(consumerId) + if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1_000) + throw EventConsumerInputError.create("claim limit must be between 1 and 1000") + if (!Number.isSafeInteger(leaseSeconds) || leaseSeconds < 5 || leaseSeconds > 300) + throw EventConsumerInputError.create("leaseSeconds must be between 5 and 300") + const nowMilliseconds = canonicalInstant(claimedAt, "claim time") + return db + .transaction(() => { + const consumer = consumerRow(tenantId, consumerId) + if (!consumer) + throw EventConsumerNotFoundError.create( + `unknown event consumer: ${consumerId}`, + consumerId, + ) + const gap = readDeliveryGap(tenantId) + if (gap.generation > asNumber(consumer.accepted_gap_generation)) + throw new EventConsumerDeliveryGapError({ + message: + "Event delivery has a gap; an operator must acknowledge the reported generation before claiming more events", + consumerId, + generation: gap.generation, + droppedEvents: gap.droppedEvents, + }) + if (asNumber(consumer.active) === 0) + throw EventConsumerConflictError.create( + `event consumer is disabled: ${consumerId}`, + consumerId, + ) + if ( + consumer.lease_expires_at !== null && + canonicalInstant(consumer.lease_expires_at, "event consumer leaseExpiresAt") > + nowMilliseconds + ) + throw EventConsumerLeaseError.create( + `event consumer already has an active lease: ${consumerId}`, + consumerId, + consumer.lease_expires_at, + ) + const reclaimedExpiredLease = consumer.lease_expires_at !== null + const lag = consumerLag(tenantId, asNumber(consumer.last_acked_sequence)) - const rows = this.#db - .query( - `SELECT readiness.sequence, event.event_json, event.staged_at, readiness.ready_at + const rows = db + .query( + `SELECT readiness.sequence, event.event_json, event.staged_at, readiness.ready_at FROM outbox_ready_events AS readiness INNER JOIN outbox_events AS event ON event.event_id = readiness.event_id WHERE event.tenant_id = ? AND event.state = 'ready' AND readiness.sequence > ? ORDER BY readiness.sequence LIMIT ?`, - ) - .all(tenantId, asNumber(consumer.last_acked_sequence), limit) - if (rows.length === 0) { - this.#db.run( - "UPDATE event_consumers SET lease_token_hash = NULL, lease_expires_at = NULL, claimed_through_sequence = NULL WHERE tenant_id = ? AND consumer_id = ?", - [tenantId, consumerId], - ) - return { - consumerId, - leaseToken: null, - leaseExpiresAt: null, - throughSequence: null, - events: [], - } - } + ) + .all(tenantId, asNumber(consumer.last_acked_sequence), limit) + if (rows.length === 0) { + db.run( + "UPDATE event_consumers SET lease_token_hash = NULL, lease_expires_at = NULL, claimed_through_sequence = NULL WHERE tenant_id = ? AND consumer_id = ?", + [tenantId, consumerId], + ) + const claim: EventConsumerClaim = { + consumerId, + leaseToken: null, + leaseExpiresAt: null, + throughSequence: null, + events: [], + } + return { claim, lag, reclaimedExpiredLease } + } - const leaseToken = randomBytes(32).toString("hex") - const leaseExpiresAt = new Date(nowMilliseconds + leaseSeconds * 1_000).toISOString() - const last = rows.at(-1) - if (last === undefined) throw EventConsumerConflictError.create("empty claim", consumerId) - const throughSequence = asNumber(last.sequence) - this.#db.run( - `UPDATE event_consumers + const leaseToken = randomBytes(32).toString("hex") + const leaseExpiresAt = new Date( + nowMilliseconds + leaseSeconds * 1_000, + ).toISOString() + const last = rows.at(-1) + if (last === undefined) + throw EventConsumerConflictError.create("empty claim", consumerId) + const throughSequence = asNumber(last.sequence) + db.run( + `UPDATE event_consumers SET lease_token_hash = ?, lease_expires_at = ?, claimed_through_sequence = ? WHERE tenant_id = ? AND consumer_id = ?`, - [tokenHash(leaseToken), leaseExpiresAt, throughSequence, tenantId, consumerId], - ) - return { - consumerId, - leaseToken, - leaseExpiresAt, - throughSequence, - events: rows.map(({ sequence, event_json, staged_at, ready_at }) => ({ - sequence: asNumber(sequence), - event: decodeEvent(event_json), - stagedAt: staged_at, - readyAt: ready_at, - })), - } - }) - .immediate() - this.#telemetry.record({ - operation: "consumer_claim", - outcome: claim.events.length === 0 ? "empty" : "success", - count: Math.max(1, claim.events.length), - }) - this.#telemetry.record({ operation: "consumer_lag", outcome: "observed", lag }) - if (reclaimedExpiredLease) - this.#telemetry.record({ operation: "consumer_lease", outcome: "reclaimed" }) - return claim - } catch (error) { - this.#telemetry.record({ operation: "consumer_claim", outcome: "failure" }) - if (Schema.is(EventConsumerLeaseError)(error)) - this.#telemetry.record({ operation: "consumer_lease", outcome: "failure" }) - throw error - } - } + [ + tokenHash(leaseToken), + leaseExpiresAt, + throughSequence, + tenantId, + consumerId, + ], + ) + const claim: EventConsumerClaim = { + consumerId, + leaseToken, + leaseExpiresAt, + throughSequence, + events: rows.map(({ sequence, event_json, staged_at, ready_at }) => ({ + sequence: asNumber(sequence), + event: decodeEvent(event_json), + stagedAt: staged_at, + readyAt: ready_at, + })), + } + return { claim, lag, reclaimedExpiredLease } + }) + .immediate() + }, + catch: consumerFailure, + }).pipe( + Effect.tapError((error) => observeConsumerFailure("consumer_claim", error)), + Effect.tap(({ claim, lag, reclaimedExpiredLease }) => + Effect.all( + [ + observeEventing({ + operation: "consumer_claim", + outcome: claim.events.length === 0 ? "empty" : "success", + count: Math.max(1, claim.events.length), + }), + observeEventing({ operation: "consumer_lag", outcome: "observed", lag }), + reclaimedExpiredLease + ? observeEventing({ operation: "consumer_lease", outcome: "reclaimed" }) + : Effect.void, + ], + { discard: true }, + ), + ), + Effect.map(({ claim }) => claim), + ) - acknowledgeClaim( - tenantId: string, - consumerId: string, - leaseToken: string, - throughSequence: number, - now = new Date().toISOString(), - ): EventConsumerAcknowledgement { - try { - validateConsumerId(consumerId) - if (!Number.isSafeInteger(throughSequence) || throughSequence < 1) - throw EventConsumerInputError.create("throughSequence must be a positive safe integer") - const nowMilliseconds = canonicalInstant(now, "acknowledgement time") - const acknowledgement = this.#db - .transaction(() => { - const consumer = this.#consumer(tenantId, consumerId) - if (!consumer) - throw EventConsumerNotFoundError.create( - `unknown event consumer: ${consumerId}`, - consumerId, - ) - if (asNumber(consumer.active) === 0) - throw EventConsumerConflictError.create( - `event consumer is disabled: ${consumerId}`, - consumerId, - ) - if ( - consumer.lease_token_hash === null || - consumer.lease_expires_at === null || - consumer.claimed_through_sequence === null - ) - throw EventConsumerLeaseError.create( - `event consumer has no active lease: ${consumerId}`, - consumerId, - consumer.lease_expires_at, - ) - if ( - canonicalInstant(consumer.lease_expires_at, "event consumer leaseExpiresAt") <= - nowMilliseconds - ) - throw EventConsumerLeaseError.create( - `event consumer lease has expired: ${consumerId}`, - consumerId, - consumer.lease_expires_at, - ) - if (!tokenHashMatches(consumer.lease_token_hash, leaseToken)) - throw EventConsumerLeaseError.create( - "event consumer lease token does not match", - consumerId, - consumer.lease_expires_at, - ) - const claimedThrough = asNumber(consumer.claimed_through_sequence) - if (throughSequence !== claimedThrough) - throw EventConsumerLeaseError.create( - `acknowledgement must cover the complete claimed batch through sequence ${claimedThrough}`, - consumerId, - consumer.lease_expires_at, + const acknowledgeClaim: LocalEventingControlStoreApi["acknowledgeClaim"] = ( + tenantId, + consumerId, + leaseToken, + throughSequence, + acknowledgedAt = now(), + ) => + Effect.try({ + try: () => { + validateConsumerId(consumerId) + if (!Number.isSafeInteger(throughSequence) || throughSequence < 1) + throw EventConsumerInputError.create( + "throughSequence must be a positive safe integer", ) - this.#db.run( - `UPDATE event_consumers + const nowMilliseconds = canonicalInstant(acknowledgedAt, "acknowledgement time") + return db + .transaction((): EventConsumerAcknowledgement => { + const consumer = consumerRow(tenantId, consumerId) + if (!consumer) + throw EventConsumerNotFoundError.create( + `unknown event consumer: ${consumerId}`, + consumerId, + ) + if (asNumber(consumer.active) === 0) + throw EventConsumerConflictError.create( + `event consumer is disabled: ${consumerId}`, + consumerId, + ) + if ( + consumer.lease_token_hash === null || + consumer.lease_expires_at === null || + consumer.claimed_through_sequence === null + ) + throw EventConsumerLeaseError.create( + `event consumer has no active lease: ${consumerId}`, + consumerId, + consumer.lease_expires_at, + ) + if ( + canonicalInstant( + consumer.lease_expires_at, + "event consumer leaseExpiresAt", + ) <= nowMilliseconds + ) + throw EventConsumerLeaseError.create( + `event consumer lease has expired: ${consumerId}`, + consumerId, + consumer.lease_expires_at, + ) + if (!tokenHashMatches(consumer.lease_token_hash, leaseToken)) + throw EventConsumerLeaseError.create( + "event consumer lease token does not match", + consumerId, + consumer.lease_expires_at, + ) + const claimedThrough = asNumber(consumer.claimed_through_sequence) + if (throughSequence !== claimedThrough) + throw EventConsumerLeaseError.create( + `acknowledgement must cover the complete claimed batch through sequence ${claimedThrough}`, + consumerId, + consumer.lease_expires_at, + ) + db.run( + `UPDATE event_consumers SET last_acked_sequence = ?, lease_token_hash = NULL, lease_expires_at = NULL, claimed_through_sequence = NULL WHERE tenant_id = ? AND consumer_id = ?`, - [throughSequence, tenantId, consumerId], - ) - return { - consumerId, - acknowledgedThrough: throughSequence, - prunedEvents: this.#pruneAcknowledgedReady(tenantId), - } - }) - .immediate() - this.#telemetry.record({ operation: "consumer_ack", outcome: "success" }) - this.#telemetry.record({ - operation: "consumer_lag", - outcome: "observed", - lag: this.#consumerLag(tenantId, acknowledgement.acknowledgedThrough), - }) - return acknowledgement - } catch (error) { - this.#telemetry.record({ operation: "consumer_ack", outcome: "failure" }) - if (Schema.is(EventConsumerLeaseError)(error)) - this.#telemetry.record({ operation: "consumer_lease", outcome: "failure" }) - throw error - } - } - - #consumerLag(tenantId: string, lastAcknowledgedSequence: number): number { - const latest = this.#db - .query( - `SELECT max(readiness.sequence) AS sequence - FROM outbox_ready_events AS readiness - INNER JOIN outbox_events AS event ON event.event_id = readiness.event_id - WHERE event.tenant_id = ? AND event.state = 'ready'`, - ) - .get(tenantId) - return Math.max( - 0, - (latest?.sequence == null ? 0 : asNumber(latest.sequence)) - lastAcknowledgedSequence, - ) - } - - #consumer(tenantId: string, consumerId: string): ConsumerRow | null { - return this.#db - .query( - `SELECT consumer_id, tenant_id, active, last_acked_sequence, accepted_gap_generation, lease_token_hash, - lease_expires_at, claimed_through_sequence, registered_at, disabled_at - FROM event_consumers - WHERE tenant_id = ? AND consumer_id = ?`, - ) - .get(tenantId, consumerId) - } - - #pruneAcknowledgedReady(tenantId: string): number { - const boundary = this.#db - .query( - "SELECT min(last_acked_sequence) AS sequence FROM event_consumers WHERE tenant_id = ? AND active = 1", - ) - .get(tenantId) - if (boundary?.sequence == null) return 0 - const rows = this.#db - .query( - `SELECT readiness.event_id - FROM outbox_ready_events AS readiness - INNER JOIN outbox_events AS event ON event.event_id = readiness.event_id - WHERE event.tenant_id = ? AND readiness.sequence <= ? - ORDER BY readiness.sequence`, + [throughSequence, tenantId, consumerId], + ) + return { + consumerId, + acknowledgedThrough: throughSequence, + prunedEvents: pruneAcknowledgedReady(tenantId), + } + }) + .immediate() + }, + catch: consumerFailure, + }).pipe( + Effect.flatMap((acknowledgement) => + Effect.try({ + try: () => consumerLag(tenantId, acknowledgement.acknowledgedThrough), + catch: storeFailure, + }).pipe( + Effect.tap((lag) => + Effect.all( + [ + observeEventing({ operation: "consumer_ack", outcome: "success" }), + observeEventing({ operation: "consumer_lag", outcome: "observed", lag }), + ], + { discard: true }, + ), + ), + Effect.as(acknowledgement), + ), + ), + Effect.tapError((error) => observeConsumerFailure("consumer_ack", error)), ) - .all(tenantId, asNumber(boundary.sequence)) - const pruneCount = Math.max(0, rows.length - this.#limits.retainAcknowledgedReadyEvents) - for (const { event_id } of rows.slice(0, pruneCount)) { - this.#db.run("DELETE FROM outbox_ready_events WHERE event_id = ?", [event_id]) - this.#db.run("DELETE FROM outbox_events WHERE event_id = ? AND state = 'ready'", [event_id]) - } - return pruneCount - } - outboxCapacity(): LocalEventingControlLimits & { - readonly currentEvents: number - readonly currentBytes: number - } { - const usage = this.#outboxUsage() - if (!usage) throw new Error("event outbox usage query returned no row") - return { - ...this.#limits, - currentEvents: asNumber(usage.count), - currentBytes: asNumber(usage.bytes), - } - } + const outboxCapacity: LocalEventingControlStoreApi["outboxCapacity"] = Effect.try({ + try: () => { + const usage = outboxUsage() + return { + ...limits, + currentEvents: asNumber(usage.count), + currentBytes: asNumber(usage.bytes), + } + }, + catch: storeFailure, + }) - recordProjectionFailures( - tenantId: string, - failures: readonly ProjectionFailure[], - createdAt = new Date().toISOString(), - ): void { - this.#db - .transaction(() => { - for (const failure of failures) - this.#db.run( - "INSERT OR IGNORE INTO projection_failures (tenant_id, projection_id, projection_revision, occurrence_id, message, created_at) VALUES (?, ?, ?, ?, ?, ?)", - [ - tenantId, - failure.projectionId, - failure.projectionRevision, - failure.occurrenceId, - failure.message.slice(0, 4_096), - createdAt, - ], - ) - this.#db.run( - "DELETE FROM projection_failures WHERE tenant_id = ? AND sequence NOT IN (SELECT sequence FROM projection_failures WHERE tenant_id = ? ORDER BY sequence DESC LIMIT ?)", - [tenantId, tenantId, MAX_FAILURES_PER_TENANT], - ) + const recordProjectionFailures: LocalEventingControlStoreApi["recordProjectionFailures"] = ( + tenantId, + failures, + createdAt = now(), + ) => + Effect.try({ + try: () => + db + .transaction(() => { + for (const failure of failures) + db.run( + "INSERT OR IGNORE INTO projection_failures (tenant_id, projection_id, projection_revision, occurrence_id, message, created_at) VALUES (?, ?, ?, ?, ?, ?)", + [ + tenantId, + failure.projectionId, + failure.projectionRevision, + failure.occurrenceId, + failure.message.slice(0, 4_096), + createdAt, + ], + ) + db.run( + "DELETE FROM projection_failures WHERE tenant_id = ? AND sequence NOT IN (SELECT sequence FROM projection_failures WHERE tenant_id = ? ORDER BY sequence DESC LIMIT ?)", + [tenantId, tenantId, MAX_FAILURES_PER_TENANT], + ) + }) + .immediate(), + catch: storeFailure, }) - .immediate() - } - - validate(): EventingControlSnapshotValidation { - return validateOpenDatabase(this.#db) - } - captureSnapshot(): Uint8Array { - checkpointWal(this.#db) - return this.#db.serialize() - } - - static async writeSnapshot(path: string, bytes: Uint8Array): Promise { - await durableWrite(path, bytes) - return LocalEventingControlStore.validateSnapshot(path) - } - - async backupTo(path: string): Promise { - return LocalEventingControlStore.writeSnapshot(path, this.captureSnapshot()) - } - - static validateSnapshot(path: string): EventingControlSnapshotValidation { - assertRealDatabaseFile(path) - if (!existsSync(path)) throw new Error(`eventing control snapshot is missing: ${path}`) - const uri = `${pathToFileURL(path).href}?immutable=1` - const db = new Database(uri, sqliteConstants.SQLITE_OPEN_READONLY | sqliteConstants.SQLITE_OPEN_URI) - try { - configure(db) - return validateOpenDatabase(db) - } finally { - db.close(true) - } - } + return { + path, + saveProjection, + loadEnabledProjections, + stageEvents, + deliveryGap, + acceptDeliveryGap, + abandonEvents, + hasStagedSourceKind, + hasStagedSourceOccurrence, + stagedEventIdsForOccurrence, + markReady, + listReady: (limit, after) => listOutbox("ready", limit, after), + listStaged: (limit, after) => listOutbox("staged", limit, after), + listConsumers, + registerConsumer, + disableConsumer, + claimReady, + acknowledgeClaim, + outboxCapacity, + recordProjectionFailures, + validate: Effect.try({ try: () => validateOpenDatabase(db), catch: storeFailure }), + captureSnapshot, + backupTo: (target) => + Effect.flatMap(captureSnapshot, (bytes) => writeControlSnapshot(target, bytes)), + } satisfies LocalEventingControlStoreApi + }) + + static readonly layer = Layer.effect(this, this.make) +} - static async restoreSnapshot(snapshotPath: string, dataDir: string): Promise { - LocalEventingControlStore.validateSnapshot(snapshotPath) - const stagingDataDir = mkdtempSync( - join(dirname(resolve(dataDir)), ".maple-eventing-control-restore-"), +/** Opens a store outside a layer graph (checkpoint restore, tests); it closes with the scope. */ +export const openControlStore = ( + dataDir: string, + limits?: LocalEventingControlLimits, +): Effect.Effect => + LocalEventingControlStore.make.pipe( + Effect.provideService( + LocalEventingControlConfig, + limits === undefined ? { dataDir } : { dataDir, limits }, + ), + ) + +export const validateControlSnapshot = ( + path: string, +): Effect.Effect => + Effect.try({ + try: () => { + assertRealDatabaseFile(path) + if (!existsSync(path)) throw storeError(`eventing control snapshot is missing: ${path}`) + const uri = `${pathToFileURL(path).href}?immutable=1` + const db = new Database( + uri, + sqliteConstants.SQLITE_OPEN_READONLY | sqliteConstants.SQLITE_OPEN_URI, + ) + try { + configure(db) + return validateOpenDatabase(db) + } finally { + db.close(true) + } + }, + catch: storeFailure, + }) + +export const writeControlSnapshot = ( + path: string, + bytes: Uint8Array, +): Effect.Effect => + Effect.tryPromise({ try: () => durableWrite(path, bytes), catch: storeFailure }).pipe( + Effect.andThen(validateControlSnapshot(path)), + ) + +/** Round-trips the snapshot through a staging store so the target is written by a clean close. */ +export const restoreControlSnapshot = ( + snapshotPath: string, + dataDir: string, +): Effect.Effect => + Effect.gen(function* () { + yield* validateControlSnapshot(snapshotPath) + const stagingDataDir = yield* Effect.acquireRelease( + Effect.try({ + try: () => mkdtempSync(join(dirname(resolve(dataDir)), ".maple-eventing-control-restore-")), + catch: storeFailure, + }), + (directory) => Effect.sync(() => rmSync(directory, { recursive: true, force: true })), ) - let restored: LocalEventingControlStore | undefined - try { - await durableWrite(eventingControlPath(stagingDataDir), readFileSync(snapshotPath)) - restored = await LocalEventingControlStore.open(stagingDataDir) - await restored.backupTo(eventingControlPath(dataDir)) - } finally { - restored?.close() - rmSync(stagingDataDir, { recursive: true, force: true }) - } - } -} + yield* Effect.tryPromise({ + try: () => durableWrite(eventingControlPath(stagingDataDir), readFileSync(snapshotPath)), + catch: storeFailure, + }) + const restored = yield* openControlStore(stagingDataDir) + yield* restored.backupTo(eventingControlPath(dataDir)) + }).pipe(Effect.scoped) diff --git a/apps/cli/src/server/eventing/otlp.ts b/apps/cli/src/server/eventing/otlp.ts index 667cdab59..a8acdba6d 100644 --- a/apps/cli/src/server/eventing/otlp.ts +++ b/apps/cli/src/server/eventing/otlp.ts @@ -6,10 +6,9 @@ import { type NormalizedSignal, type SignalFieldCatalogEntry, type SignalScalar, - type SignalSourceAdapter, type SignalSourceDefinition, } from "@maple/eventing-core" -import { Result, Schema } from "effect" +import { Effect, Result, Schema } from "effect" import { OtlpFieldError, spanIdHex, traceIdHex, type AnyValue, type KeyValue } from "../otlp/encode" const NumberOrString = Schema.Union([Schema.String, Schema.Number]) @@ -70,11 +69,12 @@ const LogsRequestSchema = Schema.Struct({ ), ), }) -const decodeLogsRequest = (request: unknown) => { - const decoded = Schema.decodeUnknownResult(LogsRequestSchema)(request ?? {}) - if (Result.isFailure(decoded)) throw new OtlpFieldError(`invalid OTLP logs: ${decoded.failure.message}`) - return decoded.success -} +const isOtlpFieldError = Schema.is(OtlpFieldError) + +const decodeLogsRequest = (request: unknown): Effect.Effect => + Schema.decodeUnknownEffect(LogsRequestSchema)(request ?? {}).pipe( + Effect.mapError((error) => new OtlpFieldError({ message: `invalid OTLP logs: ${error.message}` })), + ) const MAX_ATTRIBUTES = 256 const MAX_STRING_BYTES = 16 * 1024 @@ -147,20 +147,21 @@ interface ValueBudget { const assertStringBound = (value: string, label: string): string => { if (Buffer.byteLength(value, "utf8") > MAX_STRING_BYTES) - throw new OtlpFieldError(`${label} exceeds ${MAX_STRING_BYTES} UTF-8 bytes`) + throw new OtlpFieldError({ message: `${label} exceeds ${MAX_STRING_BYTES} UTF-8 bytes` }) return value } const int64 = (value: string | number, label: string): string => { if (typeof value === "number" && !Number.isSafeInteger(value)) - throw new OtlpFieldError( - `${label} must encode int64 as a decimal string when outside safe integer range`, - ) + throw new OtlpFieldError({ + message: `${label} must encode int64 as a decimal string when outside safe integer range`, + }) const decimal = String(value) - if (!/^-?(?:0|[1-9][0-9]*)$/.test(decimal)) throw new OtlpFieldError(`${label} is not an int64`) + if (!/^-?(?:0|[1-9][0-9]*)$/.test(decimal)) + throw new OtlpFieldError({ message: `${label} is not an int64` }) const parsed = BigInt(decimal) if (parsed < -(1n << 63n) || parsed > (1n << 63n) - 1n) - throw new OtlpFieldError(`${label} is outside the int64 range`) + throw new OtlpFieldError({ message: `${label} is outside the int64 range` }) return decimal } @@ -171,7 +172,8 @@ const anyValueScalar = (value: AnyValue | undefined, label: string): SignalScala if (value.boolValue !== undefined) return { type: "boolean", value: value.boolValue } if (value.intValue !== undefined) return { type: "int64", value: int64(value.intValue, label) } if (value.doubleValue !== undefined) { - if (!Number.isFinite(value.doubleValue)) throw new OtlpFieldError(`${label} must be finite`) + if (!Number.isFinite(value.doubleValue)) + throw new OtlpFieldError({ message: `${label} must be finite` }) return { type: "float64", value: value.doubleValue } } return null @@ -184,8 +186,9 @@ const anyValueJson = ( budget: ValueBudget = { nodes: 0 }, ): JsonValue | null => { budget.nodes += 1 - if (budget.nodes > MAX_VALUE_NODES) throw new OtlpFieldError(`${label} exceeds value node limit`) - if (depth > MAX_VALUE_DEPTH) throw new OtlpFieldError(`${label} exceeds value depth limit`) + if (budget.nodes > MAX_VALUE_NODES) + throw new OtlpFieldError({ message: `${label} exceeds value node limit` }) + if (depth > MAX_VALUE_DEPTH) throw new OtlpFieldError({ message: `${label} exceeds value depth limit` }) const scalar = anyValueScalar(value, label) if (scalar) return scalar.value if (!value) return null @@ -213,7 +216,7 @@ interface NormalizedAttributes { const attributes = (values: readonly KeyValue[] | undefined, label: string): NormalizedAttributes => { if ((values?.length ?? 0) > MAX_ATTRIBUTES) - throw new OtlpFieldError(`${label} exceeds ${MAX_ATTRIBUTES} attributes`) + throw new OtlpFieldError({ message: `${label} exceeds ${MAX_ATTRIBUTES} attributes` }) const scalars = new Map() const data: Record = Object.create(null) for (const [index, entry] of (values ?? []).entries()) { @@ -228,12 +231,8 @@ const attributes = (values: readonly KeyValue[] | undefined, label: string): Nor const epochNanos = (value: string | number | undefined): bigint | null => { if (value === undefined || value === "" || value === 0 || value === "0") return null - try { - const parsed = BigInt(value) - return parsed >= 0 ? parsed : null - } catch { - return null - } + const parsed = Result.try(() => BigInt(value)) + return Result.isSuccess(parsed) && parsed.success >= 0 ? parsed.success : null } const nanosToTimestamp = (nanos: bigint): string => { @@ -242,7 +241,7 @@ const nanosToTimestamp = (nanos: bigint): string => { const milliseconds = Number(seconds) * 1_000 const date = new Date(milliseconds) if (!Number.isFinite(milliseconds) || Number.isNaN(date.getTime())) - throw new OtlpFieldError("OTLP timestamp is outside the supported date range") + throw new OtlpFieldError({ message: "OTLP timestamp is outside the supported date range" }) return `${date.toISOString().slice(0, 19)}.${fraction.toString().padStart(9, "0")}Z` } @@ -327,12 +326,11 @@ const recoveryIdentity = ( const occurredNanos = epochNanos(log.timeUnixNano) ?? epochNanos(log.observedTimeUnixNano) let occurredAt: string | null = null - if (occurredNanos !== null) - try { - occurredAt = nanosToTimestamp(occurredNanos) - } catch (error) { - if (!(error instanceof OtlpFieldError)) throw error - } + if (occurredNanos !== null) { + const timestamp = Result.try(() => nanosToTimestamp(occurredNanos)) + if (Result.isSuccess(timestamp)) occurredAt = timestamp.success + else if (!isOtlpFieldError(timestamp.failure)) throw timestamp.failure + } return { sourceKind: "otel.log", source, tenantId, occurrenceId, occurredAt } } @@ -375,7 +373,7 @@ const normalizeLogRecord = ( }, } if (Buffer.byteLength(canonicalJson(data), "utf8") > MAX_DATA_BYTES) - throw new OtlpFieldError(`normalized log event exceeds ${MAX_DATA_BYTES} UTF-8 bytes`) + throw new OtlpFieldError({ message: `normalized log event exceeds ${MAX_DATA_BYTES} UTF-8 bytes` }) const source = sourceUri(resource, record) const occurrenceId = sourceOccurrenceId(record) const subject = stringAttribute(record, "event.subject") ?? stringAttribute(record, "cloudevents.subject") @@ -490,12 +488,10 @@ export interface OtlpLogNormalizationResult { } /** Projection limits isolate individual records; resource and scope attributes are normalized once per group. */ -export const normalizeOtlpLogsWithDiagnostics = ( - request: unknown, - _acceptedAt = new Date().toISOString(), - tenantId = "local", +const normalizeDecodedLogs = ( + input: typeof LogsRequestSchema.Type, + tenantId: string, ): OtlpLogNormalizationResult => { - const input = decodeLogsRequest(request) const signals: NormalizedSignal[] = [] const unprojectedIdentities: OtlpRecoveryIdentity[] = [] let ineligible = 0 @@ -515,7 +511,7 @@ export const normalizeOtlpLogsWithDiagnostics = ( ) }) if (Result.isFailure(normalized)) { - if (!(normalized.failure instanceof OtlpFieldError)) throw normalized.failure + if (!isOtlpFieldError(normalized.failure)) throw normalized.failure failures += 1 } else if (normalized.success === null) ineligible += 1 else { @@ -530,16 +526,20 @@ export const normalizeOtlpLogsWithDiagnostics = ( return { signals, unprojectedIdentities, ineligible, failures } } +/** An undecodable request fails as a whole; a malformed record is counted and skipped. */ +export const normalizeOtlpLogsWithDiagnostics = ( + request: unknown, + _acceptedAt = new Date().toISOString(), + tenantId = "local", +): Effect.Effect => + decodeLogsRequest(request).pipe( + // Per-record field errors are already counted, so anything thrown here is a defect. + Effect.flatMap((input) => Effect.sync(() => normalizeDecodedLogs(input, tenantId))), + ) + export const normalizeOtlpLogs = ( request: unknown, acceptedAt = new Date().toISOString(), tenantId = "local", -): readonly NormalizedSignal[] => normalizeOtlpLogsWithDiagnostics(request, acceptedAt, tenantId).signals - -export const OTLP_LOG_ADAPTER: SignalSourceAdapter< - unknown, - { readonly acceptedAt: string; readonly tenantId: string } -> = { - definition: OTLP_LOG_SOURCE, - normalize: (raw, context) => normalizeOtlpLogs(raw, context.acceptedAt, context.tenantId), -} +): Effect.Effect => + normalizeOtlpLogsWithDiagnostics(request, acceptedAt, tenantId).pipe(Effect.map(({ signals }) => signals)) diff --git a/apps/cli/src/server/eventing/runtime.ts b/apps/cli/src/server/eventing/runtime.ts index 956c0e9b3..c2de82651 100644 --- a/apps/cli/src/server/eventing/runtime.ts +++ b/apps/cli/src/server/eventing/runtime.ts @@ -1,4 +1,5 @@ -import { Result } from "effect" +// BOUNDARY: activation candidates and decoded OTLP bodies arrive unparsed; this runtime decodes them. +import { Context, Duration, Effect, Exit, Layer, Result, Schema, SynchronizedRef } from "effect" import { createHash } from "node:crypto" import { canonicalJson, @@ -12,13 +13,28 @@ import { type JsonValue, type NormalizedSignal, type ProjectionFailure, + type ProjectionInvalid, type SignalProjectionSpec, + type SignalSourceInvalid, } from "@maple/eventing-core" -import { Schema } from "effect" -import { LocalEventingControlStore } from "./control-store" -import type { EventConsumerStart } from "./control-store" -import { normalizeOtlpLogsWithDiagnostics, OTLP_LOG_ADAPTER, type OtlpRecoveryIdentity } from "./otlp" -import { NOOP_EVENTING_TELEMETRY, type EventingTelemetry } from "./telemetry" +import type { OtlpFieldError } from "../otlp/encode" +import { + type DeliveryGap, + type EventConsumer, + type EventConsumerAcknowledgement, + type EventConsumerClaim, + type EventConsumerFailure, + type EventConsumerStart, + type EventingControlSnapshotValidation, + type EventingControlStoreError, + type EventingOutboxPage, + LocalEventingControlStore, + type OutboxAdministrationInvalid, + type OutboxCapacity, + type StageEventsResult, +} from "./control-store" +import { normalizeOtlpLogsWithDiagnostics, OTLP_LOG_SOURCE, type OtlpRecoveryIdentity } from "./otlp" +import { observeEventing } from "./telemetry" const TENANT_ID = "local" @@ -34,9 +50,15 @@ export class StagedOccurrenceUnrecoverable extends Schema.TaggedError()( + "@maple/cli/eventing/SourceOccurrenceInvalid", + { message: Schema.String }, +) {} + export class ProjectionActivationInvalid extends Schema.TaggedError()( "@maple/cli/eventing/ProjectionActivationInvalid", - { message: Schema.String }, + { message: Schema.String, cause: Schema.optionalKey(Schema.Defect()) }, ) {} /** Another activation committed between prepare and commit; the request can be retried. */ @@ -45,6 +67,15 @@ export class ProjectionActivationConflict extends Schema.TaggedError @@ -60,6 +91,84 @@ export interface LocalProjectionActivation { readonly generation: number } +export interface LocalEventingHealth extends EventingControlSnapshotValidation { + readonly activeProjections: number + readonly deliveryGap: DeliveryGap + readonly outboxCapacity: OutboxCapacity +} + +export interface LocalEventingRuntimeApi { + readonly hasActiveSource: (sourceKind: string) => Effect.Effect + /** Validation and full registry compilation, run while ingest admission stays open. */ + readonly prepareActivation: ( + candidate: unknown, + ) => Effect.Effect + readonly commitActivation: ( + activation: LocalProjectionActivation, + ) => Effect.Effect + readonly activate: ( + candidate: unknown, + ) => Effect.Effect< + void, + ProjectionActivationInvalid | ProjectionActivationConflict | EventingControlStoreError + > + readonly listActive: Effect.Effect + readonly evaluateOtlp: ( + signal: "traces" | "logs" | "metrics", + decoded: unknown, + isRetiredUtcDay?: (rangeDate: string) => boolean, + ) => Effect.Effect + readonly persistFailures: ( + failures: readonly ProjectionFailure[], + ) => Effect.Effect + readonly stage: ( + events: readonly MapleCloudEvent[], + sourceFingerprints?: ReadonlyMap, + ) => Effect.Effect + readonly markReady: (eventIds: readonly string[]) => Effect.Effect + readonly listReady: ( + limit?: number, + after?: number, + ) => Effect.Effect + readonly listStaged: ( + limit?: number, + after?: number, + ) => Effect.Effect + readonly listConsumers: Effect.Effect + readonly registerConsumer: ( + consumerId: string, + startAt: EventConsumerStart, + ) => Effect.Effect + readonly disableConsumer: (consumerId: string) => Effect.Effect + readonly claimReady: ( + consumerId: string, + limit: number, + leaseSeconds: number, + ) => Effect.Effect + readonly acknowledgeClaim: ( + consumerId: string, + leaseToken: string, + throughSequence: number, + ) => Effect.Effect + readonly acceptDeliveryGap: ( + consumerId: string, + generation: number, + ) => Effect.Effect + readonly abandonEvents: ( + eventIds: readonly string[], + ) => Effect.Effect< + { readonly abandoned: number; readonly gap: DeliveryGap }, + OutboxAdministrationInvalid | EventingControlStoreError + > + readonly health: Effect.Effect +} + +/** The projector implementations the registry compiles against; tests register examples here. */ +export const LocalEventingProjectors = Context.Reference( + "@maple/cli/eventing/LocalEventingProjectors", + { defaultValue: () => new ProjectorRegistry() }, +) + const emptyEvaluation = (): LocalProjectionEvaluation => ({ events: [], eventSourceFingerprints: new Map(), @@ -68,8 +177,13 @@ const emptyEvaluation = (): LocalProjectionEvaluation => ({ typeMismatchFields: [], }) -export const sourceOccurrenceFingerprint = (signal: NormalizedSignal): string => { - if (!isJsonValue(signal.data)) throw new Error("normalized source occurrence must contain finite JSON") +export const sourceOccurrenceFingerprint = ( + signal: NormalizedSignal, +): Result.Result => { + if (!isJsonValue(signal.data)) + return Result.fail( + new SourceOccurrenceInvalid({ message: "normalized source occurrence must contain finite JSON" }), + ) const content: JsonValue = { sourceKind: signal.sourceKind, source: signal.source, @@ -84,7 +198,7 @@ export const sourceOccurrenceFingerprint = (signal: NormalizedSignal): string => .map(([key, value]) => ({ key, value })), data: signal.data, } - return `sha256:${createHash("sha256").update(canonicalJson(content)).digest("hex")}` + return Result.succeed(`sha256:${createHash("sha256").update(canonicalJson(content)).digest("hex")}`) } const sourceOccurrenceKey = ( @@ -102,262 +216,285 @@ const sourceOccurrenceKey = ( const recoveryIdentityKey = (identity: OtlpRecoveryIdentity): string => canonicalJson([identity.tenantId, identity.sourceKind, identity.source, identity.occurrenceId]) -export class LocalEventingRuntime { - readonly #store: LocalEventingControlStore - readonly #sources: SignalSourceRegistry - readonly #projectors: ProjectorRegistry - readonly #telemetry: EventingTelemetry - #compiled: CompiledProjectionRegistry - #activeSourceKinds = new Set() - #generation = 0 +interface RegistryState { + readonly compiled: CompiledProjectionRegistry + readonly activeSourceKinds: ReadonlySet + readonly generation: number +} + +const activationInvalid = (cause: unknown): ProjectionActivationInvalid => + new ProjectionActivationInvalid({ + message: cause instanceof Error ? cause.message : String(cause), + cause, + }) - constructor( - store: LocalEventingControlStore, - telemetry: EventingTelemetry = NOOP_EVENTING_TELEMETRY, - projectors: ProjectorRegistry = new ProjectorRegistry(), - ) { - this.#store = store - this.#telemetry = telemetry - this.#sources = Result.getOrThrow(new SignalSourceRegistry().register(OTLP_LOG_ADAPTER.definition)) - this.#projectors = projectors - const specs = store.loadEnabledProjections(TENANT_ID) - this.#compiled = Result.getOrThrow( - CompiledProjectionRegistry.compile(specs, this.#sources, this.#projectors), +export class LocalEventingRuntime extends Context.Service()( + "@maple/cli/eventing/LocalEventingRuntime", +) { + static readonly make: Effect.Effect< + LocalEventingRuntimeApi, + SignalSourceInvalid | ProjectionInvalid | EventingControlStoreError, + LocalEventingControlStore + > = Effect.gen(function* () { + const store = yield* LocalEventingControlStore + const projectors = yield* LocalEventingProjectors + const sources = yield* Effect.fromResult(new SignalSourceRegistry().register(OTLP_LOG_SOURCE)) + const specs = yield* store.loadEnabledProjections(TENANT_ID) + const compiled = yield* Effect.fromResult( + CompiledProjectionRegistry.compile(specs, sources, projectors), ) - this.#activeSourceKinds = new Set(specs.map(({ sourceKind }) => sourceKind)) - } + // Commits are serialized by the ref, so a stale activation always fails as a conflict. + const registry = yield* SynchronizedRef.make({ + compiled, + activeSourceKinds: new Set(specs.map(({ sourceKind }) => sourceKind)), + generation: 0, + }) - hasActiveSource(sourceKind: string): boolean { - return this.#activeSourceKinds.has(sourceKind) - } + const hasActiveSource = (sourceKind: string): Effect.Effect => + Effect.map(SynchronizedRef.get(registry), ({ activeSourceKinds }) => + activeSourceKinds.has(sourceKind), + ) - prepareActivation(candidate: unknown): LocalProjectionActivation { - assertSignalProjectionInputBudget(candidate) - const spec = Schema.decodeUnknownSync(SignalProjectionSpecSchema)(candidate) - if (spec.tenantId !== TENANT_ID) - throw new ProjectionActivationInvalid({ - message: `Maple Local only accepts projections for tenant ${TENANT_ID}`, - }) - const active = this.#store - .loadEnabledProjections(TENANT_ID) - .filter((candidate) => candidate.id !== spec.id) - const next = spec.enabled ? [...active, spec] : active - const compiled = Result.getOrThrow( - CompiledProjectionRegistry.compile(next, this.#sources, this.#projectors), - ) - return { spec, next, compiled, generation: this.#generation } - } + const listActive = store.loadEnabledProjections(TENANT_ID) - commitActivation(activation: LocalProjectionActivation): void { - if (activation.generation !== this.#generation) - throw new ProjectionActivationConflict({ - message: "projection registry changed during activation; retry the request", + const prepareActivation = Effect.fn("LocalEventing.prepareActivation")(function* ( + candidate: unknown, + ) { + const spec = yield* Effect.try({ + try: () => { + assertSignalProjectionInputBudget(candidate) + return Schema.decodeUnknownSync(SignalProjectionSpecSchema)(candidate) + }, + catch: activationInvalid, }) - this.#store.saveProjection(activation.spec) - this.#compiled = activation.compiled - this.#activeSourceKinds = new Set(activation.next.map(({ sourceKind }) => sourceKind)) - this.#generation += 1 - } - - activate(candidate: unknown): void { - this.commitActivation(this.prepareActivation(candidate)) - } + if (spec.tenantId !== TENANT_ID) + return yield* new ProjectionActivationInvalid({ + message: `Maple Local only accepts projections for tenant ${TENANT_ID}`, + }) + const active = (yield* listActive).filter((candidate) => candidate.id !== spec.id) + const next = spec.enabled ? [...active, spec] : active + const nextCompiled = yield* Effect.fromResult( + CompiledProjectionRegistry.compile(next, sources, projectors), + ).pipe(Effect.mapError(activationInvalid)) + const { generation } = yield* SynchronizedRef.get(registry) + return { spec, next, compiled: nextCompiled, generation } satisfies LocalProjectionActivation + }) - listActive(): readonly SignalProjectionSpec[] { - return this.#store.loadEnabledProjections(TENANT_ID) - } + const commitActivation = Effect.fn("LocalEventing.commitActivation")(function* ( + activation: LocalProjectionActivation, + ) { + yield* SynchronizedRef.modifyEffect( + registry, + ( + state, + ): Effect.Effect< + readonly [void, RegistryState], + ProjectionActivationConflict | EventingControlStoreError + > => + activation.generation !== state.generation + ? Effect.fail( + new ProjectionActivationConflict({ + message: + "projection registry changed during activation; retry the request", + }), + ) + : Effect.as(store.saveProjection(activation.spec), [ + undefined, + { + compiled: activation.compiled, + activeSourceKinds: new Set( + activation.next.map(({ sourceKind }) => sourceKind), + ), + generation: state.generation + 1, + } satisfies RegistryState, + ]), + ) + }) - evaluateOtlp( - signal: "traces" | "logs" | "metrics", - decoded: unknown, - isRetiredUtcDay: (rangeDate: string) => boolean = () => false, - ): LocalProjectionEvaluation { - const sourceKind = signal === "logs" ? "otel.log" : signal === "traces" ? "otel.span" : "otel.metric" - if (!this.hasActiveSource(sourceKind) && !this.#store.hasStagedSourceKind(TENANT_ID, sourceKind)) - return emptyEvaluation() - const startedAt = performance.now() - const acceptedAt = new Date().toISOString() - let normalized - let unprojectedIdentities: readonly OtlpRecoveryIdentity[] - try { - const result = - signal === "logs" - ? normalizeOtlpLogsWithDiagnostics(decoded, acceptedAt, TENANT_ID) - : { signals: [], unprojectedIdentities: [], ineligible: 0, failures: 0 } - normalized = result.signals - unprojectedIdentities = result.unprojectedIdentities - this.#telemetry.record({ + const evaluateOtlp = Effect.fn("LocalEventing.evaluateOtlp")(function* ( + signal: "traces" | "logs" | "metrics", + decoded: unknown, + isRetiredUtcDay: (rangeDate: string) => boolean = () => false, + ) { + const sourceKind = + signal === "logs" ? "otel.log" : signal === "traces" ? "otel.span" : "otel.metric" + if ( + !(yield* hasActiveSource(sourceKind)) && + !(yield* store.hasStagedSourceKind(TENANT_ID, sourceKind)) + ) + return emptyEvaluation() + const acceptedAt = new Date().toISOString() + const [elapsed, normalization] = yield* Effect.timed( + Effect.exit( + signal === "logs" + ? normalizeOtlpLogsWithDiagnostics(decoded, acceptedAt, TENANT_ID) + : Effect.succeed({ + signals: [], + unprojectedIdentities: [], + ineligible: 0, + failures: 0, + }), + ), + ) + if (Exit.isFailure(normalization)) { + yield* observeEventing({ + operation: "normalization", + outcome: "failure", + durationMs: Duration.toMillis(elapsed), + sourceKind, + }) + return yield* Effect.failCause(normalization.cause) + } + const { + signals: normalized, + unprojectedIdentities, + failures: normalizationFailures, + } = normalization.value + yield* observeEventing({ operation: "normalization", outcome: "success", count: normalized.length, - durationMs: performance.now() - startedAt, + durationMs: Duration.toMillis(elapsed), sourceKind, }) - if (result.failures > 0) - this.#telemetry.record({ + if (normalizationFailures > 0) + yield* observeEventing({ operation: "normalization", outcome: "failure", - count: result.failures, + count: normalizationFailures, sourceKind, }) - } catch (error) { - this.#telemetry.record({ - operation: "normalization", - outcome: "failure", - durationMs: performance.now() - startedAt, - sourceKind, - }) - throw error - } - const sourceFingerprints = new Map() - for (const occurrence of normalized) { - const key = sourceOccurrenceKey(occurrence) - if (key === null) continue - const fingerprint = sourceOccurrenceFingerprint(occurrence) - const prior = sourceFingerprints.get(key) - if (prior !== undefined && prior !== fingerprint) - throw new SourceOccurrenceCollision({ - message: `source occurrence collision within one ingest batch: ${occurrence.occurrenceId}`, - occurrenceId: occurrence.occurrenceId, - }) - sourceFingerprints.set(key, fingerprint) - } - for (const identity of unprojectedIdentities) { - if (sourceFingerprints.has(recoveryIdentityKey(identity))) - throw new SourceOccurrenceCollision({ - message: `source occurrence collision with an unprojectable record within one ingest batch: ${identity.occurrenceId}`, - occurrenceId: identity.occurrenceId, - }) - if ( - this.#store.hasStagedSourceOccurrence( - identity.tenantId, - identity.sourceKind, - identity.source, - identity.occurrenceId, + const sourceFingerprints = new Map() + for (const occurrence of normalized) { + const key = sourceOccurrenceKey(occurrence) + if (key === null) continue + const fingerprint = yield* Effect.fromResult(sourceOccurrenceFingerprint(occurrence)) + const prior = sourceFingerprints.get(key) + if (prior !== undefined && prior !== fingerprint) + return yield* new SourceOccurrenceCollision({ + message: `source occurrence collision within one ingest batch: ${occurrence.occurrenceId}`, + occurrenceId: occurrence.occurrenceId, + }) + sourceFingerprints.set(key, fingerprint) + } + for (const identity of unprojectedIdentities) { + if (sourceFingerprints.has(recoveryIdentityKey(identity))) + return yield* new SourceOccurrenceCollision({ + message: `source occurrence collision with an unprojectable record within one ingest batch: ${identity.occurrenceId}`, + occurrenceId: identity.occurrenceId, + }) + if ( + yield* store.hasStagedSourceOccurrence( + identity.tenantId, + identity.sourceKind, + identity.source, + identity.occurrenceId, + ) ) - ) - throw new StagedOccurrenceUnrecoverable({ - message: `cannot safely recover staged source occurrence after projection normalization failed: ${identity.occurrenceId}`, - occurrenceId: identity.occurrenceId, + return yield* new StagedOccurrenceUnrecoverable({ + message: `cannot safely recover staged source occurrence after projection normalization failed: ${identity.occurrenceId}`, + occurrenceId: identity.occurrenceId, + }) + } + const { compiled: snapshot } = yield* SynchronizedRef.get(registry) + const events: MapleCloudEvent[] = [] + const eventSourceFingerprints = new Map() + const recoveredEventIds: string[] = [] + const failures: ProjectionFailure[] = [] + const typeMismatchFields = new Set() + for (const occurrence of normalized) { + const sourceFingerprint = yield* Effect.fromResult(sourceOccurrenceFingerprint(occurrence)) + if (occurrence.occurrenceId !== null) { + const staged = yield* store.stagedEventIdsForOccurrence( + occurrence.tenantId, + occurrence.sourceKind, + occurrence.source, + occurrence.occurrenceId, + sourceFingerprint, + ) + if (staged.length > 0) { + recoveredEventIds.push(...staged) + continue + } + } + if (isRetiredUtcDay(occurrence.occurredAt.slice(0, 10))) continue + const result = yield* Effect.fromResult(snapshot.evaluate(occurrence, acceptedAt)) + yield* observeEventing({ + operation: "projection", + outcome: "success", + count: result.events.length, + sourceKind, }) - } - const snapshot = this.#compiled - const events: MapleCloudEvent[] = [] - const eventSourceFingerprints = new Map() - const recoveredEventIds: string[] = [] - const failures: ProjectionFailure[] = [] - const typeMismatchFields = new Set() - for (const occurrence of normalized) { - const sourceFingerprint = sourceOccurrenceFingerprint(occurrence) - if (occurrence.occurrenceId !== null) { - const staged = this.#store.stagedEventIdsForOccurrence( - occurrence.tenantId, - occurrence.sourceKind, - occurrence.source, - occurrence.occurrenceId, - sourceFingerprint, - ) - if (staged.length > 0) { - recoveredEventIds.push(...staged) - continue + yield* observeEventing({ + operation: "projection", + outcome: "failure", + count: result.failures.length, + sourceKind, + }) + events.push(...result.events) + for (const event of result.events) { + const priorFingerprint = eventSourceFingerprints.get(event.id) + if (priorFingerprint !== undefined && priorFingerprint !== sourceFingerprint) + return yield* new SourceOccurrenceCollision({ + message: `source occurrence collision within one ingest batch: ${event.id}`, + occurrenceId: occurrence.occurrenceId, + }) + eventSourceFingerprints.set(event.id, sourceFingerprint) } + failures.push(...result.failures) + for (const mismatch of result.typeMismatchFields) typeMismatchFields.add(mismatch) } - if (isRetiredUtcDay(occurrence.occurredAt.slice(0, 10))) continue - const result = Result.getOrThrow(snapshot.evaluate(occurrence, acceptedAt)) - this.#telemetry.record({ - operation: "projection", - outcome: "success", - count: result.events.length, - sourceKind, - }) - this.#telemetry.record({ - operation: "projection", - outcome: "failure", - count: result.failures.length, - sourceKind, - }) - events.push(...result.events) - for (const event of result.events) { - const priorFingerprint = eventSourceFingerprints.get(event.id) - if (priorFingerprint !== undefined && priorFingerprint !== sourceFingerprint) - throw new SourceOccurrenceCollision({ - message: `source occurrence collision within one ingest batch: ${event.id}`, - occurrenceId: occurrence.occurrenceId, - }) - eventSourceFingerprints.set(event.id, sourceFingerprint) - } - failures.push(...result.failures) - for (const mismatch of result.typeMismatchFields) typeMismatchFields.add(mismatch) - } - if (typeMismatchFields.size > 0) - this.#telemetry.record({ - operation: "selector_type_mismatch", - outcome: "observed", - count: typeMismatchFields.size, - sourceKind, - }) - return { - events, - eventSourceFingerprints, - recoveredEventIds, - failures, - typeMismatchFields: [...typeMismatchFields], - } - } - - persistFailures(failures: readonly ProjectionFailure[]): void { - if (failures.length > 0) this.#store.recordProjectionFailures(TENANT_ID, failures) - } - - stage(events: readonly MapleCloudEvent[], sourceFingerprints: ReadonlyMap = new Map()) { - return this.#store.stageEvents(events, sourceFingerprints) - } - - markReady(eventIds: readonly string[]): void { - this.#store.markReady(eventIds) - } - - listReady(limit?: number, after?: number) { - return this.#store.listReady(limit, after) - } - - listStaged(limit?: number, after?: number) { - return this.#store.listStaged(limit, after) - } - - listConsumers() { - return this.#store.listConsumers(TENANT_ID) - } - - registerConsumer(consumerId: string, startAt: EventConsumerStart) { - return this.#store.registerConsumer(TENANT_ID, consumerId, startAt) - } - - disableConsumer(consumerId: string) { - return this.#store.disableConsumer(TENANT_ID, consumerId) - } - - claimReady(consumerId: string, limit: number, leaseSeconds: number) { - return this.#store.claimReady(TENANT_ID, consumerId, limit, leaseSeconds) - } - - acknowledgeClaim(consumerId: string, leaseToken: string, throughSequence: number) { - return this.#store.acknowledgeClaim(TENANT_ID, consumerId, leaseToken, throughSequence) - } - - acceptDeliveryGap(consumerId: string, generation: number) { - return this.#store.acceptDeliveryGap(TENANT_ID, consumerId, generation) - } - abandonEvents(eventIds: readonly string[]) { - return this.#store.abandonEvents(TENANT_ID, eventIds) - } + if (typeMismatchFields.size > 0) + yield* observeEventing({ + operation: "selector_type_mismatch", + outcome: "observed", + count: typeMismatchFields.size, + sourceKind, + }) + return { + events, + eventSourceFingerprints, + recoveredEventIds, + failures, + typeMismatchFields: [...typeMismatchFields], + } satisfies LocalProjectionEvaluation + }) - health() { return { - activeProjections: this.listActive().length, - deliveryGap: this.#store.deliveryGap(TENANT_ID), - outboxCapacity: this.#store.outboxCapacity(), - ...this.#store.validate(), - } - } + hasActiveSource, + prepareActivation, + commitActivation, + activate: (candidate) => Effect.flatMap(prepareActivation(candidate), commitActivation), + listActive, + evaluateOtlp, + persistFailures: (failures) => + failures.length > 0 ? store.recordProjectionFailures(TENANT_ID, failures) : Effect.void, + stage: (events, sourceFingerprints = new Map()) => + store.stageEvents(events, sourceFingerprints).pipe(Effect.withSpan("LocalEventing.stage")), + markReady: (eventIds) => + store.markReady(eventIds).pipe(Effect.withSpan("LocalEventing.markReady")), + listReady: store.listReady, + listStaged: store.listStaged, + listConsumers: store.listConsumers(TENANT_ID), + registerConsumer: (consumerId, startAt) => store.registerConsumer(TENANT_ID, consumerId, startAt), + disableConsumer: (consumerId) => store.disableConsumer(TENANT_ID, consumerId), + claimReady: (consumerId, limit, leaseSeconds) => + store.claimReady(TENANT_ID, consumerId, limit, leaseSeconds), + acknowledgeClaim: (consumerId, leaseToken, throughSequence) => + store.acknowledgeClaim(TENANT_ID, consumerId, leaseToken, throughSequence), + acceptDeliveryGap: (consumerId, generation) => + store.acceptDeliveryGap(TENANT_ID, consumerId, generation), + abandonEvents: (eventIds) => store.abandonEvents(TENANT_ID, eventIds), + health: Effect.gen(function* () { + return { + activeProjections: (yield* listActive).length, + deliveryGap: yield* store.deliveryGap(TENANT_ID), + outboxCapacity: yield* store.outboxCapacity, + ...(yield* store.validate), + } + }), + } satisfies LocalEventingRuntimeApi + }) + + static readonly layer = Layer.effect(this, this.make).pipe(Layer.provide(LocalEventingControlStore.layer)) } diff --git a/apps/cli/src/server/eventing/telemetry.ts b/apps/cli/src/server/eventing/telemetry.ts index 89475a4ae..da8523b49 100644 --- a/apps/cli/src/server/eventing/telemetry.ts +++ b/apps/cli/src/server/eventing/telemetry.ts @@ -35,12 +35,6 @@ export interface EventingTelemetryObservation { readonly sourceKind?: EventingTelemetrySourceKind } -export interface EventingTelemetry { - record(observation: EventingTelemetryObservation): void -} - -export const NOOP_EVENTING_TELEMETRY: EventingTelemetry = { record: () => {} } - const operations = Metric.counter("maple.eventing.operations_total", { description: "Eventing operations by bounded operation and outcome", incremental: true, @@ -54,30 +48,24 @@ const consumerLag = Metric.histogram("maple.eventing.consumer_lag_events", { boundaries: [0, 1, 5, 10, 50, 100, 500, 1_000, 10_000], }) -export const makeEffectEventingTelemetry = ( - run: (effect: Effect.Effect) => void, -): EventingTelemetry => ({ - record(observation) { - const attributes = { - operation: observation.operation, - outcome: observation.outcome, - source_kind: observation.sourceKind ?? "unknown", - } - const effects: Effect.Effect[] = [] - const count = observation.count ?? 1 - if (Number.isFinite(count) && count > 0) - effects.push(Metric.update(Metric.withAttributes(operations, attributes), count)) - if (observation.durationMs !== undefined && Number.isFinite(observation.durationMs)) - effects.push( - Metric.update( - Metric.withAttributes(durations, attributes), - Math.max(0, observation.durationMs), - ), - ) - if (observation.lag !== undefined && Number.isSafeInteger(observation.lag)) - effects.push( - Metric.update(Metric.withAttributes(consumerLag, attributes), Math.max(0, observation.lag)), - ) - if (effects.length > 0) run(Effect.all(effects, { discard: true })) - }, -}) +/** Records one bounded observation into the ambient metric registry. */ +export const observeEventing = (observation: EventingTelemetryObservation): Effect.Effect => { + const attributes = { + operation: observation.operation, + outcome: observation.outcome, + source_kind: observation.sourceKind ?? "unknown", + } + const effects: Effect.Effect[] = [] + const count = observation.count ?? 1 + if (Number.isFinite(count) && count > 0) + effects.push(Metric.update(Metric.withAttributes(operations, attributes), count)) + if (observation.durationMs !== undefined && Number.isFinite(observation.durationMs)) + effects.push( + Metric.update(Metric.withAttributes(durations, attributes), Math.max(0, observation.durationMs)), + ) + if (observation.lag !== undefined && Number.isSafeInteger(observation.lag)) + effects.push( + Metric.update(Metric.withAttributes(consumerLag, attributes), Math.max(0, observation.lag)), + ) + return Effect.all(effects, { discard: true }) +} diff --git a/apps/cli/src/server/local-token.ts b/apps/cli/src/server/local-token.ts index af9c4c928..62f4e0118 100644 --- a/apps/cli/src/server/local-token.ts +++ b/apps/cli/src/server/local-token.ts @@ -3,9 +3,16 @@ import { lstatSync, readFileSync } from "node:fs" import { Result, Schema } from "effect" import { durableWrite } from "./durable-files" +/** A credential or ledger path that is a symlink or a non-file; refused rather than followed. */ +export class LocalFileNotReal extends Schema.TaggedError()("@maple/cli/LocalFileNotReal", { + message: Schema.String, + path: Schema.String, +}) {} + export const readRealFile = (path: string, label: string): string => { const stat = lstatSync(path) - if (stat.isSymbolicLink() || !stat.isFile()) throw new Error(`${label} is not a real file: ${path}`) + if (stat.isSymbolicLink() || !stat.isFile()) + throw new LocalFileNotReal({ message: `${label} is not a real file: ${path}`, path }) return readFileSync(path, "utf8") } diff --git a/apps/cli/src/server/otlp/encode.ts b/apps/cli/src/server/otlp/encode.ts index 72c936505..fd576478c 100644 --- a/apps/cli/src/server/otlp/encode.ts +++ b/apps/cli/src/server/otlp/encode.ts @@ -19,6 +19,8 @@ * - attributes are arrays of `{ key, value: AnyValue }` */ +import { Schema } from "effect" + export interface EncodedBatch { datasource: string rowCount: number @@ -96,14 +98,10 @@ function base64ToBytes(b64: string): Uint8Array { * 400 from the ingest handler rather than the generic 500 an encoder crash * would produce. */ -// The pure encoder throws this sentinel for its HTTP adapter to translate into a 400 response. -// oxlint-disable-next-line effecttsgo/extends-native-error -export class OtlpFieldError extends Error { - constructor(message: string) { - super(message) - this.name = "OtlpFieldError" - } -} +// The pure encoder throws this for its HTTP adapter; eventing normalization fails with it. +export class OtlpFieldError extends Schema.TaggedError()("@maple/cli/OtlpFieldError", { + message: Schema.String, +}) {} const HEX_ONLY = /^[0-9a-fA-F]+$/ const ALL_ZERO_HEX = /^0+$/ @@ -151,9 +149,9 @@ function idHex(value: string | undefined, byteLength: number, field: string): st // noise ("deadbeef" is 4 hex bytes, but decodes as 6 base64 bytes). const base64Chars = Math.ceil(byteLength / 3) * 4 const shown = value.length > 80 ? `${value.slice(0, 80)}…` : value - throw new OtlpFieldError( - `${field} must be ${byteLength} bytes: expected ${byteLength * 2} hex chars (OTLP/JSON) or ${base64Chars} base64 chars (OTLP/protobuf), got ${JSON.stringify(shown)} (${value.length} chars)`, - ) + throw new OtlpFieldError({ + message: `${field} must be ${byteLength} bytes: expected ${byteLength * 2} hex chars (OTLP/JSON) or ${base64Chars} base64 chars (OTLP/protobuf), got ${JSON.stringify(shown)} (${value.length} chars)`, + }) } return hexFromBytes(bytes) } diff --git a/apps/cli/src/server/serve.ts b/apps/cli/src/server/serve.ts index b8b4978e8..5364098e4 100644 --- a/apps/cli/src/server/serve.ts +++ b/apps/cli/src/server/serve.ts @@ -3,7 +3,7 @@ // SPA, all on one port, backed by an embedded chDB. Replaces the Rust // `apps/ingest/src/bin/local.rs`. `maple start` calls `startServer`. -import { Effect, Predicate, Result, Schema, type Scope } from "effect" +import { Effect, Layer, Predicate, Result, Schema, type Scope } from "effect" import * as ManagedRuntime from "effect/ManagedRuntime" import { gunzipSync } from "node:zlib" import { TelemetryLayer } from "../core/telemetry" @@ -20,22 +20,14 @@ import { import { buildInsertStatements } from "./inserts" import { eventingControlSnapshotPath, - EventConsumerConflictError, - EventConsumerLeaseError, - EventConsumerDeliveryGapError, - OutboxAdministrationInvalid, - EventConsumerInputError, - EventConsumerNotFoundError, + type EventConsumerFailure, + type EventingControlStoreError, + LocalEventingControlConfig, LocalEventingControlStore, + writeControlSnapshot, } from "./eventing/control-store" import { ensureEventConsumerToken, eventConsumerTokenMatches } from "./eventing/consumer-auth" -import { - LocalEventingRuntime, - ProjectionActivationConflict, - ProjectionActivationInvalid, - SourceOccurrenceCollision, -} from "./eventing/runtime" -import { makeEffectEventingTelemetry } from "./eventing/telemetry" +import { LocalEventingRuntime, type LocalEventingRuntimeApi } from "./eventing/runtime" import { encodeLogs, encodeMetrics, encodeTraces, type EncodedBatch, OtlpFieldError } from "./otlp/encode" import { decodeLogsRequest, @@ -234,147 +226,143 @@ interface IngestResult { readonly requestBytes: number } +/** An ingest step that ends the request early; the carried result is the response. */ +class IngestStopped extends Schema.TaggedError()("@maple/cli/IngestStopped", { + message: Schema.String, + response: Schema.instanceOf(Response), + accepted: Schema.Number, + requestBytes: Schema.Number, +}) {} + +const stopIngest = (response: Response, requestBytes: number, accepted = 0): IngestStopped => + new IngestStopped({ message: `HTTP ${response.status}`, response, accepted, requestBytes }) + /** * A malformed field or an in-batch identity collision fails identically on every * retry, so it is a 400; OTLP exporters retry 503 and would resend the batch forever. */ -const projectionFailureStatus = (error: unknown): 400 | 503 => - error instanceof OtlpFieldError || error instanceof SourceOccurrenceCollision ? 400 : 503 +const projectionFailure = + (signal: Signal, requestBytes: number, status: 400 | 503) => + (error: { readonly message: string }): Effect.Effect => + Effect.fail(stopIngest(text(`event projection ${signal}: ${error.message}`, status), requestBytes)) -async function ingest( +const ingest = ( db: Pick, authority: RetiredDayAuthority, - eventing: LocalEventingRuntime, signal: Signal, req: Request, -): Promise { - let raw: Uint8Array - try { - raw = new Uint8Array(await req.arrayBuffer()) - } catch (error) { - // A client that hangs up mid-body rejects here. Unguarded, this became an - // Effect *defect* — no `error.type`, no 4xx suppression, and a span reading - // only "The connection was closed." It is a caller outcome, so 400 it. - return { - response: text(`read ${signal} body: ${describeThrown(error)}`, 400), - accepted: 0, - requestBytes: 0, - } - } - const requestBytes = raw.length - const contentType = req.headers.get("content-type") ?? "" - const contentEncoding = req.headers.get("content-encoding") - let decoded: unknown - try { - decoded = decodeOtlp(signal, raw, contentType, contentEncoding) - } catch (error) { - return { - response: text(`decode ${signal}: ${describeThrown(error)}`, 400), - accepted: 0, - requestBytes, - } - } - let evaluation: ReturnType - try { - evaluation = eventing.evaluateOtlp(signal, decoded, (rangeDate) => authority.isRetired(rangeDate)) - } catch (error) { - return { - response: text( - `event projection ${signal}: ${describeThrown(error)}`, - projectionFailureStatus(error), - ), - accepted: 0, - requestBytes, - } - } - let batches: EncodedBatch[] - try { - batches = encodeFor(signal, decoded) - } catch (error) { - // A malformed field is the sender's fault, not ours — reject the batch - // with a 400 naming the field instead of silently storing a bad value. - const status = error instanceof OtlpFieldError ? 400 : 500 - const stage = status === 400 ? "decode" : "encode" - return { - response: text(`${stage} ${signal}: ${describeThrown(error)}`, status), - accepted: 0, - requestBytes, - } - } - let stagedEventIds: readonly string[] = [] - let droppedEvents = 0 - try { - eventing.persistFailures(evaluation.failures) - if (evaluation.events.length > 0) { - const staged = eventing.stage(evaluation.events, evaluation.eventSourceFingerprints) - stagedEventIds = staged.eventIds - droppedEvents = staged.dropped - } - } catch (error) { - return { - response: text( - `event projection ${signal}: ${describeThrown(error)}`, - projectionFailureStatus(error), +): Effect.Effect => + Effect.gen(function* () { + const eventing = yield* LocalEventingRuntime + // A client that hangs up mid-body rejects here. It is a caller outcome, so 400 it. + const raw = yield* Effect.tryPromise({ + try: async () => new Uint8Array(await req.arrayBuffer()), + catch: (error) => stopIngest(text(`read ${signal} body: ${describeThrown(error)}`, 400), 0), + }) + const requestBytes = raw.length + const contentType = req.headers.get("content-type") ?? "" + const contentEncoding = req.headers.get("content-encoding") + const decoded = yield* Effect.try({ + try: () => decodeOtlp(signal, raw, contentType, contentEncoding), + catch: (error) => + stopIngest(text(`decode ${signal}: ${describeThrown(error)}`, 400), requestBytes), + }) + const rejectProjection = (status: 400 | 503) => projectionFailure(signal, requestBytes, status) + const evaluation = yield* eventing + .evaluateOtlp(signal, decoded, (rangeDate) => authority.isRetired(rangeDate)) + .pipe( + Effect.catchTags({ + "@maple/cli/OtlpFieldError": rejectProjection(400), + "@maple/cli/eventing/SourceOccurrenceCollision": rejectProjection(400), + "@maple/cli/eventing/StagedOccurrenceUnrecoverable": rejectProjection(503), + "@maple/cli/eventing/SourceOccurrenceInvalid": rejectProjection(503), + "@maple/eventing-core/ProjectionInvalid": rejectProjection(503), + "@maple/cli/eventing/ControlStoreFailed": rejectProjection(503), + }), + Effect.catchDefect((defect) => rejectProjection(503)({ message: describeThrown(defect) })), + ) + // A malformed field is the sender's fault, not ours: reject the batch with a + // 400 naming the field instead of silently storing a bad value. + let batches = yield* Effect.try({ + try: () => encodeFor(signal, decoded), + catch: (error) => + error instanceof OtlpFieldError + ? stopIngest(text(`decode ${signal}: ${error.message}`, 400), requestBytes) + : stopIngest(text(`encode ${signal}: ${describeThrown(error)}`, 500), requestBytes), + }) + const staged = yield* eventing.persistFailures(evaluation.failures).pipe( + Effect.andThen( + evaluation.events.length > 0 + ? eventing.stage(evaluation.events, evaluation.eventSourceFingerprints) + : Effect.succeed({ inserted: 0, deduplicated: 0, dropped: 0, eventIds: [] }), ), - accepted: 0, - requestBytes, - } - } - let rejected = 0 - batches = batches.map((batch) => { - const filtered = authority.filterBatch(batch.datasource, batch.ndjson) - rejected += filtered.rejected - return { ...batch, ndjson: filtered.ndjson, rowCount: filtered.accepted } - }) - let accepted = 0 - for (const batch of batches) { - if (batch.rowCount === 0) continue - for (const statement of buildInsertStatements(batch.datasource, batch.ndjson)) { - try { - db.exec(statement.sql) - } catch (error) { - return { - response: text(`chDB insert (${batch.datasource}): ${describeThrown(error)}`, 500), - accepted, - requestBytes, - } + Effect.catchTag("@maple/cli/eventing/ControlStoreFailed", rejectProjection(503)), + Effect.catchDefect((defect) => rejectProjection(503)({ message: describeThrown(defect) })), + ) + let rejected = 0 + batches = batches.map((batch) => { + const filtered = authority.filterBatch(batch.datasource, batch.ndjson) + rejected += filtered.rejected + return { ...batch, ndjson: filtered.ndjson, rowCount: filtered.accepted } + }) + let accepted = 0 + for (const batch of batches) { + if (batch.rowCount === 0) continue + for (const statement of buildInsertStatements(batch.datasource, batch.ndjson)) { + const inserted = Result.try(() => db.exec(statement.sql)) + if (Result.isFailure(inserted)) + return yield* stopIngest( + text(`chDB insert (${batch.datasource}): ${describeThrown(inserted.failure)}`, 500), + requestBytes, + accepted, + ) + accepted += statement.rowCount } - accepted += statement.rowCount } - } - try { - const readyEventIds = [...evaluation.recoveredEventIds, ...stagedEventIds] - if (readyEventIds.length > 0) eventing.markReady(readyEventIds) - } catch (error) { - return { - response: text(`event outbox readiness ${signal}: ${describeThrown(error)}`, 503), - accepted, - requestBytes, + const readyEventIds = [...evaluation.recoveredEventIds, ...staged.eventIds] + const readinessFailed = (message: string) => + Effect.fail( + stopIngest(text(`event outbox readiness ${signal}: ${message}`, 503), requestBytes, accepted), + ) + if (readyEventIds.length > 0) + yield* eventing.markReady(readyEventIds).pipe( + Effect.catchTag("@maple/cli/eventing/ControlStoreFailed", (error) => + readinessFailed(error.message), + ), + Effect.catchDefect((defect) => readinessFailed(describeThrown(defect))), + ) + const droppedEvents = staged.dropped + const errorMessage = rejected > 0 ? "telemetry from permanently retired UTC days was rejected" : "" + if (contentType.includes("json")) { + const rejectedField = + signal === "traces" + ? { rejectedSpans: rejected } + : signal === "logs" + ? { rejectedLogRecords: rejected } + : { rejectedDataPoints: rejected } + const response = json(rejected > 0 ? { partialSuccess: { ...rejectedField, errorMessage } } : {}) + if (droppedEvents > 0) response.headers.set("x-maple-eventing-dropped", String(droppedEvents)) + return { response, accepted, requestBytes } } - } - const errorMessage = rejected > 0 ? "telemetry from permanently retired UTC days was rejected" : "" - if (contentType.includes("json")) { - const rejectedField = - signal === "traces" - ? { rejectedSpans: rejected } - : signal === "logs" - ? { rejectedLogRecords: rejected } - : { rejectedDataPoints: rejected } - const response = json(rejected > 0 ? { partialSuccess: { ...rejectedField, errorMessage } } : {}) + const response = new Response(encodeExportResponse(signal, rejected, errorMessage), { + status: 200, + headers: { "content-type": "application/x-protobuf" }, + }) if (droppedEvents > 0) response.headers.set("x-maple-eventing-dropped", String(droppedEvents)) - return { - response, - accepted, - requestBytes, - } - } - const response = new Response(encodeExportResponse(signal, rejected, errorMessage), { - status: 200, - headers: { "content-type": "application/x-protobuf" }, - }) - if (droppedEvents > 0) response.headers.set("x-maple-eventing-dropped", String(droppedEvents)) - return { response, accepted, requestBytes } -} + return { response, accepted, requestBytes } + }).pipe( + Effect.catchTag("@maple/cli/IngestStopped", ({ response, accepted, requestBytes }) => + Effect.succeed({ response, accepted, requestBytes }), + ), + // Anything thrown outside a mapped step keeps its old meaning: a 500 with a real message. + Effect.catchDefect((defect) => + Effect.succeed({ + response: text(`ingest ${signal}: ${describeThrown(defect)}`, 500), + accepted: 0, + requestBytes: 0, + }), + ), + ) /** * Strip a trailing `FORMAT ` clause (optionally followed by `;`) and @@ -468,9 +456,12 @@ function serveAsset(assets: AssetResolver, pathname: string): Response { const MAX_DB_QUERY_TEXT = 16 * 1024 const truncateSql = (sql: string) => (sql.length > MAX_DB_QUERY_TEXT ? sql.slice(0, MAX_DB_QUERY_TEXT) : sql) -/** Runs a request's span effect on the server's tracing runtime (see - * `startServer`). The effect always succeeds with a `Response`. */ -type SpanRunner = (effect: Effect.Effect) => Promise +/** The services every request effect may use; `startServer` builds them into one runtime. */ +type RequestServices = LocalEventingRuntime | LocalEventingControlStore + +/** Runs a request's effect on the server's tracing runtime (see `startServer`). + * The effect always succeeds with a `Response`. */ +type SpanRunner = (effect: Effect.Effect) => Promise // A rejected 5xx ingest/query response, surfaced through the Effect error // channel. `message` carries the handler's descriptive body so the span records @@ -478,12 +469,6 @@ type SpanRunner = (effect: Effect.Effect) => Promise // hand back to the client in `recoverResponse`. (Failing with a bare `Response` // recorded an empty `{}` — a `Response` has no enumerable own fields — which lost // the cause entirely and bucketed every failure under one "Error" fingerprint.) -/** A rejection out of `ingest`, carried as a typed failure so it becomes a 500 - * with a real message rather than an untyped defect. */ -class IngestFailed extends Schema.TaggedError()("@maple/cli/IngestFailed", { - message: Schema.String, -}) {} - class IngestRejected extends Schema.TaggedError()("@maple/cli/IngestRejected", { response: Schema.instanceOf(Response), status: Schema.Number, @@ -508,87 +493,60 @@ const recordServerResponse = (response: Response): Effect.Effect): Effect.Effect => +const recoverResponse = ( + self: Effect.Effect, +): Effect.Effect => Effect.match(self, { onFailure: (error) => error.response, onSuccess: (response) => response }) /** OTLP-ingest request as a `Server`-kind span, mirroring the Rust gateway * (`apps/ingest`): `maple.signal`, item count, request size, HTTP semconv. */ const ingestSpan = ( - runSpan: SpanRunner, db: Chdb, authority: RetiredDayAuthority, - eventing: LocalEventingRuntime, signal: Signal, req: Request, -): Promise => - runSpan( - recoverResponse( - Effect.gen(function* () { - // `Effect.promise` is for promises that cannot reject, and `ingest` - // can: it awaits the request body and drives chDB. A rejection there - // became a DEFECT, which `recoverResponse`'s `Effect.match` does not - // catch — so it escaped as an untyped, unlabelled span error instead of - // the 500 the caller should have received. - const { response, accepted, requestBytes } = yield* Effect.tryPromise({ - try: () => ingest(db, authority, eventing, signal, req), - catch: (error): IngestFailed => new IngestFailed({ message: describeThrown(error) }), - }).pipe( - Effect.catchTag("@maple/cli/IngestFailed", (error) => - Effect.succeed({ - response: text(`ingest ${signal}: ${error.message}`, 500), - accepted: 0, - requestBytes: 0, - }), - ), - ) - yield* Effect.annotateCurrentSpan({ - "http.request.body.size": requestBytes, - "maple.ingest.item_count": accepted, - "http.response.status_code": response.status, - }) - return yield* recordServerResponse(response) - }).pipe( - Effect.withSpan(`POST /v1/${signal}`, { - kind: "server", - attributes: { - "maple.signal": signal, - "http.request.method": "POST", - "http.route": `/v1/${signal}`, - }, - }), - ), +): Effect.Effect => + recoverResponse( + Effect.gen(function* () { + const { response, accepted, requestBytes } = yield* ingest(db, authority, signal, req) + yield* Effect.annotateCurrentSpan({ + "http.request.body.size": requestBytes, + "maple.ingest.item_count": accepted, + "http.response.status_code": response.status, + }) + return yield* recordServerResponse(response) + }).pipe( + Effect.withSpan(`POST /v1/${signal}`, { + kind: "server", + attributes: { + "maple.signal": signal, + "http.request.method": "POST", + "http.route": `/v1/${signal}`, + }, + }), ), ) /** `/local/query` request as a `Server`-kind span with the canonical DB attrs. */ -const querySpan = ( - runSpan: SpanRunner, - db: Chdb, - authority: RetiredDayAuthority, - req: Request, -): Promise => - runSpan( - recoverResponse( - Effect.gen(function* () { - const { response, rowCount, durationMs, sql } = yield* Effect.promise(() => - handleQuery(db, authority, req), - ) - yield* Effect.annotateCurrentSpan({ - "db.system.name": "clickhouse", - "db.duration_ms": durationMs, - "result.rowCount": rowCount, - "http.response.status_code": response.status, - ...(sql - ? { "db.query.text": truncateSql(sql), "db.query.length": sql.length } - : undefined), - }) - return yield* recordServerResponse(response) - }).pipe( - Effect.withSpan("POST /local/query", { - kind: "server", - attributes: { "http.request.method": "POST", "http.route": "/local/query" }, - }), - ), +const querySpan = (db: Chdb, authority: RetiredDayAuthority, req: Request): Effect.Effect => + recoverResponse( + Effect.gen(function* () { + const { response, rowCount, durationMs, sql } = yield* Effect.promise(() => + handleQuery(db, authority, req), + ) + yield* Effect.annotateCurrentSpan({ + "db.system.name": "clickhouse", + "db.duration_ms": durationMs, + "result.rowCount": rowCount, + "http.response.status_code": response.status, + ...(sql ? { "db.query.text": truncateSql(sql), "db.query.length": sql.length } : undefined), + }) + return yield* recordServerResponse(response) + }).pipe( + Effect.withSpan("POST /local/query", { + kind: "server", + attributes: { "http.request.method": "POST", "http.route": "/local/query" }, + }), ), ) @@ -613,16 +571,40 @@ export class RequestQuiescenceGate { } } + #drain(): Promise { + return this.#active > 0 + ? new Promise((resolve) => this.#drained.push(resolve)) + : Promise.resolve() + } + async exclusive(work: () => Promise): Promise { if (this.#closed) throw MaintenanceInProgressError.create() this.#closed = true try { - if (this.#active > 0) await new Promise((resolve) => this.#drained.push(resolve)) + await this.#drain() return await work() } finally { this.#closed = false } } + + /** `exclusive` for Effect work; closing is uninterruptible and reopening always runs. */ + exclusiveEffect( + work: Effect.Effect, + ): Effect.Effect { + return Effect.acquireUseRelease( + Effect.suspend(() => { + if (this.#closed) return Effect.fail(MaintenanceInProgressError.create()) + this.#closed = true + return Effect.void + }), + () => Effect.promise(() => this.#drain()).pipe(Effect.andThen(work)), + () => + Effect.sync(() => { + this.#closed = false + }), + ) + } } class MaintenanceInProgressError extends Schema.TaggedError()( @@ -646,57 +628,83 @@ class RequestBodyTooLargeError extends Schema.TaggedError => { - const contentLength = req.headers.get("content-length") - if (contentLength !== null && /^[0-9]+$/.test(contentLength)) { - const declared = Number(contentLength) - if (!Number.isSafeInteger(declared) || declared > maximumBytes) - throw RequestBodyTooLargeError.create(maximumBytes) - } - if (req.body === null) return Schema.decodeUnknownSync(Schema.fromJsonString(Schema.Unknown))("") - const reader = req.body.getReader() - const chunks: Uint8Array[] = [] - let total = 0 - try { - while (true) { - const { done, value } = await reader.read() - if (done) break - total += value.byteLength - if (total > maximumBytes) { - await reader.cancel() - throw RequestBodyTooLargeError.create(maximumBytes) - } - chunks.push(value) +/** The body could not be read or is not JSON. */ +class RequestBodyInvalidError extends Schema.TaggedError()( + "@maple/cli/RequestBodyInvalid", + { message: Schema.String, cause: Schema.Defect() }, +) {} + +const bodyInvalid = (cause: unknown): RequestBodyInvalidError => + new RequestBodyInvalidError({ message: describeThrown(cause), cause }) + +const decodeJsonBody = (body: string): Effect.Effect => + Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Unknown))(body).pipe(Effect.mapError(bodyInvalid)) + +const readBoundedJson = ( + req: Request, + maximumBytes: number, +): Effect.Effect => + Effect.gen(function* () { + const contentLength = req.headers.get("content-length") + if (contentLength !== null && /^[0-9]+$/.test(contentLength)) { + const declared = Number(contentLength) + if (!Number.isSafeInteger(declared) || declared > maximumBytes) + return yield* RequestBodyTooLargeError.create(maximumBytes) } - } finally { - reader.releaseLock() - } - const bytes = new Uint8Array(total) - let offset = 0 - for (const chunk of chunks) { - bytes.set(chunk, offset) - offset += chunk.byteLength - } - return Schema.decodeUnknownSync(Schema.fromJsonString(Schema.Unknown))(new TextDecoder().decode(bytes)) -} + const body = req.body + if (body === null) return yield* decodeJsonBody("") + const bytes = yield* Effect.acquireUseRelease( + Effect.sync(() => body.getReader()), + (reader) => + Effect.gen(function* () { + const chunks: Uint8Array[] = [] + let total = 0 + while (true) { + const { done, value } = yield* Effect.tryPromise({ + try: () => reader.read(), + catch: bodyInvalid, + }) + if (done) break + total += value.byteLength + if (total > maximumBytes) { + yield* Effect.tryPromise({ try: () => reader.cancel(), catch: bodyInvalid }) + return yield* RequestBodyTooLargeError.create(maximumBytes) + } + chunks.push(value) + } + const bytes = new Uint8Array(total) + let offset = 0 + for (const chunk of chunks) { + bytes.set(chunk, offset) + offset += chunk.byteLength + } + return bytes + }), + (reader) => Effect.sync(() => reader.releaseLock()), + ) + return yield* decodeJsonBody(new TextDecoder().decode(bytes)) + }) -const recoverMaintenanceError = (error: unknown, fallback: Response): Response => { - if (error instanceof MaintenanceInProgressError) return text(error.message, 409) - if (error instanceof RequestBodyTooLargeError) return text(error.message, 413) - return fallback -} -const invalidJsonResponse = (error: unknown): Response => - recoverMaintenanceError(error, text("invalid JSON body", 400)) +/** The 409 and 413 a maintenance request reports on purpose; any other body failure is a 400. */ +const recoverBodyFailure = ( + self: Effect.Effect, +): Effect.Effect => + self.pipe( + Effect.catchTags({ + "@maple/cli/RequestBodyTooLarge": (error) => Effect.succeed(text(error.message, 413)), + "@maple/cli/RequestBodyInvalid": () => Effect.succeed(text("invalid JSON body", 400)), + }), + ) -const admitted = async (gate: RequestQuiescenceGate, work: () => Promise): Promise => { - const leave = gate.enter() - if (!leave) return text("server maintenance in progress", 503) - try { - return await work() - } finally { - leave() - } -} +const admitted = ( + gate: RequestQuiescenceGate, + work: Effect.Effect, +): Effect.Effect => + Effect.acquireUseRelease( + Effect.sync(() => gate.enter()), + (leave) => (leave ? work : Effect.succeed(text("server maintenance in progress", 503))), + (leave) => Effect.sync(() => leave?.()), + ) const handleRetirement = async ( db: Chdb, @@ -749,47 +757,64 @@ const MAX_CHECKPOINT_BODY_BYTES = 4 * 1024 const MAX_PROJECTION_BODY_BYTES = 512 * 1024 const MAX_CONSUMER_BODY_BYTES = 16 * 1024 +/** The chDB half of a checkpoint backup failed. */ +class CheckpointBackupFailed extends Schema.TaggedError()( + "@maple/cli/CheckpointBackupFailed", + { message: Schema.String, cause: Schema.Defect() }, +) {} + /** Typed, authenticated replacement for sending BACKUP through /local/query. */ -const handleCheckpointBackup = async ( +const handleCheckpointBackup = ( db: Chdb, - controlStore: LocalEventingControlStore, dataDir: string, gate: RequestQuiescenceGate, token: string, req: Request, -): Promise => { +): Effect.Effect => { if (!maintenanceTokenMatches(token, req.headers.get("x-maple-maintenance-token"))) - return text("maintenance authorization required", 403) - let body: unknown - try { - body = await readBoundedJson(req, MAX_CHECKPOINT_BODY_BYTES) - } catch (error) { - return invalidJsonResponse(error) - } - const decoded = Schema.decodeUnknownResult( - Schema.Struct({ - checkpointId: Schema.String.check(Schema.isPattern(CHECKPOINT_ID)), - }), - { onExcessProperty: "error" }, - )(body) - if (Result.isFailure(decoded)) return text("invalid checkpoint fields", 400) - const record = decoded.success - try { - const checkpointId = record.checkpointId.toLowerCase() - const controlBytes = await gate.exclusive(async () => { - // Both captures are synchronous: no request can mutate either database between them. - const bytes = controlStore.captureSnapshot() - db.exec(`BACKUP DATABASE default TO Disk('default', 'backups/snapshots/${checkpointId}/backup')`) - return bytes - }) - const control = await LocalEventingControlStore.writeSnapshot( + return Effect.succeed(text("maintenance authorization required", 403)) + return Effect.gen(function* () { + const body = yield* readBoundedJson(req, MAX_CHECKPOINT_BODY_BYTES) + const decoded = Schema.decodeUnknownResult( + Schema.Struct({ + checkpointId: Schema.String.check(Schema.isPattern(CHECKPOINT_ID)), + }), + { onExcessProperty: "error" }, + )(body) + if (Result.isFailure(decoded)) return text("invalid checkpoint fields", 400) + const checkpointId = decoded.success.checkpointId.toLowerCase() + const controlStore = yield* LocalEventingControlStore + // The gate has drained every admitted request, so neither database changes between captures. + const controlBytes = yield* gate.exclusiveEffect( + controlStore.captureSnapshot.pipe( + Effect.tap(() => + Effect.try({ + try: () => + db.exec( + `BACKUP DATABASE default TO Disk('default', 'backups/snapshots/${checkpointId}/backup')`, + ), + catch: (cause) => + new CheckpointBackupFailed({ message: describeThrown(cause), cause }), + }), + ), + ), + ) + const control = yield* writeControlSnapshot( eventingControlSnapshotPath(dataDir, checkpointId), controlBytes, ) return json({ checkpointId, control }) - } catch (error) { - return recoverMaintenanceError(error, text(`checkpoint backup failed: ${describeThrown(error)}`, 400)) - } + }).pipe( + Effect.catchTags({ + "@maple/cli/MaintenanceInProgress": (error) => Effect.succeed(text(error.message, 409)), + "@maple/cli/RequestBodyTooLarge": (error) => Effect.succeed(text(error.message, 413)), + "@maple/cli/RequestBodyInvalid": () => Effect.succeed(text("invalid JSON body", 400)), + "@maple/cli/CheckpointBackupFailed": (error) => + Effect.succeed(text(`checkpoint backup failed: ${error.message}`, 400)), + "@maple/cli/eventing/ControlStoreFailed": (error) => + Effect.succeed(text(`checkpoint backup failed: ${error.message}`, 400)), + }), + ) } const eventingAuthorized = (token: string, req: Request): Response | null => @@ -797,233 +822,235 @@ const eventingAuthorized = (token: string, req: Request): Response | null => ? null : text("maintenance authorization required", 403) -const handleProjectionActivation = async ( - eventing: LocalEventingRuntime, +const handleProjectionActivation = ( gate: RequestQuiescenceGate, token: string, req: Request, -): Promise => { +): Effect.Effect => { const unauthorized = eventingAuthorized(token, req) - if (unauthorized) return unauthorized - let body: unknown - try { - body = await readBoundedJson(req, MAX_PROJECTION_BODY_BYTES) - } catch (error) { - return invalidJsonResponse(error) - } - let activation - try { + if (unauthorized) return Effect.succeed(unauthorized) + return Effect.gen(function* () { + const eventing = yield* LocalEventingRuntime + const body = yield* readBoundedJson(req, MAX_PROJECTION_BODY_BYTES) // Recursive schema validation and full registry compilation happen while // normal ingest/query admission remains open. - activation = eventing.prepareActivation(body) - } catch (error) { - return text(`invalid event projection: ${describeThrown(error)}`, 400) - } - try { - await gate.exclusive(async () => eventing.commitActivation(activation)) - return json({ active: eventing.listActive() }) - } catch (error) { - if (error instanceof ProjectionActivationConflict) return text(error.message, 409) - if (error instanceof ProjectionActivationInvalid) return text(error.message, 400) - return recoverMaintenanceError( - error, - text(`event projection activation failed: ${describeThrown(error)}`, 500), + const activation = yield* eventing.prepareActivation(body).pipe(Effect.result) + if (Result.isFailure(activation)) + return text(`invalid event projection: ${activation.failure.message}`, 400) + return yield* gate.exclusiveEffect(eventing.commitActivation(activation.success)).pipe( + Effect.andThen(eventing.listActive), + Effect.map((active) => json({ active })), + Effect.catchTags({ + "@maple/cli/eventing/ProjectionActivationConflict": (error) => + Effect.succeed(text(error.message, 409)), + "@maple/cli/MaintenanceInProgress": (error) => Effect.succeed(text(error.message, 409)), + "@maple/cli/eventing/ControlStoreFailed": (error) => + Effect.succeed(text(`event projection activation failed: ${error.message}`, 500)), + }), + Effect.catchDefect((defect) => + Effect.succeed(text(`event projection activation failed: ${describeThrown(defect)}`, 500)), + ), ) - } + }).pipe(recoverBodyFailure) } -const eventConsumerErrorResponse = (error: unknown): Response => { - if (error instanceof EventConsumerDeliveryGapError) - return json( - { - error: error._tag, - message: error.message, - consumerId: error.consumerId, - generation: error.generation, - droppedEvents: error.droppedEvents, - }, - 409, - ) - if (error instanceof OutboxAdministrationInvalid || error instanceof EventConsumerInputError) - return text(error.message, 400) - if (error instanceof EventConsumerNotFoundError) return text(error.message, 404) - if (error instanceof EventConsumerConflictError || error instanceof EventConsumerLeaseError) - return text(error.message, 409) - return text(`event consumer operation failed: ${describeThrown(error)}`, 500) -} +const recoverEventConsumerFailure = ( + self: Effect.Effect, +): Effect.Effect => + self.pipe( + Effect.catchTags({ + "@maple/cli/eventing/EventConsumerDeliveryGap": (error) => + Effect.succeed( + json( + { + error: error._tag, + message: error.message, + consumerId: error.consumerId, + generation: error.generation, + droppedEvents: error.droppedEvents, + }, + 409, + ), + ), + "@maple/cli/eventing/EventConsumerInputInvalid": (error) => + Effect.succeed(text(error.message, 400)), + "@maple/cli/eventing/EventConsumerNotFound": (error) => Effect.succeed(text(error.message, 404)), + "@maple/cli/eventing/EventConsumerConflict": (error) => Effect.succeed(text(error.message, 409)), + "@maple/cli/eventing/EventConsumerLeaseConflict": (error) => + Effect.succeed(text(error.message, 409)), + "@maple/cli/eventing/ControlStoreFailed": (error) => + Effect.succeed(text(`event consumer operation failed: ${error.message}`, 500)), + }), + Effect.catchDefect((defect) => + Effect.succeed(text(`event consumer operation failed: ${describeThrown(defect)}`, 500)), + ), + ) const ConsumerIdSchema = Schema.String.check(Schema.isPattern(/^[a-z][a-z0-9._-]{0,63}$/)) -const handleConsumerRegistration = async ( - eventing: LocalEventingRuntime, +/** Decodes a bounded consumer body, then runs the store call under ordinary admission. */ +const consumerRequest = ( + gate: RequestQuiescenceGate, + req: Request, + schema: S, + invalidMessage: string, + run: (eventing: LocalEventingRuntimeApi, decoded: S["Type"]) => Effect.Effect, + status = 200, +): Effect.Effect => + Effect.gen(function* () { + const eventing = yield* LocalEventingRuntime + const body = yield* readBoundedJson(req, MAX_CONSUMER_BODY_BYTES) + const decoded = Schema.decodeUnknownResult(schema, { onExcessProperty: "error" })(body) + if (Result.isFailure(decoded)) return text(invalidMessage, 400) + return yield* admitted( + gate, + run(eventing, decoded.success).pipe( + Effect.map((result) => json(result, status)), + recoverEventConsumerFailure, + ), + ) + }).pipe(recoverBodyFailure) + +const handleConsumerRegistration = ( gate: RequestQuiescenceGate, maintenanceToken: string, req: Request, -): Promise => { - const unauthorized = eventingAuthorized(maintenanceToken, req) - if (unauthorized) return unauthorized - let body: unknown - try { - body = await readBoundedJson(req, MAX_CONSUMER_BODY_BYTES) - } catch (error) { - return invalidJsonResponse(error) - } - const decoded = Schema.decodeUnknownResult( - Schema.Struct({ consumerId: ConsumerIdSchema, startAt: Schema.Literals(["beginning", "latest"]) }), - { onExcessProperty: "error" }, - )(body) - if (Result.isFailure(decoded)) return text("invalid event consumer registration fields", 400) - const { consumerId, startAt } = decoded.success - return admitted(gate, async () => { - try { - return json(eventing.registerConsumer(consumerId, startAt), 201) - } catch (error) { - return eventConsumerErrorResponse(error) - } +): Effect.Effect => + Effect.suspend(() => { + const unauthorized = eventingAuthorized(maintenanceToken, req) + if (unauthorized) return Effect.succeed(unauthorized) + return consumerRequest( + gate, + req, + Schema.Struct({ + consumerId: ConsumerIdSchema, + startAt: Schema.Literals(["beginning", "latest"]), + }), + "invalid event consumer registration fields", + (eventing, { consumerId, startAt }) => eventing.registerConsumer(consumerId, startAt), + 201, + ) }) -} -const handleConsumerDisable = async ( - eventing: LocalEventingRuntime, +const handleConsumerDisable = ( gate: RequestQuiescenceGate, maintenanceToken: string, req: Request, -): Promise => { - const unauthorized = eventingAuthorized(maintenanceToken, req) - if (unauthorized) return unauthorized - let body: unknown - try { - body = await readBoundedJson(req, MAX_CONSUMER_BODY_BYTES) - } catch (error) { - return invalidJsonResponse(error) - } - const decoded = Schema.decodeUnknownResult(Schema.Struct({ consumerId: ConsumerIdSchema }), { - onExcessProperty: "error", - })(body) - if (Result.isFailure(decoded)) return text("invalid event consumer disable fields", 400) - const { consumerId } = decoded.success - return admitted(gate, async () => { - try { - return json(eventing.disableConsumer(consumerId)) - } catch (error) { - return eventConsumerErrorResponse(error) - } +): Effect.Effect => + Effect.suspend(() => { + const unauthorized = eventingAuthorized(maintenanceToken, req) + if (unauthorized) return Effect.succeed(unauthorized) + return consumerRequest( + gate, + req, + Schema.Struct({ consumerId: ConsumerIdSchema }), + "invalid event consumer disable fields", + (eventing, { consumerId }) => eventing.disableConsumer(consumerId), + ) }) -} -const handleConsumerClaim = async ( - eventing: Pick, +const consumerUnauthorized = (consumerToken: string, req: Request): Response | null => + eventConsumerTokenMatches(consumerToken, req.headers.get("x-maple-event-consumer-token")) + ? null + : text("event consumer authorization required", 403) + +const handleConsumerClaim = ( gate: RequestQuiescenceGate, consumerToken: string, req: Request, -): Promise => { - if (!eventConsumerTokenMatches(consumerToken, req.headers.get("x-maple-event-consumer-token"))) - return text("event consumer authorization required", 403) - let body: unknown - try { - body = await readBoundedJson(req, MAX_CONSUMER_BODY_BYTES) - } catch (error) { - return invalidJsonResponse(error) - } - const decoded = Schema.decodeUnknownResult( - Schema.Struct({ - consumerId: ConsumerIdSchema, - limit: Schema.Int.check(Schema.isBetween({ minimum: 1, maximum: 1000 })), - leaseSeconds: Schema.Int.check(Schema.isBetween({ minimum: 5, maximum: 300 })), - }), - { onExcessProperty: "error" }, - )(body) - if (Result.isFailure(decoded)) return text("invalid event consumer claim fields", 400) - const { consumerId, limit, leaseSeconds } = decoded.success - return admitted(gate, async () => { - try { - return json(eventing.claimReady(consumerId, limit, leaseSeconds)) - } catch (error) { - return eventConsumerErrorResponse(error) - } +): Effect.Effect => + Effect.suspend(() => { + const unauthorized = consumerUnauthorized(consumerToken, req) + if (unauthorized) return Effect.succeed(unauthorized) + return consumerRequest( + gate, + req, + Schema.Struct({ + consumerId: ConsumerIdSchema, + limit: Schema.Int.check(Schema.isBetween({ minimum: 1, maximum: 1000 })), + leaseSeconds: Schema.Int.check(Schema.isBetween({ minimum: 5, maximum: 300 })), + }), + "invalid event consumer claim fields", + (eventing, { consumerId, limit, leaseSeconds }) => + eventing.claimReady(consumerId, limit, leaseSeconds), + ) }) -} -const handleConsumerAcknowledgement = async ( - eventing: LocalEventingRuntime, +const handleConsumerAcknowledgement = ( gate: RequestQuiescenceGate, consumerToken: string, req: Request, -): Promise => { - if (!eventConsumerTokenMatches(consumerToken, req.headers.get("x-maple-event-consumer-token"))) - return text("event consumer authorization required", 403) - let body: unknown - try { - body = await readBoundedJson(req, MAX_CONSUMER_BODY_BYTES) - } catch (error) { - return invalidJsonResponse(error) - } - const decoded = Schema.decodeUnknownResult( - Schema.Struct({ - consumerId: ConsumerIdSchema, - leaseToken: Schema.String.check(Schema.isPattern(/^[0-9a-f]{64}$/)), - throughSequence: Schema.Int.check(Schema.isGreaterThanOrEqualTo(1)), - }), - { onExcessProperty: "error" }, - )(body) - if (Result.isFailure(decoded)) return text("invalid event consumer acknowledgement fields", 400) - const { consumerId, leaseToken, throughSequence } = decoded.success - return admitted(gate, async () => { - try { - return json(eventing.acknowledgeClaim(consumerId, leaseToken, throughSequence)) - } catch (error) { - return eventConsumerErrorResponse(error) - } +): Effect.Effect => + Effect.suspend(() => { + const unauthorized = consumerUnauthorized(consumerToken, req) + if (unauthorized) return Effect.succeed(unauthorized) + return consumerRequest( + gate, + req, + Schema.Struct({ + consumerId: ConsumerIdSchema, + leaseToken: Schema.String.check(Schema.isPattern(/^[0-9a-f]{64}$/)), + throughSequence: Schema.Int.check(Schema.isGreaterThanOrEqualTo(1)), + }), + "invalid event consumer acknowledgement fields", + (eventing, { consumerId, leaseToken, throughSequence }) => + eventing.acknowledgeClaim(consumerId, leaseToken, throughSequence), + ) }) -} -const handleOutboxAdministration = async ( - eventing: Pick, +const handleOutboxAdministration = ( gate: RequestQuiescenceGate, token: string, req: Request, action: "abandon" | "accept-gap", -): Promise => { +): Effect.Effect => { const unauthorized = eventingAuthorized(token, req) - if (unauthorized) return unauthorized - let body: unknown - try { - body = await readBoundedJson(req, MAX_PROJECTION_BODY_BYTES) - } catch (error) { - return invalidJsonResponse(error) - } - if (action === "abandon") { + if (unauthorized) return Effect.succeed(unauthorized) + return Effect.gen(function* () { + const eventing = yield* LocalEventingRuntime + const body = yield* readBoundedJson(req, MAX_PROJECTION_BODY_BYTES) + if (action === "abandon") { + const decoded = Schema.decodeUnknownResult( + Schema.Struct({ + eventIds: Schema.Array(Schema.NonEmptyString.check(Schema.isMaxLength(256))).check( + Schema.isMinLength(1), + Schema.isMaxLength(1000), + ), + }), + { onExcessProperty: "error" }, + )(body) + if (Result.isFailure(decoded)) return text("invalid outbox abandonment fields", 400) + return yield* gate.exclusiveEffect(eventing.abandonEvents(decoded.success.eventIds)).pipe( + Effect.map((result) => json(result)), + Effect.catchTags({ + "@maple/cli/MaintenanceInProgress": (error) => Effect.succeed(text(error.message, 409)), + "@maple/cli/eventing/OutboxAdministrationInvalid": (error) => + Effect.succeed(text(error.message, 400)), + "@maple/cli/eventing/ControlStoreFailed": (error) => + Effect.succeed(text(`event consumer operation failed: ${error.message}`, 500)), + }), + Effect.catchDefect((defect) => + Effect.succeed(text(`event consumer operation failed: ${describeThrown(defect)}`, 500)), + ), + ) + } const decoded = Schema.decodeUnknownResult( Schema.Struct({ - eventIds: Schema.Array(Schema.NonEmptyString.check(Schema.isMaxLength(256))).check( - Schema.isMinLength(1), - Schema.isMaxLength(1000), - ), + consumerId: ConsumerIdSchema, + generation: Schema.Int.check(Schema.isGreaterThan(0)), }), { onExcessProperty: "error" }, )(body) - if (Result.isFailure(decoded)) return text("invalid outbox abandonment fields", 400) - try { - return json(await gate.exclusive(async () => eventing.abandonEvents(decoded.success.eventIds))) - } catch (error) { - return recoverMaintenanceError(error, eventConsumerErrorResponse(error)) - } - } - const decoded = Schema.decodeUnknownResult( - Schema.Struct({ - consumerId: ConsumerIdSchema, - generation: Schema.Int.check(Schema.isGreaterThan(0)), - }), - { onExcessProperty: "error" }, - )(body) - if (Result.isFailure(decoded)) return text("invalid delivery gap acknowledgement fields", 400) - return admitted(gate, async () => { - try { - return json(eventing.acceptDeliveryGap(decoded.success.consumerId, decoded.success.generation)) - } catch (error) { - return eventConsumerErrorResponse(error) - } - }) + if (Result.isFailure(decoded)) return text("invalid delivery gap acknowledgement fields", 400) + const { consumerId, generation } = decoded.success + return yield* admitted( + gate, + eventing.acceptDeliveryGap(consumerId, generation).pipe( + Effect.map((gap) => json(gap)), + recoverEventConsumerFailure, + ), + ) + }).pipe(recoverBodyFailure) } const decodeOutboxQuery = Schema.decodeUnknownResult( @@ -1039,30 +1066,44 @@ const decodeOutboxQuery = Schema.decodeUnknownResult( { onExcessProperty: "error" }, ) +/** A read the store could not serve is a 500 naming the read. */ +const readFailed = + (label: string) => + (self: Effect.Effect): Effect.Effect => + self.pipe( + Effect.catchTag("@maple/cli/eventing/ControlStoreFailed", (error) => + Effect.succeed(text(`${label} failed: ${error.message}`, 500)), + ), + Effect.catchDefect((defect) => + Effect.succeed(text(`${label} failed: ${describeThrown(defect)}`, 500)), + ), + ) + const handleEventingRead = ( - eventing: LocalEventingRuntime, token: string, req: Request, url: URL, -): Response => { +): Effect.Effect => { const unauthorized = eventingAuthorized(token, req) - if (unauthorized) return unauthorized - if (url.pathname === "/local/eventing/health") return json(eventing.health()) - if (url.pathname === "/local/eventing/projections") return json(eventing.listActive()) - if (url.pathname === "/local/eventing/consumers") return json(eventing.listConsumers()) - if (url.pathname === "/local/eventing/outbox") { - const query = decodeOutboxQuery(Object.fromEntries(url.searchParams)) - if (Result.isFailure(query)) return text(`invalid outbox query: ${query.failure.message}`, 400) - const { state = "ready", limit = 100, after = 0 } = query.success - try { - return json( - state === "ready" ? eventing.listReady(limit, after) : eventing.listStaged(limit, after), - ) - } catch (error) { - return text(`outbox read failed: ${describeThrown(error)}`, 500) + if (unauthorized) return Effect.succeed(unauthorized) + return Effect.gen(function* () { + const eventing = yield* LocalEventingRuntime + if (url.pathname === "/local/eventing/health") + return yield* eventing.health.pipe(Effect.map(json), readFailed("eventing health read")) + if (url.pathname === "/local/eventing/projections") + return yield* eventing.listActive.pipe(Effect.map(json), readFailed("projection read")) + if (url.pathname === "/local/eventing/consumers") + return yield* eventing.listConsumers.pipe(Effect.map(json), readFailed("consumer read")) + if (url.pathname === "/local/eventing/outbox") { + const query = decodeOutboxQuery(Object.fromEntries(url.searchParams)) + if (Result.isFailure(query)) return text(`invalid outbox query: ${query.failure.message}`, 400) + const { state = "ready", limit = 100, after = 0 } = query.success + return yield* ( + state === "ready" ? eventing.listReady(limit, after) : eventing.listStaged(limit, after) + ).pipe(Effect.map(json), readFailed("outbox read")) } - } - return text("not found", 404) + return text("not found", 404) + }) } /** The `Bun.serve` fetch handler, closed over the chDB connection. Each ingest @@ -1077,8 +1118,6 @@ const makeFetch = gate: RequestQuiescenceGate, maintenanceToken: string, consumerToken: string, - controlStore: LocalEventingControlStore, - eventing: LocalEventingRuntime, ) => async (req: Request): Promise => { const url = new URL(req.url) @@ -1092,53 +1131,40 @@ const makeFetch = if (url.pathname === "/health") return respond(text("OK")) if (req.method === "POST") { if (url.pathname === "/v1/traces") - return respond( - await admitted(gate, () => ingestSpan(runSpan, db, authority, eventing, "traces", req)), - ) + return respond(await runSpan(admitted(gate, ingestSpan(db, authority, "traces", req)))) if (url.pathname === "/v1/logs") - return respond( - await admitted(gate, () => ingestSpan(runSpan, db, authority, eventing, "logs", req)), - ) + return respond(await runSpan(admitted(gate, ingestSpan(db, authority, "logs", req)))) if (url.pathname === "/v1/metrics") - return respond( - await admitted(gate, () => ingestSpan(runSpan, db, authority, eventing, "metrics", req)), - ) + return respond(await runSpan(admitted(gate, ingestSpan(db, authority, "metrics", req)))) if (url.pathname === "/local/query") - return respond(await admitted(gate, () => querySpan(runSpan, db, authority, req))) + return respond(await runSpan(admitted(gate, querySpan(db, authority, req)))) if (url.pathname === "/local/eventing/outbox/abandon") return respond( - await handleOutboxAdministration(eventing, gate, maintenanceToken, req, "abandon"), + await runSpan(handleOutboxAdministration(gate, maintenanceToken, req, "abandon")), ) if (url.pathname === "/local/eventing/consumers/accept-gap") return respond( - await handleOutboxAdministration(eventing, gate, maintenanceToken, req, "accept-gap"), + await runSpan(handleOutboxAdministration(gate, maintenanceToken, req, "accept-gap")), ) if (url.pathname === "/local/checkpoint/backup") return respond( - await handleCheckpointBackup( - db, - controlStore, - options.dataDir, - gate, - maintenanceToken, - req, - ), + await runSpan(handleCheckpointBackup(db, options.dataDir, gate, maintenanceToken, req)), ) if (url.pathname === "/local/eventing/projections") - return respond(await handleProjectionActivation(eventing, gate, maintenanceToken, req)) + return respond(await runSpan(handleProjectionActivation(gate, maintenanceToken, req))) if (url.pathname === "/local/eventing/consumers") - return respond(await handleConsumerRegistration(eventing, gate, maintenanceToken, req)) + return respond(await runSpan(handleConsumerRegistration(gate, maintenanceToken, req))) if (url.pathname === "/local/eventing/consumers/disable") - return respond(await handleConsumerDisable(eventing, gate, maintenanceToken, req)) + return respond(await runSpan(handleConsumerDisable(gate, maintenanceToken, req))) if (url.pathname === "/local/eventing/claims") - return respond(await handleConsumerClaim(eventing, gate, consumerToken, req)) + return respond(await runSpan(handleConsumerClaim(gate, consumerToken, req))) if (url.pathname === "/local/eventing/acks") - return respond(await handleConsumerAcknowledgement(eventing, gate, consumerToken, req)) + return respond(await runSpan(handleConsumerAcknowledgement(gate, consumerToken, req))) if (url.pathname === "/local/retention/retire") return respond(await handleRetirement(db, authority, gate, maintenanceToken, req)) } if (req.method === "GET" && url.pathname.startsWith("/local/eventing/")) - return respond(handleEventingRead(eventing, maintenanceToken, req, url)) + return respond(await runSpan(handleEventingRead(maintenanceToken, req, url))) if (req.method === "GET" && options.assets) return respond(serveAsset(options.assets, url.pathname)) return respond(text("not found", 404)) } @@ -1178,44 +1204,28 @@ export const startServer = ( configFile: options.configFile, rawTelemetryRetentionDays: retention.effective, }) - // The request handler and synchronous eventing store share one telemetry - // runtime; eventing observations contain only bounded operation labels. - const telemetry = yield* Effect.acquireRelease( - Effect.sync(() => ManagedRuntime.make(TelemetryLayer)), + // Request spans, eventing metrics, and the eventing services share one runtime; + // disposing it closes the control store (WAL checkpoint) and flushes telemetry. + const eventingLayer = Layer.mergeAll( + LocalEventingRuntime.layer, + LocalEventingControlStore.layer, + ).pipe(Layer.provide(Layer.succeed(LocalEventingControlConfig, { dataDir: options.dataDir }))) + const runtime = yield* Effect.acquireRelease( + Effect.sync(() => ManagedRuntime.make(Layer.mergeAll(TelemetryLayer, eventingLayer))), (rt) => Effect.promise(() => rt.dispose()), ) - const eventingTelemetry = makeEffectEventingTelemetry((effect) => { - telemetry.runFork(effect) - }) - const controlStore = yield* Effect.acquireRelease( - Effect.tryPromise({ - try: () => LocalEventingControlStore.open(options.dataDir, undefined, eventingTelemetry), - catch: (error) => + yield* runtime.contextEffect.pipe( + Effect.mapError( + (error) => new EventingStartupError({ cause: error, - message: `failed to open local eventing control store: ${describeThrown(error)}`, + message: + error._tag === "@maple/cli/eventing/ControlStoreFailed" + ? `failed to open local eventing control store: ${error.message}` + : `failed to compile local event projections: ${error.message}`, }), - }), - (store) => - Effect.try({ - try: () => store.close(), - catch: (cause) => - new EventingStartupError({ - message: "failed to close eventing control store", - cause, - }), - }).pipe( - Effect.catchTag("@maple/cli/eventing/StartupFailed", (error) => Effect.logError(error)), - ), + ), ) - const eventing = yield* Effect.try({ - try: () => new LocalEventingRuntime(controlStore, eventingTelemetry), - catch: (error) => - new EventingStartupError({ - cause: error, - message: `failed to compile local event projections: ${describeThrown(error)}`, - }), - }) // `CREATE ... IF NOT EXISTS` does not repair a table whose physical // definition was altered out of band. Inspect the opened store before the // listener is bound; a mismatch fails startup rather than allowing new @@ -1274,16 +1284,17 @@ export const startServer = ( message: `failed to load maintenance token: ${error instanceof Error ? error.message : String(error)}`, }), }) - const consumerToken = yield* Effect.tryPromise({ - try: () => ensureEventConsumerToken(options.dataDir), - catch: (error) => - new EventingStartupError({ - cause: error, - message: `failed to load event consumer token: ${describeThrown(error)}`, - }), - }) + const consumerToken = yield* ensureEventConsumerToken(options.dataDir).pipe( + Effect.mapError( + (error) => + new EventingStartupError({ + cause: error, + message: `failed to load event consumer token: ${error.message}`, + }), + ), + ) const gate = new RequestQuiescenceGate() - const runSpan: SpanRunner = (effect) => telemetry.runPromise(effect) + const runSpan: SpanRunner = (effect) => runtime.runPromise(effect) const server = yield* Effect.acquireRelease( Effect.try({ try: () => @@ -1298,8 +1309,6 @@ export const startServer = ( gate, maintenanceToken, consumerToken, - controlStore, - eventing, ), }), catch: (error) => diff --git a/apps/cli/test/checkpoints.test.ts b/apps/cli/test/checkpoints.test.ts index 49e739bd1..6ee4dd8c1 100644 --- a/apps/cli/test/checkpoints.test.ts +++ b/apps/cli/test/checkpoints.test.ts @@ -60,7 +60,8 @@ import { import { SCHEMA_FINGERPRINT } from "../src/server/schema-identity" import { storeMarkerPath, storeOpenMarkerPath } from "../src/server/store-version" import { CHDB_VERSION, MAPLE_VERSION } from "../src/version" -import { eventingControlSnapshotPath, LocalEventingControlStore } from "../src/server/eventing/control-store" +import { eventingControlSnapshotPath } from "../src/server/eventing/control-store" +import { openStore, runAsync } from "./eventing-test-support" const withDataDir = async (run: (dataDir: string) => Promise | void): Promise => { const parent = mkdtempSync(join(tmpdir(), "maple-checkpoint-test-")) @@ -356,10 +357,10 @@ describe("checkpoint state resolution", () => { mkdirSync(join(snapshot, "backup"), { recursive: true }) writeFileSync(join(snapshot, "backup", "data.bin"), "backup") - const store = await LocalEventingControlStore.open(dataDir) + const store = await openStore(dataDir) const controlPath = eventingControlSnapshotPath(dataDir, checkpointId) - const controlValidation = await store.backupTo(controlPath) - store.close() + const controlValidation = await runAsync(store.backupTo(controlPath)) + await store.close() const controlBytes = readFileSync(controlPath) writeFileSync( join(snapshot, "manifest.json"), diff --git a/apps/cli/test/eventing-test-support.ts b/apps/cli/test/eventing-test-support.ts new file mode 100644 index 000000000..5c49ae5bf --- /dev/null +++ b/apps/cli/test/eventing-test-support.ts @@ -0,0 +1,82 @@ +import { Context, Effect, Exit, Metric, Scope } from "effect" +import type { ProjectorRegistry } from "@maple/eventing-core" +import { + LocalEventingControlStore, + openControlStore, + type LocalEventingControlLimits, + type LocalEventingControlStoreApi, +} from "../src/server/eventing/control-store" +import { + LocalEventingProjectors, + LocalEventingRuntime, + type LocalEventingRuntimeApi, +} from "../src/server/eventing/runtime" + +/** Runs a synchronous store or runtime step, throwing its typed failure like the old API did. */ +export const run = (effect: Effect.Effect): A => Effect.runSync(effect) + +export const runAsync = (effect: Effect.Effect): Promise => Effect.runPromise(effect) + +export interface OpenedStore extends LocalEventingControlStoreApi { + /** Closes the store's scope: the WAL checkpoint and database close. */ + readonly close: () => Promise +} + +export const openStore = async ( + dataDir: string, + limits?: LocalEventingControlLimits, +): Promise => { + const scope = Effect.runSync(Scope.make()) + const store = await Effect.runPromise( + openControlStore(dataDir, limits).pipe( + Scope.provide(scope), + Effect.onError(() => Scope.close(scope, Exit.void)), + ), + ) + return { ...store, close: () => Effect.runPromise(Scope.close(scope, Exit.void)) } +} + +export const makeRuntime = ( + store: LocalEventingControlStoreApi, + projectors?: ProjectorRegistry, +): LocalEventingRuntimeApi => { + const make = LocalEventingRuntime.make.pipe(Effect.provideService(LocalEventingControlStore, store)) + return Effect.runSync( + projectors === undefined + ? make + : make.pipe(Effect.provideService(LocalEventingProjectors, projectors)), + ) +} + +/** An isolated metric registry; `observe` runs an effect against it. */ +export const metricRecorder = () => { + const registry = new Map>() + const context = Context.make(Metric.MetricRegistry, registry) + return { + observe: (effect: Effect.Effect): Effect.Effect => + Effect.provideService(effect, Metric.MetricRegistry, registry), + /** `operation:outcome` pairs of every eventing counter with a non-zero count. */ + operationOutcomes: (): string[] => + Metric.snapshotUnsafe(context).flatMap((snapshot) => + snapshot.id === "maple.eventing.operations_total" && + snapshot.type === "Counter" && + Number(snapshot.state.count) > 0 + ? [`${snapshot.attributes?.operation}:${snapshot.attributes?.outcome}`] + : [], + ), + attributes: (): string => + JSON.stringify(Metric.snapshotUnsafe(context).map(({ attributes }) => attributes)), + } +} + +/** Runs a serve.ts request effect against a runtime (real or a partial stub). */ +export const serveWith = ( + eventing: LocalEventingRuntimeApi, + effect: Effect.Effect, +): Promise => Effect.runPromise(Effect.provideService(effect, LocalEventingRuntime, eventing)) + +/** Runs the checkpoint backup handler against a control store (real or a stub). */ +export const serveCheckpointWith = ( + store: LocalEventingControlStoreApi, + effect: Effect.Effect, +): Promise => Effect.runPromise(Effect.provideService(effect, LocalEventingControlStore, store)) diff --git a/apps/cli/test/local-eventing-consumer-auth.test.ts b/apps/cli/test/local-eventing-consumer-auth.test.ts index 6e264bdcd..24fe47fef 100644 --- a/apps/cli/test/local-eventing-consumer-auth.test.ts +++ b/apps/cli/test/local-eventing-consumer-auth.test.ts @@ -2,6 +2,7 @@ import { strictEqual } from "node:assert" import { mkdirSync, mkdtempSync, rmSync, statSync, symlinkSync } from "node:fs" import { tmpdir } from "node:os" import { join } from "node:path" +import { Effect } from "effect" import { describe, it } from "vitest" import { ensureEventConsumerToken, @@ -15,8 +16,8 @@ describe("local event consumer authorization", () => { const dataDir = join(parent, "data") mkdirSync(dataDir) try { - const first = await ensureEventConsumerToken(dataDir) - const second = await ensureEventConsumerToken(dataDir) + const first = await Effect.runPromise(ensureEventConsumerToken(dataDir)) + const second = await Effect.runPromise(ensureEventConsumerToken(dataDir)) strictEqual(first.length, 64) strictEqual(second, first) strictEqual(statSync(eventConsumerTokenPath(dataDir)).mode & 0o777, 0o600) @@ -36,7 +37,7 @@ describe("local event consumer authorization", () => { symlinkSync(join(parent, "target"), eventConsumerTokenPath(dataDir)) let message = "" try { - await ensureEventConsumerToken(dataDir) + await Effect.runPromise(ensureEventConsumerToken(dataDir)) } catch (error) { message = error instanceof Error ? error.message : String(error) } diff --git a/apps/cli/test/local-eventing-control-store.test.ts b/apps/cli/test/local-eventing-control-store.test.ts index 359701909..3523e191d 100644 --- a/apps/cli/test/local-eventing-control-store.test.ts +++ b/apps/cli/test/local-eventing-control-store.test.ts @@ -14,8 +14,14 @@ import { tmpdir } from "node:os" import { join } from "node:path" import { describe, it } from "vitest" import type { MapleCloudEvent, SignalProjectionSpec } from "@maple/eventing-core" -import { eventingControlPath, LocalEventingControlStore } from "../src/server/eventing/control-store" -import type { EventingTelemetryObservation } from "../src/server/eventing/telemetry" +import { Effect } from "effect" +import { + eventingControlPath, + restoreControlSnapshot, + validateControlSnapshot, + writeControlSnapshot, +} from "../src/server/eventing/control-store" +import { metricRecorder, openStore, run, runAsync } from "./eventing-test-support" const withDataDir = async (run: (dataDir: string) => Promise): Promise => { const parent = mkdtempSync(join(tmpdir(), "maple-eventing-control-")) @@ -67,44 +73,45 @@ const SOURCE_FINGERPRINT = `sha256:${"a".repeat(64)}` describe("LocalEventingControlStore", () => { it("records bounded outbox and consumer telemetry without identifiers or payloads", async () => withDataDir(async (dataDir) => { - const observations: EventingTelemetryObservation[] = [] - const store = await LocalEventingControlStore.open(dataDir, undefined, { - record: (observation) => observations.push(observation), - }) + const metrics = metricRecorder() + const observed = (effect: Effect.Effect): A => run(metrics.observe(effect)) + const store = await openStore(dataDir) try { const sensitiveEvent = event({ data: { recordId: 42, label: "PAYLOAD-MUST-NOT-BE-METRIC-DATA" }, }) - store.stageEvents([sensitiveEvent, sensitiveEvent]) - throws(() => store.stageEvents([event({ data: { recordId: 43 } })]), /collision/) - throws(() => store.markReady(["unknown-event-identifier"]), /unknown event/) - store.markReady([sensitiveEvent.id]) - store.registerConsumer("tenant-a", "private-consumer-identifier", "beginning") - const claim = store.claimReady("tenant-a", "private-consumer-identifier", 10, 30) + observed(store.stageEvents([sensitiveEvent, sensitiveEvent])) + throws(() => observed(store.stageEvents([event({ data: { recordId: 43 } })])), /collision/) + throws(() => observed(store.markReady(["unknown-event-identifier"])), /unknown event/) + observed(store.markReady([sensitiveEvent.id])) + observed(store.registerConsumer("tenant-a", "private-consumer-identifier", "beginning")) + const claim = observed(store.claimReady("tenant-a", "private-consumer-identifier", 10, 30)) throws( - () => store.claimReady("tenant-a", "private-consumer-identifier", 10, 30), + () => observed(store.claimReady("tenant-a", "private-consumer-identifier", 10, 30)), /active lease/, ) throws( () => - store.acknowledgeClaim( - "tenant-a", - "private-consumer-identifier", - "incorrect-private-token", - claim.throughSequence!, + observed( + store.acknowledgeClaim( + "tenant-a", + "private-consumer-identifier", + "incorrect-private-token", + claim.throughSequence!, + ), ), /token does not match/, ) - store.acknowledgeClaim( - "tenant-a", - "private-consumer-identifier", - claim.leaseToken!, - claim.throughSequence!, + observed( + store.acknowledgeClaim( + "tenant-a", + "private-consumer-identifier", + claim.leaseToken!, + claim.throughSequence!, + ), ) - const operationOutcomes = observations.map( - ({ operation, outcome }) => `${operation}:${outcome}`, - ) + const operationOutcomes = metrics.operationOutcomes() for (const expected of [ "outbox_stage:success", "outbox_stage:failure", @@ -120,7 +127,7 @@ describe("LocalEventingControlStore", () => { ]) ok(operationOutcomes.includes(expected), `missing telemetry observation ${expected}`) - const serialized = JSON.stringify(observations) + const serialized = metrics.attributes() for (const forbidden of [ "PAYLOAD-MUST-NOT-BE-METRIC-DATA", "private-consumer-identifier", @@ -130,37 +137,39 @@ describe("LocalEventingControlStore", () => { ]) strictEqual(serialized.includes(forbidden), false) } finally { - store.close() + await store.close() } })) it("stores immutable sequential revisions and only loads the active revision", async () => withDataDir(async (dataDir) => { - const store = await LocalEventingControlStore.open(dataDir) + const store = await openStore(dataDir) try { - store.saveProjection(projection()) - deepStrictEqual(store.loadEnabledProjections("tenant-a"), [projection()]) + run(store.saveProjection(projection())) + deepStrictEqual(run(store.loadEnabledProjections("tenant-a")), [projection()]) throws( () => - store.saveProjection( - projection({ projector: { id: "changed", version: 1, config: {} } }), + run( + store.saveProjection( + projection({ projector: { id: "changed", version: 1, config: {} } }), + ), ), /immutable/, ) - throws(() => store.saveProjection(projection({ revision: 3 })), /must be 2/) + throws(() => run(store.saveProjection(projection({ revision: 3 }))), /must be 2/) - store.saveProjection(projection({ revision: 2, enabled: false })) - deepStrictEqual(store.loadEnabledProjections("tenant-a"), []) - store.saveProjection(projection({ revision: 3 })) - deepStrictEqual(store.loadEnabledProjections("tenant-a"), [projection({ revision: 3 })]) + run(store.saveProjection(projection({ revision: 2, enabled: false }))) + deepStrictEqual(run(store.loadEnabledProjections("tenant-a")), []) + run(store.saveProjection(projection({ revision: 3 }))) + deepStrictEqual(run(store.loadEnabledProjections("tenant-a")), [projection({ revision: 3 })]) throws( - () => store.saveProjection(projection({ revision: 2, enabled: false })), + () => run(store.saveProjection(projection({ revision: 2, enabled: false }))), /stale projection revision/, ) - throws(() => store.saveProjection(projection()), /stale projection revision/) - store.saveProjection(projection({ revision: 3 })) - deepStrictEqual(store.loadEnabledProjections("tenant-a"), [projection({ revision: 3 })]) - deepStrictEqual(store.validate(), { + throws(() => run(store.saveProjection(projection())), /stale projection revision/) + run(store.saveProjection(projection({ revision: 3 }))) + deepStrictEqual(run(store.loadEnabledProjections("tenant-a")), [projection({ revision: 3 })]) + deepStrictEqual(run(store.validate), { schemaVersion: 1, projectionRevisions: 3, projectionFailures: 0, @@ -168,93 +177,99 @@ describe("LocalEventingControlStore", () => { readyEvents: 0, }) } finally { - store.close() + await store.close() } })) it("deduplicates staged events, rejects collisions, and preserves ready order", async () => withDataDir(async (dataDir) => { - const store = await LocalEventingControlStore.open(dataDir) + const store = await openStore(dataDir) try { - deepStrictEqual(store.stageEvents([event(), event()]), { + deepStrictEqual(run(store.stageEvents([event(), event()])), { inserted: 1, deduplicated: 1, dropped: 0, eventIds: [event().id, event().id], }) - throws(() => store.stageEvents([event({ data: { recordId: 43 } })]), /collision/) - throws(() => store.markReady(["unknown"]), /unknown event/) - store.markReady([event().id]) - store.markReady([event().id]) - deepStrictEqual(store.listStaged().events, []) + throws(() => run(store.stageEvents([event({ data: { recordId: 43 } })])), /collision/) + throws(() => run(store.markReady(["unknown"])), /unknown event/) + run(store.markReady([event().id])) + run(store.markReady([event().id])) + deepStrictEqual(run(store.listStaged()).events, []) deepStrictEqual( - store.listReady().events.map(({ event }) => event), + run(store.listReady()).events.map(({ event }) => event), [event()], ) } finally { - store.close() + await store.close() } })) it("binds staged source recovery to the normalized occurrence fingerprint", async () => withDataDir(async (dataDir) => { - const store = await LocalEventingControlStore.open(dataDir) + const store = await openStore(dataDir) try { - store.saveProjection(projection()) + run(store.saveProjection(projection())) const sourced = event({ sourceoccurrenceid: "record-42" }) - throws(() => store.stageEvents([sourced]), /requires a source fingerprint/) - store.stageEvents([sourced], new Map([[sourced.id, SOURCE_FINGERPRINT]])) + throws(() => run(store.stageEvents([sourced])), /requires a source fingerprint/) + run(store.stageEvents([sourced], new Map([[sourced.id, SOURCE_FINGERPRINT]]))) deepStrictEqual( - store.stagedEventIdsForOccurrence( - sourced.tenantid, - "otel.log", - sourced.source, - sourced.sourceoccurrenceid!, - SOURCE_FINGERPRINT, - ), - [sourced.id], - ) - throws( - () => + run( store.stagedEventIdsForOccurrence( sourced.tenantid, "otel.log", sourced.source, sourced.sourceoccurrenceid!, - `sha256:${"b".repeat(64)}`, + SOURCE_FINGERPRINT, + ), + ), + [sourced.id], + ) + throws( + () => + run( + store.stagedEventIdsForOccurrence( + sourced.tenantid, + "otel.log", + sourced.source, + sourced.sourceoccurrenceid!, + `sha256:${"b".repeat(64)}`, + ), ), /staged source occurrence collision/, ) - strictEqual(store.listStaged().events.length, 1) + strictEqual(run(store.listStaged()).events.length, 1) } finally { - store.close() + await store.close() } })) it("survives restart and round-trips through a validated standalone snapshot", async () => withDataDir(async (dataDir) => { - let store = await LocalEventingControlStore.open(dataDir) - store.saveProjection(projection()) - store.stageEvents([event()]) - store.markReady([event().id]) - store.recordProjectionFailures("tenant-a", [ - { - projectionId: "example-record-observed", - projectionRevision: 1, - occurrenceId: "record-42", - message: "test failure", - }, - ]) - store.close() + let store = await openStore(dataDir) + run(store.saveProjection(projection())) + run(store.stageEvents([event()])) + run(store.markReady([event().id])) + run( + store.recordProjectionFailures("tenant-a", [ + { + projectionId: "example-record-observed", + projectionRevision: 1, + occurrenceId: "record-42", + message: "test failure", + }, + ]), + ) + await store.close() - store = await LocalEventingControlStore.open(dataDir) - deepStrictEqual(store.loadEnabledProjections("tenant-a"), [projection()]) + store = await openStore(dataDir) + deepStrictEqual(run(store.loadEnabledProjections("tenant-a")), [projection()]) deepStrictEqual( - store.listReady().events.map(({ event }) => event), + run(store.listReady()).events.map(({ event }) => event), [event()], ) const snapshot = join(dataDir, "backups", "snapshot", "control.sqlite") - const validation = await store.backupTo(snapshot) + const validation = await runAsync(store.backupTo(snapshot)) deepStrictEqual(validation, { schemaVersion: 1, projectionRevisions: 1, @@ -262,168 +277,167 @@ describe("LocalEventingControlStore", () => { stagedEvents: 0, readyEvents: 1, }) - store.close() + await store.close() const restored = join(dataDir, "restored") - await LocalEventingControlStore.restoreSnapshot(snapshot, restored) - deepStrictEqual( - LocalEventingControlStore.validateSnapshot(eventingControlPath(restored)), - validation, - ) - const restoredStore = await LocalEventingControlStore.open(restored) + await runAsync(restoreControlSnapshot(snapshot, restored)) + deepStrictEqual(run(validateControlSnapshot(eventingControlPath(restored))), validation) + const restoredStore = await openStore(restored) try { - deepStrictEqual(restoredStore.loadEnabledProjections("tenant-a"), [projection()]) + deepStrictEqual(run(restoredStore.loadEnabledProjections("tenant-a")), [projection()]) deepStrictEqual( - restoredStore.listReady().events.map(({ event }) => event), + run(restoredStore.listReady()).events.map(({ event }) => event), [event()], ) } finally { - restoredStore.close() + await restoredStore.close() } })) it("writes the captured SQLite state even when the live store changes before archive I/O", async () => withDataDir(async (dataDir) => { - const store = await LocalEventingControlStore.open(dataDir) + const store = await openStore(dataDir) try { - store.saveProjection(projection()) - store.stageEvents([event()]) - const bytes = store.captureSnapshot() - store.markReady([event().id]) - store.saveProjection(projection({ revision: 2, enabled: false })) + run(store.saveProjection(projection())) + run(store.stageEvents([event()])) + const bytes = run(store.captureSnapshot) + run(store.markReady([event().id])) + run(store.saveProjection(projection({ revision: 2, enabled: false }))) const snapshot = join(dataDir, "backups", "captured", "control.sqlite") - const validation = await LocalEventingControlStore.writeSnapshot(snapshot, bytes) + const validation = await runAsync(writeControlSnapshot(snapshot, bytes)) strictEqual(validation.stagedEvents, 1) strictEqual(validation.readyEvents, 0) strictEqual(validation.projectionRevisions, 1) - strictEqual(store.validate().readyEvents, 1) - strictEqual(store.validate().projectionRevisions, 2) + strictEqual(run(store.validate).readyEvents, 1) + strictEqual(run(store.validate).projectionRevisions, 2) const restored = join(dataDir, "restored-capture") - await LocalEventingControlStore.restoreSnapshot(snapshot, restored) - const recovered = await LocalEventingControlStore.open(restored) + await runAsync(restoreControlSnapshot(snapshot, restored)) + const recovered = await openStore(restored) try { - deepStrictEqual(recovered.loadEnabledProjections("tenant-a"), [projection()]) + deepStrictEqual(run(recovered.loadEnabledProjections("tenant-a")), [projection()]) deepStrictEqual( - recovered.listStaged().events.map((row) => row.event), + run(recovered.listStaged()).events.map((row) => row.event), [event()], ) } finally { - recovered.close() + await recovered.close() } } finally { - store.close() + await store.close() } })) it("checkpoints committed live WAL state before serializing", async () => withDataDir(async (dataDir) => { - const store = await LocalEventingControlStore.open(dataDir) + const store = await openStore(dataDir) try { - store.saveProjection(projection()) - store.stageEvents([event()]) - store.markReady([event().id]) + run(store.saveProjection(projection())) + run(store.stageEvents([event()])) + run(store.markReady([event().id])) const walPath = `${eventingControlPath(dataDir)}-wal` ok(existsSync(walPath)) ok(statSync(walPath).size > 0, "test requires uncheckpointed WAL frames") const snapshot = join(dataDir, "backups", "live-wal", "control.sqlite") - await store.backupTo(snapshot) + await runAsync(store.backupTo(snapshot)) strictEqual(statSync(walPath).size, 0) const restored = join(dataDir, "restored-live-wal") - await LocalEventingControlStore.restoreSnapshot(snapshot, restored) - const restoredStore = await LocalEventingControlStore.open(restored) + await runAsync(restoreControlSnapshot(snapshot, restored)) + const restoredStore = await openStore(restored) try { - deepStrictEqual(restoredStore.loadEnabledProjections("tenant-a"), [projection()]) + deepStrictEqual(run(restoredStore.loadEnabledProjections("tenant-a")), [projection()]) deepStrictEqual( - restoredStore.listReady().events.map(({ event }) => event), + run(restoredStore.listReady()).events.map(({ event }) => event), [event()], ) } finally { - restoredStore.close() + await restoredStore.close() } } finally { - store.close() + await store.close() } })) it("paginates every ready event and reports bounded outbox overflow", async () => withDataDir(async (dataDir) => { - const store = await LocalEventingControlStore.open(dataDir, { + const store = await openStore(dataDir, { maxOutboxEvents: 2, maxOutboxBytes: 1024 * 1024, }) try { const second = event({ id: "event-2", data: { recordId: 43, label: "Second" } }) const third = event({ id: "event-3", data: { recordId: 44, label: "Third" } }) - const staged = store.stageEvents([event(), second]) - store.markReady(staged.eventIds) + const staged = run(store.stageEvents([event(), second])) + run(store.markReady(staged.eventIds)) - const firstPage = store.listReady(1) + const firstPage = run(store.listReady(1)) strictEqual(firstPage.events.length, 1) strictEqual(firstPage.nextCursor, firstPage.events[0]?.sequence) - const secondPage = store.listReady(1, firstPage.nextCursor!) + const secondPage = run(store.listReady(1, firstPage.nextCursor!)) deepStrictEqual( [...firstPage.events, ...secondPage.events].map(({ event }) => event.id), [event().id, second.id], ) strictEqual(secondPage.nextCursor, null) - deepStrictEqual(store.stageEvents([event()]).deduplicated, 1) - deepStrictEqual(store.stageEvents([third]), { + deepStrictEqual(run(store.stageEvents([event()])).deduplicated, 1) + deepStrictEqual(run(store.stageEvents([third])), { inserted: 0, deduplicated: 0, dropped: 1, eventIds: [], }) - strictEqual(store.deliveryGap("tenant-a").generation, 1) + strictEqual(run(store.deliveryGap("tenant-a")).generation, 1) } finally { - store.close() + await store.close() } })) it("pages recovered events by first readiness transition instead of staging order", async () => withDataDir(async (dataDir) => { - const store = await LocalEventingControlStore.open(dataDir) + const store = await openStore(dataDir) try { const first = event({ id: "event-a" }) const second = event({ id: "event-b" }) - store.stageEvents([first]) - store.stageEvents([second]) - store.markReady([second.id]) + run(store.stageEvents([first])) + run(store.stageEvents([second])) + run(store.markReady([second.id])) - const initialPage = store.listReady(1) + const initialPage = run(store.listReady(1)) deepStrictEqual( initialPage.events.map(({ event }) => event.id), [second.id], ) const cursor = initialPage.events[0]!.sequence - store.markReady([first.id]) - const recoveredPage = store.listReady(1, cursor) + run(store.markReady([first.id])) + const recoveredPage = run(store.listReady(1, cursor)) deepStrictEqual( recoveredPage.events.map(({ event }) => event.id), [first.id], ) strictEqual(recoveredPage.events[0]!.sequence > cursor, true) } finally { - store.close() + await store.close() } })) it("rejects invalid schema-1 staged fingerprints during snapshot validation", async () => withDataDir(async (dataDir) => { - const store = await LocalEventingControlStore.open(dataDir) - store.saveProjection(projection()) + const store = await openStore(dataDir) + run(store.saveProjection(projection())) const missing = event({ id: "event-missing-fingerprint", sourceoccurrenceid: "record-1" }) const malformed = event({ id: "event-malformed-fingerprint", sourceoccurrenceid: "record-2" }) - store.stageEvents( - [missing, malformed], - new Map([ - [missing.id, SOURCE_FINGERPRINT], - [malformed.id, SOURCE_FINGERPRINT], - ]), + run( + store.stageEvents( + [missing, malformed], + new Map([ + [missing.id, SOURCE_FINGERPRINT], + [malformed.id, SOURCE_FINGERPRINT], + ]), + ), ) - store.close() + await store.close() const database = new Database(eventingControlPath(dataDir), { readwrite: true, @@ -440,15 +454,15 @@ describe("LocalEventingControlStore", () => { database.close(true) throws( - () => LocalEventingControlStore.validateSnapshot(eventingControlPath(dataDir)), + () => run(validateControlSnapshot(eventingControlPath(dataDir))), /invalid staged source fingerprint/, ) - await rejects(() => LocalEventingControlStore.open(dataDir), /invalid staged source fingerprint/) + await rejects(() => openStore(dataDir), /invalid staged source fingerprint/) })) it("leases whole batches, redelivers after expiry, and rejects stale acknowledgements", async () => withDataDir(async (dataDir) => { - const store = await LocalEventingControlStore.open(dataDir, { + const store = await openStore(dataDir, { maxOutboxEvents: 10, maxOutboxBytes: 1024 * 1024, retainAcknowledgedReadyEvents: 0, @@ -456,16 +470,12 @@ describe("LocalEventingControlStore", () => { try { const second = event({ id: "event-2" }) const third = event({ id: "event-3" }) - const staged = store.stageEvents([event(), second, third]) - store.markReady(staged.eventIds) - store.registerConsumer("tenant-a", "automation", "beginning", "2026-08-13T12:00:00.000Z") + const staged = run(store.stageEvents([event(), second, third])) + run(store.markReady(staged.eventIds)) + run(store.registerConsumer("tenant-a", "automation", "beginning", "2026-08-13T12:00:00.000Z")) - const firstClaim = store.claimReady( - "tenant-a", - "automation", - 2, - 10, - "2026-08-13T12:00:01.000Z", + const firstClaim = run( + store.claimReady("tenant-a", "automation", 2, 10, "2026-08-13T12:00:01.000Z"), ) strictEqual(firstClaim.leaseToken?.length, 64) deepStrictEqual( @@ -473,45 +483,53 @@ describe("LocalEventingControlStore", () => { [event().id, second.id], ) throws( - () => store.claimReady("tenant-a", "automation", 2, 10, "2026-08-13T12:00:02.000Z"), + () => run(store.claimReady("tenant-a", "automation", 2, 10, "2026-08-13T12:00:02.000Z")), /active lease/, ) throws( () => - store.acknowledgeClaim( - "tenant-a", - "automation", - "0".repeat(64), - firstClaim.throughSequence!, - "2026-08-13T12:00:03.000Z", + run( + store.acknowledgeClaim( + "tenant-a", + "automation", + "0".repeat(64), + firstClaim.throughSequence!, + "2026-08-13T12:00:03.000Z", + ), ), /token does not match/, ) throws( () => - store.acknowledgeClaim( - "tenant-a", - "automation", - firstClaim.leaseToken!, - firstClaim.events[0]!.sequence, - "2026-08-13T12:00:03.000Z", + run( + store.acknowledgeClaim( + "tenant-a", + "automation", + firstClaim.leaseToken!, + firstClaim.events[0]!.sequence, + "2026-08-13T12:00:03.000Z", + ), ), /complete claimed batch/, ) - const retry = store.claimReady("tenant-a", "automation", 2, 10, "2026-08-13T12:00:12.000Z") + const retry = run( + store.claimReady("tenant-a", "automation", 2, 10, "2026-08-13T12:00:12.000Z"), + ) deepStrictEqual( retry.events.map(({ event }) => event.id), [event().id, second.id], ) strictEqual(retry.leaseToken === firstClaim.leaseToken, false) deepStrictEqual( - store.acknowledgeClaim( - "tenant-a", - "automation", - retry.leaseToken!, - retry.throughSequence!, - "2026-08-13T12:00:13.000Z", + run( + store.acknowledgeClaim( + "tenant-a", + "automation", + retry.leaseToken!, + retry.throughSequence!, + "2026-08-13T12:00:13.000Z", + ), ), { consumerId: "automation", @@ -520,28 +538,30 @@ describe("LocalEventingControlStore", () => { }, ) deepStrictEqual( - store.listReady().events.map(({ event }) => event.id), + run(store.listReady()).events.map(({ event }) => event.id), [third.id], ) throws( () => - store.acknowledgeClaim( - "tenant-a", - "automation", - firstClaim.leaseToken!, - firstClaim.throughSequence!, - "2026-08-13T12:00:14.000Z", + run( + store.acknowledgeClaim( + "tenant-a", + "automation", + firstClaim.leaseToken!, + firstClaim.throughSequence!, + "2026-08-13T12:00:14.000Z", + ), ), /no active lease/, ) } finally { - store.close() + await store.close() } })) it("prunes only after every active consumer advances and never prunes staged events", async () => withDataDir(async (dataDir) => { - const store = await LocalEventingControlStore.open(dataDir, { + const store = await openStore(dataDir, { maxOutboxEvents: 10, maxOutboxBytes: 1024 * 1024, retainAcknowledgedReadyEvents: 0, @@ -550,107 +570,110 @@ describe("LocalEventingControlStore", () => { const second = event({ id: "event-2" }) const third = event({ id: "event-3" }) const stranded = event({ id: "event-staged" }) - const ready = store.stageEvents([event(), second, third]) - store.markReady(ready.eventIds) - store.stageEvents([stranded]) - store.registerConsumer("tenant-a", "automation-a", "beginning") - store.registerConsumer("tenant-a", "automation-b", "beginning") + const ready = run(store.stageEvents([event(), second, third])) + run(store.markReady(ready.eventIds)) + run(store.stageEvents([stranded])) + run(store.registerConsumer("tenant-a", "automation-a", "beginning")) + run(store.registerConsumer("tenant-a", "automation-b", "beginning")) - const fast = store.claimReady("tenant-a", "automation-a", 3, 30) + const fast = run(store.claimReady("tenant-a", "automation-a", 3, 30)) strictEqual( - store.acknowledgeClaim( - "tenant-a", - "automation-a", - fast.leaseToken!, - fast.throughSequence!, + run( + store.acknowledgeClaim( + "tenant-a", + "automation-a", + fast.leaseToken!, + fast.throughSequence!, + ), ).prunedEvents, 0, ) - const slow = store.claimReady("tenant-a", "automation-b", 2, 30) + const slow = run(store.claimReady("tenant-a", "automation-b", 2, 30)) strictEqual( - store.acknowledgeClaim( - "tenant-a", - "automation-b", - slow.leaseToken!, - slow.throughSequence!, + run( + store.acknowledgeClaim( + "tenant-a", + "automation-b", + slow.leaseToken!, + slow.throughSequence!, + ), ).prunedEvents, 2, ) deepStrictEqual( - store.listReady().events.map(({ event }) => event.id), + run(store.listReady()).events.map(({ event }) => event.id), [third.id], ) - store.disableConsumer("tenant-a", "automation-b") - deepStrictEqual(store.listReady().events, []) + run(store.disableConsumer("tenant-a", "automation-b")) + deepStrictEqual(run(store.listReady()).events, []) deepStrictEqual( - store.listStaged().events.map(({ event }) => event.id), + run(store.listStaged()).events.map(({ event }) => event.id), [stranded.id], ) } finally { - store.close() + await store.close() } })) it("starts latest consumers after backlog and checkpoints active leases", async () => withDataDir(async (dataDir) => { - let store = await LocalEventingControlStore.open(dataDir) - store.stageEvents([event()]) - store.markReady([event().id]) - const registered = store.registerConsumer( - "tenant-a", - "automation", - "latest", - "2099-01-01T00:00:00.000Z", + let store = await openStore(dataDir) + run(store.stageEvents([event()])) + run(store.markReady([event().id])) + const registered = run( + store.registerConsumer("tenant-a", "automation", "latest", "2099-01-01T00:00:00.000Z"), ) - strictEqual(registered.lastAcknowledgedSequence, store.listReady().events[0]!.sequence) + strictEqual(registered.lastAcknowledgedSequence, run(store.listReady()).events[0]!.sequence) deepStrictEqual( - store.claimReady("tenant-a", "automation", 10, 300, "2099-01-01T00:00:01.000Z").events, + run(store.claimReady("tenant-a", "automation", 10, 300, "2099-01-01T00:00:01.000Z")).events, [], ) const second = event({ id: "event-2" }) - store.stageEvents([second]) - store.markReady([second.id]) - const claim = store.claimReady("tenant-a", "automation", 10, 300, "2099-01-01T00:00:02.000Z") + run(store.stageEvents([second])) + run(store.markReady([second.id])) + const claim = run(store.claimReady("tenant-a", "automation", 10, 300, "2099-01-01T00:00:02.000Z")) const snapshot = join(dataDir, "backups", "consumer", "control.sqlite") - await store.backupTo(snapshot) - store.close() + await runAsync(store.backupTo(snapshot)) + await store.close() const restored = join(dataDir, "restored-consumer") - await LocalEventingControlStore.restoreSnapshot(snapshot, restored) - store = await LocalEventingControlStore.open(restored) + await runAsync(restoreControlSnapshot(snapshot, restored)) + store = await openStore(restored) try { deepStrictEqual( - store.listConsumers("tenant-a")[0]?.claimedThroughSequence, + run(store.listConsumers("tenant-a"))[0]?.claimedThroughSequence, claim.throughSequence, ) strictEqual( - store.acknowledgeClaim( - "tenant-a", - "automation", - claim.leaseToken!, - claim.throughSequence!, - "2099-01-01T00:00:03.000Z", + run( + store.acknowledgeClaim( + "tenant-a", + "automation", + claim.leaseToken!, + claim.throughSequence!, + "2099-01-01T00:00:03.000Z", + ), ).acknowledgedThrough, claim.throughSequence, ) } finally { - store.close() + await store.close() } })) it("rejects a corrupted durable outbox counter at reopen", async () => withDataDir(async (dataDir) => { - const store = await LocalEventingControlStore.open(dataDir) - store.stageEvents([event()]) - store.close() + const store = await openStore(dataDir) + run(store.stageEvents([event()])) + await store.close() const db = new Database(eventingControlPath(dataDir)) try { db.run("UPDATE outbox_usage SET bytes = bytes + 1 WHERE singleton = 1") } finally { db.close() } - await rejects(() => LocalEventingControlStore.open(dataDir), /accounting is inconsistent/) + await rejects(() => openStore(dataDir), /accounting is inconsistent/) })) it("refuses a symlink in place of the database", async () => @@ -658,7 +681,7 @@ describe("LocalEventingControlStore", () => { const controlPath = eventingControlPath(dataDir) mkdirSync(join(dataDir, "control"), { recursive: true }) symlinkSync(join(dataDir, "target.sqlite"), controlPath) - await rejects(() => LocalEventingControlStore.open(dataDir), /not a real file/) + await rejects(() => openStore(dataDir), /not a real file/) strictEqual(controlPath.endsWith("control/eventing.sqlite"), true) })) }) diff --git a/apps/cli/test/local-eventing-ingest.test.ts b/apps/cli/test/local-eventing-ingest.test.ts index a1fef70cf..df782c22c 100644 --- a/apps/cli/test/local-eventing-ingest.test.ts +++ b/apps/cli/test/local-eventing-ingest.test.ts @@ -1,36 +1,45 @@ import { deepStrictEqual, ok, rejects, strictEqual } from "node:assert" +import { Effect } from "effect" import { describe, it } from "vitest" +import { EventingControlStoreError } from "../src/server/eventing/control-store" import { normalizeOtlpLogs } from "../src/server/eventing/otlp" -import { ProjectionActivationConflict, SourceOccurrenceCollision } from "../src/server/eventing/runtime" +import { + type LocalEventingRuntimeApi, + ProjectionActivationConflict, + SourceOccurrenceCollision, +} from "../src/server/eventing/runtime" import { __testables } from "../src/server/serve" +import { run, serveCheckpointWith, serveWith } from "./eventing-test-support" describe("Local eventing ingest seam", () => { it("requires maintenance authorization and exposes staged records only when requested", async () => { const eventing = { - health: () => ({ activeProjections: 1 }), - listActive: () => [], - listReady: () => ({ events: [{ sequence: 1, event: { id: "ready" } }], nextCursor: null }), - listStaged: (_limit: number, after: number) => ({ - events: [{ sequence: after + 1, event: { id: "staged" } }], - nextCursor: null, - }), + health: Effect.succeed({ activeProjections: 1 }), + listActive: Effect.succeed([]), + listReady: () => + Effect.succeed({ events: [{ sequence: 1, event: { id: "ready" } }], nextCursor: null }), + listStaged: (_limit: number, after: number) => + Effect.succeed({ + events: [{ sequence: after + 1, event: { id: "staged" } }], + nextCursor: null, + }), } - const unauthorized = __testables.handleEventingRead( + const unauthorized = await serveWith( eventing as never, - "maintenance-secret", - new Request("http://127.0.0.1/local/eventing/outbox?state=staged"), - new URL("http://127.0.0.1/local/eventing/outbox?state=staged"), + __testables.handleEventingRead( + "maintenance-secret", + new Request("http://127.0.0.1/local/eventing/outbox?state=staged"), + new URL("http://127.0.0.1/local/eventing/outbox?state=staged"), + ), ) strictEqual(unauthorized.status, 403) const request = new Request("http://127.0.0.1/local/eventing/outbox?state=staged&after=41", { headers: { "x-maple-maintenance-token": "maintenance-secret" }, }) - const authorized = __testables.handleEventingRead( + const authorized = await serveWith( eventing as never, - "maintenance-secret", - request, - new URL(request.url), + __testables.handleEventingRead("maintenance-secret", request, new URL(request.url)), ) strictEqual(authorized.status, 200) deepStrictEqual(await authorized.json(), { @@ -42,27 +51,24 @@ describe("Local eventing ingest seam", () => { it("authenticates and reads activation bodies before closing admission", async () => { const gate = new __testables.RequestQuiescenceGate() const neverClosed = new ReadableStream() - const unauthorized = await __testables.handleProjectionActivation( + const unauthorized = await serveWith( {} as never, - gate, - "maintenance-secret", - { + __testables.handleProjectionActivation(gate, "maintenance-secret", { headers: new Headers(), body: neverClosed, - } as Request, + } as Request), ) strictEqual(unauthorized.status, 403) const afterUnauthorized = gate.enter() ok(afterUnauthorized, "invalid authorization must not close admission") afterUnauthorized() - const checkpointUnauthorized = await __testables.handleCheckpointBackup( - {} as never, + const checkpointUnauthorized = await serveCheckpointWith( {} as never, - "/unused", - gate, - "maintenance-secret", - { headers: new Headers(), body: neverClosed } as Request, + __testables.handleCheckpointBackup({} as never, "/unused", gate, "maintenance-secret", { + headers: new Headers(), + body: neverClosed, + } as Request), ) strictEqual(checkpointUnauthorized.status, 403) const afterCheckpointUnauthorized = gate.enter() @@ -76,20 +82,19 @@ describe("Local eventing ingest seam", () => { }, }) let committed = false - const pending = __testables.handleProjectionActivation( + const pending = serveWith( { - prepareActivation: (body: unknown) => ({ body }), - commitActivation: () => { - committed = true - }, - listActive: () => [], + prepareActivation: (body: unknown) => Effect.succeed({ body }), + commitActivation: () => + Effect.sync(() => { + committed = true + }), + listActive: Effect.succeed([]), } as never, - gate, - "maintenance-secret", - { + __testables.handleProjectionActivation(gate, "maintenance-secret", { headers: new Headers({ "x-maple-maintenance-token": "maintenance-secret" }), body: slowBody, - } as Request, + } as Request), ) await Promise.resolve() const whileReading = gate.enter() @@ -106,7 +111,7 @@ describe("Local eventing ingest seam", () => { method: "POST", body: "123456789", }) - await rejects(() => __testables.readBoundedJson(oversized, 8), /exceeds 8 bytes/) + await rejects(() => Effect.runPromise(__testables.readBoundedJson(oversized, 8)), /exceeds 8 bytes/) const gate = new __testables.RequestQuiescenceGate() let releaseMaintenance!: () => void @@ -117,22 +122,24 @@ describe("Local eventing ingest seam", () => { }), ) await Promise.resolve() - const response = await __testables.handleProjectionActivation( + const response = await serveWith( { - prepareActivation: () => ({}), - commitActivation: () => undefined, - listActive: () => [], + prepareActivation: () => Effect.succeed({}), + commitActivation: () => Effect.void, + listActive: Effect.succeed([]), } as never, - gate, - "maintenance-secret", - new Request("http://127.0.0.1/local/eventing/projections", { - method: "POST", - headers: { - "content-type": "application/json", - "x-maple-maintenance-token": "maintenance-secret", - }, - body: "{}", - }), + __testables.handleProjectionActivation( + gate, + "maintenance-secret", + new Request("http://127.0.0.1/local/eventing/projections", { + method: "POST", + headers: { + "content-type": "application/json", + "x-maple-maintenance-token": "maintenance-secret", + }, + body: "{}", + }), + ), ) strictEqual(response.status, 409) releaseMaintenance() @@ -143,91 +150,103 @@ describe("Local eventing ingest seam", () => { const gate = new __testables.RequestQuiescenceGate() const calls: string[] = [] const eventing = { - registerConsumer: (consumerId: string, startAt: string) => { - calls.push(`register:${consumerId}:${startAt}`) - return { consumerId, active: true } - }, - disableConsumer: (consumerId: string) => { - calls.push(`disable:${consumerId}`) - return { consumerId, active: false } - }, - claimReady: (consumerId: string, limit: number, leaseSeconds: number) => { - calls.push(`claim:${consumerId}:${limit}:${leaseSeconds}`) - return { - consumerId, - leaseToken: "a".repeat(64), - throughSequence: 7, - events: [{ sequence: 7, event: { id: "event-7" } }], - } - }, - acknowledgeClaim: (consumerId: string, _leaseToken: string, throughSequence: number) => { - calls.push(`ack:${consumerId}:${throughSequence}`) - return { consumerId, acknowledgedThrough: throughSequence, prunedEvents: 0 } - }, + registerConsumer: (consumerId: string, startAt: string) => + Effect.sync(() => { + calls.push(`register:${consumerId}:${startAt}`) + return { consumerId, active: true } + }), + disableConsumer: (consumerId: string) => + Effect.sync(() => { + calls.push(`disable:${consumerId}`) + return { consumerId, active: false } + }), + claimReady: (consumerId: string, limit: number, leaseSeconds: number) => + Effect.sync(() => { + calls.push(`claim:${consumerId}:${limit}:${leaseSeconds}`) + return { + consumerId, + leaseToken: "a".repeat(64), + throughSequence: 7, + events: [{ sequence: 7, event: { id: "event-7" } }], + } + }), + acknowledgeClaim: (consumerId: string, _leaseToken: string, throughSequence: number) => + Effect.sync(() => { + calls.push(`ack:${consumerId}:${throughSequence}`) + return { consumerId, acknowledgedThrough: throughSequence, prunedEvents: 0 } + }), } - const registration = await __testables.handleConsumerRegistration( + const registration = await serveWith( eventing as never, - gate, - "maintenance-secret", - new Request("http://127.0.0.1/local/eventing/consumers", { - method: "POST", - headers: { - "content-type": "application/json", - "x-maple-maintenance-token": "maintenance-secret", - }, - body: JSON.stringify({ consumerId: "automation", startAt: "beginning" }), - }), + __testables.handleConsumerRegistration( + gate, + "maintenance-secret", + new Request("http://127.0.0.1/local/eventing/consumers", { + method: "POST", + headers: { + "content-type": "application/json", + "x-maple-maintenance-token": "maintenance-secret", + }, + body: JSON.stringify({ consumerId: "automation", startAt: "beginning" }), + }), + ), ) strictEqual(registration.status, 201) - const wrongClaimCredential = await __testables.handleConsumerClaim( + const wrongClaimCredential = await serveWith( eventing as never, - gate, - "consumer-secret", - new Request("http://127.0.0.1/local/eventing/claims", { - method: "POST", - headers: { - "content-type": "application/json", - "x-maple-maintenance-token": "maintenance-secret", - }, - body: JSON.stringify({ consumerId: "automation", limit: 10, leaseSeconds: 30 }), - }), + __testables.handleConsumerClaim( + gate, + "consumer-secret", + new Request("http://127.0.0.1/local/eventing/claims", { + method: "POST", + headers: { + "content-type": "application/json", + "x-maple-maintenance-token": "maintenance-secret", + }, + body: JSON.stringify({ consumerId: "automation", limit: 10, leaseSeconds: 30 }), + }), + ), ) strictEqual(wrongClaimCredential.status, 403) - const claim = await __testables.handleConsumerClaim( + const claim = await serveWith( eventing as never, - gate, - "consumer-secret", - new Request("http://127.0.0.1/local/eventing/claims", { - method: "POST", - headers: { - "content-type": "application/json", - "x-maple-event-consumer-token": "consumer-secret", - }, - body: JSON.stringify({ consumerId: "automation", limit: 10, leaseSeconds: 30 }), - }), + __testables.handleConsumerClaim( + gate, + "consumer-secret", + new Request("http://127.0.0.1/local/eventing/claims", { + method: "POST", + headers: { + "content-type": "application/json", + "x-maple-event-consumer-token": "consumer-secret", + }, + body: JSON.stringify({ consumerId: "automation", limit: 10, leaseSeconds: 30 }), + }), + ), ) strictEqual(claim.status, 200) const claimed = (await claim.json()) as { leaseToken: string; throughSequence: number } - const acknowledgement = await __testables.handleConsumerAcknowledgement( + const acknowledgement = await serveWith( eventing as never, - gate, - "consumer-secret", - new Request("http://127.0.0.1/local/eventing/acks", { - method: "POST", - headers: { - "content-type": "application/json", - "x-maple-event-consumer-token": "consumer-secret", - }, - body: JSON.stringify({ - consumerId: "automation", - leaseToken: claimed.leaseToken, - throughSequence: claimed.throughSequence, + __testables.handleConsumerAcknowledgement( + gate, + "consumer-secret", + new Request("http://127.0.0.1/local/eventing/acks", { + method: "POST", + headers: { + "content-type": "application/json", + "x-maple-event-consumer-token": "consumer-secret", + }, + body: JSON.stringify({ + consumerId: "automation", + leaseToken: claimed.leaseToken, + throughSequence: claimed.throughSequence, + }), }), - }), + ), ) strictEqual(acknowledgement.status, 200) deepStrictEqual(calls, [ @@ -244,18 +263,20 @@ describe("Local eventing ingest seam", () => { }), ) await Promise.resolve() - const blockedClaim = await __testables.handleConsumerClaim( + const blockedClaim = await serveWith( eventing as never, - gate, - "consumer-secret", - new Request("http://127.0.0.1/local/eventing/claims", { - method: "POST", - headers: { - "content-type": "application/json", - "x-maple-event-consumer-token": "consumer-secret", - }, - body: JSON.stringify({ consumerId: "automation", limit: 10, leaseSeconds: 30 }), - }), + __testables.handleConsumerClaim( + gate, + "consumer-secret", + new Request("http://127.0.0.1/local/eventing/claims", { + method: "POST", + headers: { + "content-type": "application/json", + "x-maple-event-consumer-token": "consumer-secret", + }, + body: JSON.stringify({ consumerId: "automation", limit: 10, leaseSeconds: 30 }), + }), + ), ) strictEqual(blockedClaim.status, 503) releaseMaintenance() @@ -263,19 +284,19 @@ describe("Local eventing ingest seam", () => { }) it("rejects fractional consumer limits before calling the store", async () => { - const response = await __testables.handleConsumerClaim( + const response = await serveWith( { - claimReady: () => { - throw new Error("invalid input reached the store") - }, + claimReady: () => Effect.die(new Error("invalid input reached the store")), }, - new __testables.RequestQuiescenceGate(), - "secret", - new Request("http://127.0.0.1/local/eventing/claims", { - method: "POST", - headers: { "x-maple-event-consumer-token": "secret" }, - body: JSON.stringify({ consumerId: "automation", limit: 1.5, leaseSeconds: 30 }), - }), + __testables.handleConsumerClaim( + new __testables.RequestQuiescenceGate(), + "secret", + new Request("http://127.0.0.1/local/eventing/claims", { + method: "POST", + headers: { "x-maple-event-consumer-token": "secret" }, + body: JSON.stringify({ consumerId: "automation", limit: 1.5, leaseSeconds: 30 }), + }), + ), ) strictEqual(response.status, 400) }) @@ -296,28 +317,30 @@ describe("Local eventing ingest seam", () => { }, } const eventing = { - evaluateOtlp: () => { - order.push("evaluate") - return { - events: [event], - recoveredEventIds: [], - failures: [ - { - projectionId: "oversized-projector", - projectionRevision: 1, - occurrenceId: "occurrence-1", - message: "CloudEvent exceeds 262144 UTF-8 bytes", - }, - ], - typeMismatchFields: [], - } - }, - persistFailures: () => order.push("persist-failures"), - stage: () => { - order.push("stage") - return { inserted: 1, deduplicated: 0, dropped: 0, eventIds: [event.id] } - }, - markReady: () => order.push("ready"), + evaluateOtlp: () => + Effect.sync(() => { + order.push("evaluate") + return { + events: [event], + recoveredEventIds: [], + failures: [ + { + projectionId: "oversized-projector", + projectionRevision: 1, + occurrenceId: "occurrence-1", + message: "CloudEvent exceeds 262144 UTF-8 bytes", + }, + ], + typeMismatchFields: [], + } + }), + persistFailures: () => Effect.sync(() => void order.push("persist-failures")), + stage: () => + Effect.sync(() => { + order.push("stage") + return { inserted: 1, deduplicated: 0, dropped: 0, eventIds: [event.id] } + }), + markReady: () => Effect.sync(() => void order.push("ready")), } const request = new Request("http://127.0.0.1/v1/logs", { method: "POST", @@ -340,12 +363,9 @@ describe("Local eventing ingest seam", () => { }), }) - const result = await __testables.ingest( - db as never, - authority as never, + const result = await serveWith( eventing as never, - "logs", - request, + __testables.ingest(db as never, authority as never, "logs", request), ) strictEqual(result.response.status, 200) strictEqual(result.accepted, 1) @@ -368,35 +388,40 @@ describe("Local eventing ingest seam", () => { resourceLogs: [{ scopeLogs: [{ logRecords: [{ body: { stringValue: "one" } }] }] }], }), }) - const result = await __testables.ingest( - { - exec: () => { - throw new Error("write failed") - }, - } as never, + const result = await serveWith( { - isRetired: () => false, - filterBatch: (_datasource: string, ndjson: string) => ({ - ndjson, - accepted: 1, - rejected: 0, - }), - } as never, - { - evaluateOtlp: () => ({ - events: [{ id: "event-1" }], - recoveredEventIds: [], - failures: [], - typeMismatchFields: [], - }), - persistFailures: () => undefined, - stage: () => ({ inserted: 1, deduplicated: 0, dropped: 0, eventIds: ["event-1"] }), - markReady: () => { - markedReady = true - }, + evaluateOtlp: () => + Effect.succeed({ + events: [{ id: "event-1" }], + recoveredEventIds: [], + failures: [], + typeMismatchFields: [], + }), + persistFailures: () => Effect.void, + stage: () => + Effect.succeed({ inserted: 1, deduplicated: 0, dropped: 0, eventIds: ["event-1"] }), + markReady: () => + Effect.sync(() => { + markedReady = true + }), } as never, - "logs", - request, + __testables.ingest( + { + exec: () => { + throw new Error("write failed") + }, + } as never, + { + isRetired: () => false, + filterBatch: (_datasource: string, ndjson: string) => ({ + ndjson, + accepted: 1, + rejected: 0, + }), + } as never, + "logs", + request, + ), ) strictEqual(result.response.status, 500) strictEqual(markedReady, false) @@ -421,31 +446,35 @@ describe("Local eventing ingest seam", () => { ], }), }) - const result = await __testables.ingest( - { exec: () => undefined } as never, + const result = await serveWith( { - isRetired: () => false, - filterBatch: (_datasource: string, ndjson: string) => ({ - ndjson, - accepted: 1, - rejected: 0, - }), - } as never, - { - evaluateOtlp: () => ({ - events: [], - recoveredEventIds: ["revision-1-event"], - failures: [], - typeMismatchFields: [], - }), - persistFailures: () => undefined, - stage: () => ({ inserted: 0, deduplicated: 0, dropped: 0, eventIds: [] }), - markReady: (eventIds: readonly string[]) => { - readyIds = eventIds - }, + evaluateOtlp: () => + Effect.succeed({ + events: [], + recoveredEventIds: ["revision-1-event"], + failures: [], + typeMismatchFields: [], + }), + persistFailures: () => Effect.void, + stage: () => Effect.succeed({ inserted: 0, deduplicated: 0, dropped: 0, eventIds: [] }), + markReady: (eventIds: readonly string[]) => + Effect.sync(() => { + readyIds = eventIds + }), } as never, - "logs", - request, + __testables.ingest( + { exec: () => undefined } as never, + { + isRetired: () => false, + filterBatch: (_datasource: string, ndjson: string) => ({ + ndjson, + accepted: 1, + rejected: 0, + }), + } as never, + "logs", + request, + ), ) strictEqual(result.response.status, 200) deepStrictEqual(readyIds, ["revision-1-event"]) @@ -520,41 +549,44 @@ describe("Local eventing ingest seam", () => { }, ], } - const result = await __testables.ingest( - { exec: () => (inserted = true) } as never, - { - isRetired: () => false, - filterBatch: (_datasource: string, ndjson: string) => ({ - ndjson, - accepted: ndjson.trim().split("\n").length, - rejected: 0, - }), - } as never, + const result = await serveWith( { evaluateOtlp: (_signal: string, decoded: unknown) => { - const projected = normalizeOtlpLogs(decoded).filter( + const projected = run(normalizeOtlpLogs(decoded)).filter( (signal) => signal.fields.get("signal:event.name")?.value === "project.me", ) - return { + return Effect.succeed({ events: projected.map((_signal, index) => ({ id: `event-${index + 1}` })), recoveredEventIds: [], failures: [], typeMismatchFields: [], - } - }, - persistFailures: () => undefined, - stage: (events: readonly { readonly id: string }[]) => { - stagedIds = events.map(({ id }) => id) - return { inserted: events.length, deduplicated: 0, dropped: 0, eventIds: stagedIds } + }) }, - markReady: () => undefined, + persistFailures: () => Effect.void, + stage: (events: readonly { readonly id: string }[]) => + Effect.sync(() => { + stagedIds = events.map(({ id }) => id) + return { inserted: events.length, deduplicated: 0, dropped: 0, eventIds: stagedIds } + }), + markReady: () => Effect.void, } as never, - "logs", - new Request("http://127.0.0.1/v1/logs", { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify(body), - }), + __testables.ingest( + { exec: () => (inserted = true) } as never, + { + isRetired: () => false, + filterBatch: (_datasource: string, ndjson: string) => ({ + ndjson, + accepted: ndjson.trim().split("\n").length, + rejected: 0, + }), + } as never, + "logs", + new Request("http://127.0.0.1/v1/logs", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }), + ), ) strictEqual(result.response.status, 200) strictEqual(result.accepted, 5) @@ -563,29 +595,35 @@ describe("Local eventing ingest seam", () => { }) it("refuses an in-batch source collision with 400 so exporters do not resend it", async () => { - const ingestWith = (failure: unknown) => - __testables.ingest( - { exec: () => undefined } as never, - { isRetired: () => false } as never, + const ingestWith = (failure: Effect.Effect) => + serveWith( { - evaluateOtlp: () => { - throw failure - }, + evaluateOtlp: () => failure, } as never, - "logs", - new Request("http://127.0.0.1/v1/logs", { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ resourceLogs: [] }), - }), + __testables.ingest( + { exec: () => undefined } as never, + { isRetired: () => false } as never, + "logs", + new Request("http://127.0.0.1/v1/logs", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ resourceLogs: [] }), + }), + ), ) const collision = await ingestWith( - new SourceOccurrenceCollision({ message: "source occurrence collision", occurrenceId: "a" }), + Effect.fail( + new SourceOccurrenceCollision({ message: "source occurrence collision", occurrenceId: "a" }), + ), ) strictEqual(collision.response.status, 400) strictEqual(collision.accepted, 0) - const transient = await ingestWith(new Error("control store unavailable")) + const transient = await ingestWith( + Effect.fail(new EventingControlStoreError({ message: "control store unavailable" })), + ) strictEqual(transient.response.status, 503) + const unexpected = await ingestWith(Effect.die(new Error("projection crashed"))) + strictEqual(unexpected.response.status, 503) }) it("reports a concurrent activation as a retryable 409", async () => { @@ -594,51 +632,49 @@ describe("Local eventing ingest seam", () => { headers: { "x-maple-maintenance-token": "maintenance-secret" }, body: JSON.stringify({}), }) - const response = await __testables.handleProjectionActivation( + const response = await serveWith( { - prepareActivation: () => ({}), - commitActivation: () => { - throw new ProjectionActivationConflict({ message: "projection registry changed" }) - }, + prepareActivation: () => Effect.succeed({}), + commitActivation: () => + Effect.fail(new ProjectionActivationConflict({ message: "projection registry changed" })), } as never, - new __testables.RequestQuiescenceGate(), - "maintenance-secret", - request, + __testables.handleProjectionActivation( + new __testables.RequestQuiescenceGate(), + "maintenance-secret", + request, + ), ) strictEqual(response.status, 409) }) it("validates outbox query parameters at the boundary and keeps store failures a 500", async () => { - const read = (query: string, eventing: Pick) => { + const read = async (query: string, eventing: Pick) => { const request = new Request(`http://127.0.0.1/local/eventing/outbox${query}`, { headers: { "x-maple-maintenance-token": "maintenance-secret" }, }) - return __testables.handleEventingRead( + return serveWith( eventing as never, - "maintenance-secret", - request, - new URL(request.url), + __testables.handleEventingRead("maintenance-secret", request, new URL(request.url)), ) } const listed: Array = [] const eventing = { - listReady: (limit: number, after: number) => { - listed.push([limit, after]) - return { events: [], nextCursor: null } - }, + listReady: (limit?: number, after?: number) => + Effect.sync(() => { + listed.push([limit ?? 100, after ?? 0]) + return { events: [], nextCursor: null } + }), } - strictEqual(read("?limit=0", eventing).status, 400) - strictEqual(read("?limit=1.5", eventing).status, 400) - strictEqual(read("?after=-1", eventing).status, 400) - strictEqual(read("?state=acked", eventing).status, 400) - strictEqual(read("?cursor=1", eventing).status, 400) - strictEqual(read("?limit=25&after=7", eventing).status, 200) + strictEqual((await read("?limit=0", eventing)).status, 400) + strictEqual((await read("?limit=1.5", eventing)).status, 400) + strictEqual((await read("?after=-1", eventing)).status, 400) + strictEqual((await read("?state=acked", eventing)).status, 400) + strictEqual((await read("?cursor=1", eventing)).status, 400) + strictEqual((await read("?limit=25&after=7", eventing)).status, 200) deepStrictEqual(listed, [[25, 7]]) const failing = { - listReady: () => { - throw new Error("database is locked") - }, + listReady: () => Effect.fail(new EventingControlStoreError({ message: "database is locked" })), } - strictEqual(read("", failing).status, 500) + strictEqual((await read("", failing)).status, 500) }) }) diff --git a/apps/cli/test/local-eventing-overflow.test.ts b/apps/cli/test/local-eventing-overflow.test.ts index 088ba3406..694c114de 100644 --- a/apps/cli/test/local-eventing-overflow.test.ts +++ b/apps/cli/test/local-eventing-overflow.test.ts @@ -6,16 +6,15 @@ import { tmpdir } from "node:os" import { join } from "node:path" import { describe, it } from "vitest" import { RetiredDayAuthority } from "../src/server/archives/retention" -import { LocalEventingControlStore } from "../src/server/eventing/control-store" -import { LocalEventingRuntime } from "../src/server/eventing/runtime" import { __testables } from "../src/server/serve" +import { makeRuntime, openStore, run, serveWith } from "./eventing-test-support" describe("Durable outbox overflow", () => { it("keeps warehouse ingestion available and requires explicit gap recovery across reopen", async () => { const parent = mkdtempSync(join(tmpdir(), "maple-overflow-")) const dataDir = join(parent, "data") mkdirSync(dataDir) - const store = await LocalEventingControlStore.open(dataDir, { + const store = await openStore(dataDir, { maxOutboxEvents: 1, maxOutboxBytes: 1024 * 1024, }) @@ -32,17 +31,22 @@ describe("Durable outbox overflow", () => { project: () => ({ data: { observed: true } }), }), ) - const runtime = new LocalEventingRuntime(store, undefined, projectors) - runtime.activate({ - id: "observed", - revision: 1, - enabled: true, - tenantId: "local", - sourceKind: "otel.log", - selector: { op: "exists", field: { namespace: "signal", key: "event.name", type: "string" } }, - projector: { id: "example.observed", version: 1, config: {} }, - activeFrom: "1970-01-01T00:00:00Z", - }) + const runtime = makeRuntime(store, projectors) + run( + runtime.activate({ + id: "observed", + revision: 1, + enabled: true, + tenantId: "local", + sourceKind: "otel.log", + selector: { + op: "exists", + field: { namespace: "signal", key: "event.name", type: "string" }, + }, + projector: { id: "example.observed", version: 1, config: {} }, + activeFrom: "1970-01-01T00:00:00Z", + }), + ) const statements: string[] = [] const warehouse = { exec: (sql: string) => { @@ -51,34 +55,36 @@ describe("Durable outbox overflow", () => { } const authority = new RetiredDayAuthority(dataDir) const ingest = (id: string) => - __testables.ingest( - warehouse, - authority, + serveWith( runtime, - "logs", - new Request("http://localhost/v1/logs", { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ - resourceLogs: [ - { - scopeLogs: [ - { - logRecords: [ - { - eventName: "example.observed", - timeUnixNano: "1786131720123456789", - attributes: [ - { key: "event.id", value: { stringValue: id } }, - ], - }, - ], - }, - ], - }, - ], + __testables.ingest( + warehouse, + authority, + "logs", + new Request("http://localhost/v1/logs", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + resourceLogs: [ + { + scopeLogs: [ + { + logRecords: [ + { + eventName: "example.observed", + timeUnixNano: "1786131720123456789", + attributes: [ + { key: "event.id", value: { stringValue: id } }, + ], + }, + ], + }, + ], + }, + ], + }), }), - }), + ), ) const first = await ingest("record-1") strictEqual(first.accepted, 1) @@ -88,69 +94,73 @@ describe("Durable outbox overflow", () => { strictEqual(second.response.status, 200) strictEqual(second.response.headers.get("x-maple-eventing-dropped"), "1") strictEqual(statements.length, 2) - const retained = store.listReady(10).events + const retained = run(store.listReady(10)).events strictEqual(retained.length, 1) const eventId = retained[0]?.event.id if (eventId === undefined) throw new Error("missing retained event") - store.registerConsumer("local", "consumer", "beginning") - throws(() => store.claimReady("local", "consumer", 10, 60), /delivery has a gap/) - throws(() => store.acceptDeliveryGap("local", "consumer", 2), /generation changed/) - store.acceptDeliveryGap("local", "consumer", 1) - strictEqual(store.claimReady("local", "consumer", 10, 60).events.length, 1) - throws(() => store.abandonEvents("other-tenant", [eventId]), /unknown event ID/) - throws(() => store.abandonEvents("local", [eventId, "missing"]), /unknown event ID/) - strictEqual(store.listReady(10).events.length, 1) - strictEqual(store.deliveryGap("local").generation, 1) - const unauthorized = await __testables.handleOutboxAdministration( + run(store.registerConsumer("local", "consumer", "beginning")) + throws(() => run(store.claimReady("local", "consumer", 10, 60)), /delivery has a gap/) + throws(() => run(store.acceptDeliveryGap("local", "consumer", 2)), /generation changed/) + run(store.acceptDeliveryGap("local", "consumer", 1)) + strictEqual(run(store.claimReady("local", "consumer", 10, 60)).events.length, 1) + throws(() => run(store.abandonEvents("other-tenant", [eventId])), /unknown event ID/) + throws(() => run(store.abandonEvents("local", [eventId, "missing"])), /unknown event ID/) + strictEqual(run(store.listReady(10)).events.length, 1) + strictEqual(run(store.deliveryGap("local")).generation, 1) + const unauthorized = await serveWith( runtime, - new __testables.RequestQuiescenceGate(), - "secret", - new Request("http://localhost/local/eventing/outbox/abandon", { - method: "POST", - body: JSON.stringify({ eventIds: [eventId] }), - }), - "abandon", + __testables.handleOutboxAdministration( + new __testables.RequestQuiescenceGate(), + "secret", + new Request("http://localhost/local/eventing/outbox/abandon", { + method: "POST", + body: JSON.stringify({ eventIds: [eventId] }), + }), + "abandon", + ), ) strictEqual(unauthorized.status, 403) - strictEqual(store.listReady(10).events.length, 1) + strictEqual(run(store.listReady(10)).events.length, 1) const gate = new __testables.RequestQuiescenceGate() const release = gate.enter() if (release === null) throw new Error("gate unexpectedly closed") - const pending = __testables.handleOutboxAdministration( + const pending = serveWith( runtime, - gate, - "secret", - new Request("http://localhost/local/eventing/outbox/abandon", { - method: "POST", - headers: { "x-maple-maintenance-token": "secret" }, - body: JSON.stringify({ eventIds: [eventId] }), - }), - "abandon", + __testables.handleOutboxAdministration( + gate, + "secret", + new Request("http://localhost/local/eventing/outbox/abandon", { + method: "POST", + headers: { "x-maple-maintenance-token": "secret" }, + body: JSON.stringify({ eventIds: [eventId] }), + }), + "abandon", + ), ) await Promise.resolve() - strictEqual(store.listReady(10).events.length, 1) + strictEqual(run(store.listReady(10)).events.length, 1) release() strictEqual((await pending).status, 200) - strictEqual(store.deliveryGap("local").generation, 2) - throws(() => store.claimReady("local", "consumer", 10, 60), /delivery has a gap/) - store.acceptDeliveryGap("local", "consumer", 2) + strictEqual(run(store.deliveryGap("local")).generation, 2) + throws(() => run(store.claimReady("local", "consumer", 10, 60)), /delivery has a gap/) + run(store.acceptDeliveryGap("local", "consumer", 2)) // Abandonment cleared the old lease and transactional counters free capacity. const third = await ingest("record-3") strictEqual(third.accepted, 1) strictEqual(third.response.headers.get("x-maple-eventing-dropped"), null) - strictEqual(store.claimReady("local", "consumer", 10, 60).events.length, 1) - store.validate() + strictEqual(run(store.claimReady("local", "consumer", 10, 60)).events.length, 1) + run(store.validate) } finally { - store.close() + await store.close() } try { - const reopened = await LocalEventingControlStore.open(dataDir) + const reopened = await openStore(dataDir) try { - strictEqual(reopened.deliveryGap("local").generation, 2) - strictEqual(reopened.deliveryGap("local").droppedEvents, 2) - strictEqual(reopened.listReady(10).events.length, 1) + strictEqual(run(reopened.deliveryGap("local")).generation, 2) + strictEqual(run(reopened.deliveryGap("local")).droppedEvents, 2) + strictEqual(run(reopened.listReady(10)).events.length, 1) } finally { - reopened.close() + await reopened.close() } } finally { rmSync(parent, { recursive: true, force: true }) diff --git a/apps/cli/test/local-eventing-runtime.test.ts b/apps/cli/test/local-eventing-runtime.test.ts index f4b0a454c..3d447f99d 100644 --- a/apps/cli/test/local-eventing-runtime.test.ts +++ b/apps/cli/test/local-eventing-runtime.test.ts @@ -1,4 +1,4 @@ -import { Result } from "effect" +import { Effect, Result } from "effect" import { deepStrictEqual, ok, strictEqual, throws } from "node:assert" import { mkdirSync, mkdtempSync, rmSync } from "node:fs" import { tmpdir } from "node:os" @@ -13,11 +13,10 @@ import { type SignalProjectionSpec, type SignalScalar, } from "@maple/eventing-core" -import { LocalEventingControlStore } from "../src/server/eventing/control-store" import { normalizeOtlpLogs, normalizeOtlpLogsWithDiagnostics } from "../src/server/eventing/otlp" -import { LocalEventingRuntime, sourceOccurrenceFingerprint } from "../src/server/eventing/runtime" -import type { EventingTelemetryObservation } from "../src/server/eventing/telemetry" +import { sourceOccurrenceFingerprint } from "../src/server/eventing/runtime" import { encodeLogs } from "../src/server/otlp/encode" +import { makeRuntime, metricRecorder, openStore, run } from "./eventing-test-support" const withDataDir = async (run: (dataDir: string) => Promise): Promise => { const parent = mkdtempSync(join(tmpdir(), "maple-eventing-runtime-")) @@ -192,22 +191,24 @@ describe("OTLP eventing input validation", () => { it("rejects non-string attribute keys before normalization", () => { throws( () => - normalizeOtlpLogs({ - resourceLogs: [ - { - scopeLogs: [ - { - logRecords: [ - { - timeUnixNano: "1786125600000000000", - attributes: [{ key: 123, value: { stringValue: "bad" } }], - }, - ], - }, - ], - }, - ], - }), + run( + normalizeOtlpLogs({ + resourceLogs: [ + { + scopeLogs: [ + { + logRecords: [ + { + timeUnixNano: "1786125600000000000", + attributes: [{ key: 123, value: { stringValue: "bad" } }], + }, + ], + }, + ], + }, + ], + }), + ), /invalid OTLP logs/, ) }) @@ -216,27 +217,25 @@ describe("OTLP eventing input validation", () => { describe("LocalEventingRuntime", () => { it("records bounded normalization and projection outcomes without signal data", async () => withDataDir(async (dataDir) => { - const observations: EventingTelemetryObservation[] = [] - const telemetry = { - record: (observation: EventingTelemetryObservation) => observations.push(observation), - } - const store = await LocalEventingControlStore.open(dataDir) + const metrics = metricRecorder() + const observed = (effect: Effect.Effect): A => run(metrics.observe(effect)) + const store = await openStore(dataDir) try { - const runtime = new LocalEventingRuntime(store, telemetry, exampleProjectors()) - runtime.activate(projection()) - strictEqual(runtime.evaluateOtlp("logs", exampleRecordObserved).events.length, 1) + const runtime = makeRuntime(store, exampleProjectors()) + run(runtime.activate(projection())) + strictEqual(observed(runtime.evaluateOtlp("logs", exampleRecordObserved)).events.length, 1) const malformed = structuredClone(exampleRecordObserved) firstLogRecord(malformed).attributes = firstLogRecord(malformed).attributes.filter( ({ key }) => key !== "example.collection.name", ) - strictEqual(runtime.evaluateOtlp("logs", malformed).failures.length, 1) + strictEqual(observed(runtime.evaluateOtlp("logs", malformed)).failures.length, 1) const mismatched = structuredClone(exampleRecordObserved) firstLogRecord(mismatched).attributes = firstLogRecord(mismatched).attributes.map((entry) => entry.key === "example.record.sequence" ? attr(entry.key, { stringValue: "42" }) : entry, ) - deepStrictEqual(runtime.evaluateOtlp("logs", mismatched).typeMismatchFields, [ + deepStrictEqual(observed(runtime.evaluateOtlp("logs", mismatched)).typeMismatchFields, [ "attribute:example.record.sequence", ]) @@ -246,28 +245,26 @@ describe("LocalEventingRuntime", () => { attr(`projection-only-${index}`, { stringValue: "warehouse-valid" }), ), ) - strictEqual(runtime.evaluateOtlp("logs", projectionBoundFailure).events.length, 0) + strictEqual(observed(runtime.evaluateOtlp("logs", projectionBoundFailure)).events.length, 0) - const operationOutcomes = observations.map( - ({ operation, outcome }) => `${operation}:${outcome}`, - ) + const operationOutcomes = metrics.operationOutcomes() ok(operationOutcomes.includes("normalization:success")) ok(operationOutcomes.includes("normalization:failure")) ok(operationOutcomes.includes("projection:success")) ok(operationOutcomes.includes("projection:failure")) ok(operationOutcomes.includes("selector_type_mismatch:observed")) - const serialized = JSON.stringify(observations) + const serialized = metrics.attributes() strictEqual(serialized.includes("Observe example events"), false) strictEqual(serialized.includes("01K20EXAMPLERECORD42"), false) strictEqual(serialized.includes("example-record-observed"), false) strictEqual(serialized.includes("example.record.sequence"), false) } finally { - store.close() + await store.close() } })) it("normalizes typed generic OTLP fields while preserving the existing warehouse encoding", () => { - const [signal] = normalizeOtlpLogs(exampleRecordObserved, "2026-08-07T20:00:00Z") + const [signal] = run(normalizeOtlpLogs(exampleRecordObserved, "2026-08-07T20:00:00Z")) strictEqual(signal?.occurrenceId, "01K20EXAMPLERECORD42") strictEqual(signal?.identityQuality, "source") strictEqual(signal?.source, "https://events.example.test") @@ -289,7 +286,7 @@ describe("LocalEventingRuntime", () => { attr("cloudevents.id", { stringValue: " cloud-event-42 " }), ...aliasedRecord.attributes.filter(({ key }) => !["event.id", "cloudevents.id"].includes(key)), ] - const [aliasedSignal] = normalizeOtlpLogs(aliased, "2026-08-07T20:00:00Z") + const [aliasedSignal] = run(normalizeOtlpLogs(aliased, "2026-08-07T20:00:00Z")) strictEqual(aliasedSignal?.occurrenceId, "cloud-event-42") strictEqual(aliasedSignal?.identityQuality, "source") @@ -302,8 +299,8 @@ describe("LocalEventingRuntime", () => { ) const derivedB = structuredClone(derivedA) firstLogRecord(derivedB).body = { stringValue: "A different record occurrence" } - const [signalA] = normalizeOtlpLogs(derivedA, "2026-08-07T20:00:00Z") - const [signalB] = normalizeOtlpLogs(derivedB, "2026-08-07T20:00:00Z") + const [signalA] = run(normalizeOtlpLogs(derivedA, "2026-08-07T20:00:00Z")) + const [signalB] = run(normalizeOtlpLogs(derivedB, "2026-08-07T20:00:00Z")) strictEqual(signalA?.identityQuality, "derived") strictEqual(signalB?.identityQuality, "derived") strictEqual(signalA?.occurrenceId?.startsWith("derived:sha256:"), true) @@ -311,8 +308,8 @@ describe("LocalEventingRuntime", () => { }) it("keeps projectable retries byte-identical and skips timestamp-less durable logs", () => { - const first = normalizeOtlpLogs(exampleRecordObserved, "2026-08-07T20:00:00Z") - const retry = normalizeOtlpLogs(exampleRecordObserved, "2026-08-08T20:00:00Z") + const first = run(normalizeOtlpLogs(exampleRecordObserved, "2026-08-07T20:00:00Z")) + const retry = run(normalizeOtlpLogs(exampleRecordObserved, "2026-08-08T20:00:00Z")) deepStrictEqual(first, retry) const timestampLess = structuredClone(exampleRecordObserved) @@ -322,9 +319,10 @@ describe("LocalEventingRuntime", () => { } delete timestampLessRecord.timeUnixNano delete timestampLessRecord.observedTimeUnixNano - deepStrictEqual(normalizeOtlpLogs(timestampLess, "2026-08-07T20:00:00Z"), []) + deepStrictEqual(run(normalizeOtlpLogs(timestampLess, "2026-08-07T20:00:00Z")), []) deepStrictEqual( - normalizeOtlpLogsWithDiagnostics(timestampLess, "2026-08-07T20:00:00Z").unprojectedIdentities, + run(normalizeOtlpLogsWithDiagnostics(timestampLess, "2026-08-07T20:00:00Z")) + .unprojectedIdentities, [ { sourceKind: "otel.log", @@ -338,15 +336,18 @@ describe("LocalEventingRuntime", () => { }) it("uses a locale-independent source-fingerprint field order", () => { - const [signal] = normalizeOtlpLogs(exampleRecordObserved, "2026-08-07T20:00:00Z") + const [signal] = run(normalizeOtlpLogs(exampleRecordObserved, "2026-08-07T20:00:00Z")) const fields = new Map(signal!.fields) fields.set("attribute:ä", { type: "string", value: "umlaut" }) fields.set("attribute:z", { type: "string", value: "ascii" }) const forward = { ...signal!, fields } const reverse = { ...signal!, fields: new Map([...fields].reverse()) } - strictEqual(sourceOccurrenceFingerprint(forward), sourceOccurrenceFingerprint(reverse)) strictEqual( - sourceOccurrenceFingerprint(forward), + Result.getOrThrow(sourceOccurrenceFingerprint(forward)), + Result.getOrThrow(sourceOccurrenceFingerprint(reverse)), + ) + strictEqual( + Result.getOrThrow(sourceOccurrenceFingerprint(forward)), "sha256:4ed4d210645f2df1959e5c56acb5b22140a01aa267fdf1fab8b62e56ea63e31e", ) }) @@ -361,7 +362,7 @@ describe("LocalEventingRuntime", () => { kvlistValue: { values: [attr("__proto__", { stringValue: "nested" })] }, }), ) - const [signal] = normalizeOtlpLogs(request, "2026-08-07T20:00:00Z") + const [signal] = run(normalizeOtlpLogs(request, "2026-08-07T20:00:00Z")) const record = (signal!.data as { record: { attributes: Record } }).record ok(Object.prototype.hasOwnProperty.call(record.attributes, "__proto__")) // Attribute maps are null-prototype on purpose, so a `__proto__` key stays data. @@ -377,43 +378,47 @@ describe("LocalEventingRuntime", () => { it("catalogs only the scalar body field that the OTLP adapter can populate", async () => withDataDir(async (dataDir) => { - const store = await LocalEventingControlStore.open(dataDir) + const store = await openStore(dataDir) try { - const runtime = new LocalEventingRuntime(store, undefined, exampleProjectors()) + const runtime = makeRuntime(store, exampleProjectors()) throws( () => - runtime.prepareActivation( - projection({ - selector: { - op: "exists", - field: { namespace: "body", key: "text", type: "string" }, - }, - }), + run( + runtime.prepareActivation( + projection({ + selector: { + op: "exists", + field: { namespace: "body", key: "text", type: "string" }, + }, + }), + ), ), /unknown field body:text/, ) - const activation = runtime.prepareActivation( - projection({ - selector: { - op: "exists", - field: { namespace: "body", key: "value", type: "boolean" }, - }, - }), + const activation = run( + runtime.prepareActivation( + projection({ + selector: { + op: "exists", + field: { namespace: "body", key: "value", type: "boolean" }, + }, + }), + ), ) strictEqual(activation.spec.selector.op, "exists") } finally { - store.close() + await store.close() } })) it("projects before storage, deduplicates retry delivery, and makes the event ready after commit", async () => withDataDir(async (dataDir) => { - const store = await LocalEventingControlStore.open(dataDir) + const store = await openStore(dataDir) try { - const runtime = new LocalEventingRuntime(store, undefined, exampleProjectors()) - strictEqual(runtime.hasActiveSource("otel.log"), false) - runtime.activate(projection()) - const first = runtime.evaluateOtlp("logs", exampleRecordObserved) + const runtime = makeRuntime(store, exampleProjectors()) + strictEqual(run(runtime.hasActiveSource("otel.log")), false) + run(runtime.activate(projection())) + const first = run(runtime.evaluateOtlp("logs", exampleRecordObserved)) strictEqual(first.failures.length, 0) strictEqual(first.events.length, 1) deepStrictEqual(first.events[0], { @@ -444,14 +449,14 @@ describe("LocalEventingRuntime", () => { serviceName: "example-service", }, }) - const staged = runtime.stage(first.events, first.eventSourceFingerprints) + const staged = run(runtime.stage(first.events, first.eventSourceFingerprints)) strictEqual(staged.inserted, 1) - strictEqual(runtime.listReady().events.length, 0) + strictEqual(run(runtime.listReady()).events.length, 0) deepStrictEqual( - runtime.listStaged().events.map(({ event }) => event), + run(runtime.listStaged()).events.map(({ event }) => event), first.events, ) - runtime.activate(projection({ revision: 2, enabled: false })) + run(runtime.activate(projection({ revision: 2, enabled: false }))) const projectionIneligibleRetry = structuredClone(exampleRecordObserved) firstLogRecord(projectionIneligibleRetry).attributes.push( ...Array.from({ length: 257 }, (_, index) => @@ -459,39 +464,39 @@ describe("LocalEventingRuntime", () => { ), ) throws( - () => runtime.evaluateOtlp("logs", projectionIneligibleRetry, () => true), + () => run(runtime.evaluateOtlp("logs", projectionIneligibleRetry, () => true)), /cannot safely recover staged source occurrence/, ) - strictEqual(runtime.listStaged().events.length, 1) - strictEqual(runtime.listReady().events.length, 0) + strictEqual(run(runtime.listStaged()).events.length, 1) + strictEqual(run(runtime.listReady()).events.length, 0) const changedRetry = structuredClone(exampleRecordObserved) firstLogRecord(changedRetry).body = { stringValue: "changed retry content" } throws( - () => runtime.evaluateOtlp("logs", changedRetry, () => true), + () => run(runtime.evaluateOtlp("logs", changedRetry, () => true)), /staged source occurrence collision/, ) - strictEqual(runtime.listStaged().events.length, 1) - strictEqual(runtime.listReady().events.length, 0) - const retry = runtime.evaluateOtlp("logs", exampleRecordObserved, () => true) + strictEqual(run(runtime.listStaged()).events.length, 1) + strictEqual(run(runtime.listReady()).events.length, 0) + const retry = run(runtime.evaluateOtlp("logs", exampleRecordObserved, () => true)) deepStrictEqual(retry.events, []) deepStrictEqual(retry.recoveredEventIds, staged.eventIds) - runtime.markReady(retry.recoveredEventIds) + run(runtime.markReady(retry.recoveredEventIds)) deepStrictEqual( - runtime.listReady().events.map(({ event }) => event), + run(runtime.listReady()).events.map(({ event }) => event), first.events, ) - deepStrictEqual(runtime.listStaged().events, []) + deepStrictEqual(run(runtime.listStaged()).events, []) } finally { - store.close() + await store.close() } })) it("rejects same event bytes with conflicting source content within one batch", async () => withDataDir(async (dataDir) => { - const store = await LocalEventingControlStore.open(dataDir) + const store = await openStore(dataDir) try { - const runtime = new LocalEventingRuntime(store, undefined, exampleProjectors()) - runtime.activate(projection()) + const runtime = makeRuntime(store, exampleProjectors()) + run(runtime.activate(projection())) const request = structuredClone(exampleRecordObserved) const first = firstLogRecord(request) first.attributes.push(attr("example.projector.ignored", { stringValue: "first" })) @@ -503,43 +508,43 @@ describe("LocalEventingRuntime", () => { ) request.resourceLogs[0]!.scopeLogs[0]!.logRecords.push(second) throws( - () => runtime.evaluateOtlp("logs", request), + () => run(runtime.evaluateOtlp("logs", request)), /source occurrence collision within one ingest batch/, ) - strictEqual(runtime.listStaged().events.length, 0) - strictEqual(runtime.listReady().events.length, 0) + strictEqual(run(runtime.listStaged()).events.length, 0) + strictEqual(run(runtime.listReady()).events.length, 0) } finally { - store.close() + await store.close() } })) it("rejects matching and nonmatching records that reuse one source occurrence", async () => withDataDir(async (dataDir) => { - const store = await LocalEventingControlStore.open(dataDir) + const store = await openStore(dataDir) try { - const runtime = new LocalEventingRuntime(store, undefined, exampleProjectors()) - runtime.activate(eventNameProjection("observed-only", "example.record.observed")) + const runtime = makeRuntime(store, exampleProjectors()) + run(runtime.activate(eventNameProjection("observed-only", "example.record.observed"))) const request = structuredClone(exampleRecordObserved) const sibling = structuredClone(firstLogRecord(request)) sibling.eventName = "example.record.ignored" request.resourceLogs[0]!.scopeLogs[0]!.logRecords.push(sibling) throws( - () => runtime.evaluateOtlp("logs", request), + () => run(runtime.evaluateOtlp("logs", request)), /source occurrence collision within one ingest batch/, ) - strictEqual(runtime.listStaged().events.length, 0) - strictEqual(runtime.listReady().events.length, 0) + strictEqual(run(runtime.listStaged()).events.length, 0) + strictEqual(run(runtime.listReady()).events.length, 0) } finally { - store.close() + await store.close() } })) it("rejects projectable and projection-ineligible records with one source occurrence", async () => withDataDir(async (dataDir) => { - const store = await LocalEventingControlStore.open(dataDir) + const store = await openStore(dataDir) try { - const runtime = new LocalEventingRuntime(store, undefined, exampleProjectors()) - runtime.activate(projection()) + const runtime = makeRuntime(store, exampleProjectors()) + run(runtime.activate(projection())) const request = structuredClone(exampleRecordObserved) const sibling = structuredClone(firstLogRecord(request)) sibling.attributes.push( @@ -549,73 +554,75 @@ describe("LocalEventingRuntime", () => { ) request.resourceLogs[0]!.scopeLogs[0]!.logRecords.push(sibling) throws( - () => runtime.evaluateOtlp("logs", request), + () => run(runtime.evaluateOtlp("logs", request)), /source occurrence collision with an unprojectable record within one ingest batch/, ) - strictEqual(runtime.listStaged().events.length, 0) - strictEqual(runtime.listReady().events.length, 0) + strictEqual(run(runtime.listStaged()).events.length, 0) + strictEqual(run(runtime.listReady()).events.length, 0) } finally { - store.close() + await store.close() } })) it("rejects disjoint projections over conflicting records with one source occurrence", async () => withDataDir(async (dataDir) => { - const store = await LocalEventingControlStore.open(dataDir) + const store = await openStore(dataDir) try { - const runtime = new LocalEventingRuntime(store, undefined, exampleProjectors()) - runtime.activate(eventNameProjection("observed-events", "example.record.observed")) - runtime.activate(eventNameProjection("alternate-events", "example.record.alternate")) + const runtime = makeRuntime(store, exampleProjectors()) + run(runtime.activate(eventNameProjection("observed-events", "example.record.observed"))) + run(runtime.activate(eventNameProjection("alternate-events", "example.record.alternate"))) const request = structuredClone(exampleRecordObserved) const sibling = structuredClone(firstLogRecord(request)) sibling.eventName = "example.record.alternate" request.resourceLogs[0]!.scopeLogs[0]!.logRecords.push(sibling) throws( - () => runtime.evaluateOtlp("logs", request), + () => run(runtime.evaluateOtlp("logs", request)), /source occurrence collision within one ingest batch/, ) - strictEqual(runtime.listStaged().events.length, 0) - strictEqual(runtime.listReady().events.length, 0) + strictEqual(run(runtime.listStaged()).events.length, 0) + strictEqual(run(runtime.listReady()).events.length, 0) } finally { - store.close() + await store.close() } })) it("activates a validated revision without restart and reloads it after restart", async () => withDataDir(async (dataDir) => { - let store = await LocalEventingControlStore.open(dataDir) - let runtime = new LocalEventingRuntime(store, undefined, exampleProjectors()) - runtime.activate(projection()) - strictEqual(runtime.evaluateOtlp("logs", exampleRecordObserved).events.length, 1) - runtime.activate( - projection({ - revision: 2, - selector: { - op: "eq", - field: { namespace: "signal", key: "event.name", type: "string" }, - value: { type: "string", value: "example.record.closed" }, - }, - }), + let store = await openStore(dataDir) + let runtime = makeRuntime(store, exampleProjectors()) + run(runtime.activate(projection())) + strictEqual(run(runtime.evaluateOtlp("logs", exampleRecordObserved)).events.length, 1) + run( + runtime.activate( + projection({ + revision: 2, + selector: { + op: "eq", + field: { namespace: "signal", key: "event.name", type: "string" }, + value: { type: "string", value: "example.record.closed" }, + }, + }), + ), ) - strictEqual(runtime.evaluateOtlp("logs", exampleRecordObserved).events.length, 0) - store.close() + strictEqual(run(runtime.evaluateOtlp("logs", exampleRecordObserved)).events.length, 0) + await store.close() - store = await LocalEventingControlStore.open(dataDir) + store = await openStore(dataDir) try { - runtime = new LocalEventingRuntime(store, undefined, exampleProjectors()) - strictEqual(runtime.listActive()[0]?.revision, 2) - strictEqual(runtime.evaluateOtlp("logs", exampleRecordObserved).events.length, 0) + runtime = makeRuntime(store, exampleProjectors()) + strictEqual(run(runtime.listActive)[0]?.revision, 2) + strictEqual(run(runtime.evaluateOtlp("logs", exampleRecordObserved)).events.length, 0) } finally { - store.close() + await store.close() } })) it("does no normalization or event work for a source with no active projection", async () => withDataDir(async (dataDir) => { - const store = await LocalEventingControlStore.open(dataDir) + const store = await openStore(dataDir) try { - const runtime = new LocalEventingRuntime(store) - deepStrictEqual(runtime.evaluateOtlp("logs", { malformed: Symbol("not decoded") }), { + const runtime = makeRuntime(store) + deepStrictEqual(run(runtime.evaluateOtlp("logs", { malformed: Symbol("not decoded") })), { events: [], eventSourceFingerprints: new Map(), recoveredEventIds: [], @@ -623,7 +630,7 @@ describe("LocalEventingRuntime", () => { typeMismatchFields: [], }) } finally { - store.close() + await store.close() } })) }) diff --git a/apps/cli/test/local-eventing-telemetry.test.ts b/apps/cli/test/local-eventing-telemetry.test.ts index 466128b7e..0d0eaa309 100644 --- a/apps/cli/test/local-eventing-telemetry.test.ts +++ b/apps/cli/test/local-eventing-telemetry.test.ts @@ -2,7 +2,7 @@ import { strictEqual, ok } from "node:assert" import { describe, it } from "vitest" import { Effect, ManagedRuntime } from "effect" import { Maple } from "@maple-dev/effect-sdk/server" -import { makeEffectEventingTelemetry } from "../src/server/eventing/telemetry" +import { observeEventing } from "../src/server/eventing/telemetry" describe("eventing metric export", () => { it("exports eventing counters through the CLI's server SDK layer", async () => { @@ -25,12 +25,9 @@ describe("eventing metric export", () => { }), ) try { - const pending: Promise[] = [] - const telemetry = makeEffectEventingTelemetry((effect) => - pending.push(runtime.runPromise(effect)), + await runtime.runPromise( + observeEventing({ operation: "outbox_stage", outcome: "success", count: 3 }), ) - telemetry.record({ operation: "outbox_stage", outcome: "success", count: 3 }) - await Promise.all(pending) await runtime.runPromise( Effect.repeat(Effect.sleep("10 millis"), { until: () => bodies.some((body) => body.includes("maple.eventing.operations_total")),